Merge branch 'master' into open-types-support

14 files changed, 302 insertions, 22 deletions

diff --git a/CHANGES.rst b/CHANGES.rst
index 4d9d6e3..a896433 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -40,9 +40,14 @@ Revision 0.2.4, released XX-03-2017
 - OctetsString and Any decoder optimized to avoid creating ASN.1
   objects for chunks of substrate. Instead they now join substrate
   chunks together and create ASN.1 object from it just once.
+- Unit tests added for the `compat` sub-package.
 - Fixed BitString named bits initialization bug.
 - Fixed non-functional tag cache (when running Python 2) at DER decoder.
 - Fixed chunked encoding restriction on DER encoder.
+- Fixed SET components ordering at DER encoder.
+- Fixed BIT STRING & OCTET STRING encoding to be always non-chunked (e.g.
+  primitive) at DER encoder
+- Fixed `compat.integer.from_bytes()` behaviour on empty input.
 
 Revision 0.2.3, released 25-02-2017
 -----------------------------------
diff --git a/pyasn1/codec/cer/encoder.py b/pyasn1/codec/cer/encoder.py
index e241e43..d04df9d 100644
--- a/pyasn1/codec/cer/encoder.py
+++ b/pyasn1/codec/cer/encoder.py
@@ -85,6 +85,11 @@ class UTCTimeEncoder(encoder.OctetStringEncoder):
 
 
 class SetOfEncoder(encoder.SequenceOfEncoder):
+    @staticmethod
+    def _sortComponents(components):
+        # sort by tags regardless of the Choice value (static sort)
+        return sorted(components, key=lambda x: isinstance(x, univ.Choice) and x.minTagSet or x.tagSet)
+
     def encodeValue(self, encodeFun, client, defMode, maxChunkSize):
         client.verifySizeSpec()
         substrate = null
@@ -102,9 +107,8 @@ class SetOfEncoder(encoder.SequenceOfEncoder):
                 if namedTypes[idx].isDefaulted and client[idx] == namedTypes[idx].asn1Object:
                     continue
                 comps.append(client[idx])
-            comps.sort(key=lambda x: isinstance(x, univ.Choice) and x.getMinTagSet() or x.tagSet)
-            for c in comps:
-                substrate += encodeFun(c, defMode, maxChunkSize)
+            for comp in self._sortComponents(comps):
+                substrate += encodeFun(comp, defMode, maxChunkSize)
         else:
             # SetOf
             compSubs = []
@@ -128,7 +132,7 @@ tagMap.update({
     univ.Real.tagSet: RealEncoder(),
     useful.GeneralizedTime.tagSet: GeneralizedTimeEncoder(),
     useful.UTCTime.tagSet: UTCTimeEncoder(),
-    univ.SetOf().tagSet: SetOfEncoder()  # conflcts with Set
+    univ.SetOf.tagSet: SetOfEncoder()  # conflcts with Set
 })
 
 typeMap = encoder.typeMap.copy()
@@ -145,6 +149,7 @@ typeMap.update({
 
 
 class Encoder(encoder.Encoder):
+
     def __call__(self, client, defMode=False, maxChunkSize=0):
         return encoder.Encoder.__call__(self, client, defMode, maxChunkSize)
 
diff --git a/pyasn1/codec/der/encoder.py b/pyasn1/codec/der/encoder.py
index f573228..3751429 100644
--- a/pyasn1/codec/der/encoder.py
+++ b/pyasn1/codec/der/encoder.py
@@ -11,24 +11,40 @@ from pyasn1 import error
 __all__ = ['encode']
 
 
+class BitStringEncoder(encoder.BitStringEncoder):
+    def encodeValue(self, encodeFun, client, defMode, maxChunkSize):
+        return encoder.BitStringEncoder.encodeValue(
+            self, encodeFun, client, defMode, 0
+        )
+
+class OctetStringEncoder(encoder.OctetStringEncoder):
+    def encodeValue(self, encodeFun, client, defMode, maxChunkSize):
+        return encoder.OctetStringEncoder.encodeValue(
+            self, encodeFun, client, defMode, 0
+        )
+
 class SetOfEncoder(encoder.SetOfEncoder):
     @staticmethod
-    def _cmpSetComponents(c1, c2):
-        tagSet1 = isinstance(c1, univ.Choice) and c1.effectiveTagSet or c1.tagSet
-        tagSet2 = isinstance(c2, univ.Choice) and c2.effectiveTagSet or c2.tagSet
-        return cmp(tagSet1, tagSet2)
-
+    def _sortComponents(components):
+        # sort by tags depending on the actual Choice value (dynamic sort)
+        return sorted(components, key=lambda x: isinstance(x, univ.Choice) and x.getComponent().tagSet or x.tagSet)
 
 tagMap = encoder.tagMap.copy()
 tagMap.update({
-    # Overload CER encoders with BER ones (a bit hackerish XXX)
-    univ.BitString.tagSet: encoder.encoder.BitStringEncoder(),
-    univ.OctetString.tagSet: encoder.encoder.OctetStringEncoder(),
+    univ.BitString.tagSet: BitStringEncoder(),
+    univ.OctetString.tagSet: OctetStringEncoder(),
     # Set & SetOf have same tags
-    univ.SetOf().tagSet: SetOfEncoder()
+    univ.SetOf.tagSet: SetOfEncoder()
 })
 
 typeMap = encoder.typeMap.copy()
+typeMap.update({
+    univ.BitString.typeId: BitStringEncoder(),
+    univ.OctetString.typeId: OctetStringEncoder(),
+    # Set & SetOf have same tags
+    univ.Set.typeId: SetOfEncoder(),
+    univ.SetOf.typeId: SetOfEncoder()
+})
 
 
 class Encoder(encoder.Encoder):
diff --git a/pyasn1/compat/integer.py b/pyasn1/compat/integer.py
index f1d1ef6..bb53e64 100644
--- a/pyasn1/compat/integer.py
+++ b/pyasn1/compat/integer.py
@@ -18,6 +18,9 @@ from pyasn1.compat.octets import oct2int, null
 
 if sys.version_info[0:2] < (3, 2) or implementation != 'CPython':
     def from_bytes(octets, signed=False):
+        if not octets:
+            return 0
+
         value = long(b2a_hex(str(octets)), 16)
 
         if signed and oct2int(octets[0]) & 0x80:
diff --git a/pyasn1/compat/octets.py b/pyasn1/compat/octets.py
index 6cf018e..f8d4708 100644
--- a/pyasn1/compat/octets.py
+++ b/pyasn1/compat/octets.py
@@ -12,6 +12,7 @@ if version_info[0] <= 2:
     ints2octs = lambda s: ''.join([int2oct(x) for x in s])
     null = ''
     oct2int = ord
+    # TODO: refactor to return a sequence of ints
     # noinspection PyPep8
     octs2ints = lambda s: [oct2int(x) for x in s]
     # noinspection PyPep8
diff --git a/pyasn1/type/namedtype.py b/pyasn1/type/namedtype.py
index 710ce93..f5b99e4 100644
--- a/pyasn1/type/namedtype.py
+++ b/pyasn1/type/namedtype.py
@@ -410,7 +410,7 @@ class NamedTypes(object):
             for namedType in self.__namedTypes:
                 asn1Object = namedType.asn1Object
                 try:
-                    tagSet = asn1Object.getMinTagSet()
+                    tagSet = asn1Object.minTagSet
 
                 except AttributeError:
                     tagSet = asn1Object.tagSet
diff --git a/pyasn1/type/univ.py b/pyasn1/type/univ.py
index 595192a..db0c4bf 100644
--- a/pyasn1/type/univ.py
+++ b/pyasn1/type/univ.py
@@ -2694,7 +2694,8 @@ class Choice(Set):
             self._componentValues[oldIdx] = None
         return self
 
-    def getMinTagSet(self):
+    @property
+    def minTagSet(self):
         if self._tagSet:
             return self._tagSet
         else:
@@ -2776,6 +2777,12 @@ class Choice(Set):
         return self._componentValues[self._currentIdx].isValue
 
 
+    # compatibility stubs
+
+    def getMinTagSet(self):
+        return self.minTagSet
+
+
 class Any(OctetString):
     __doc__ = OctetString.__doc__
 
diff --git a/tests/__main__.py b/tests/__main__.py
index 70bd9ea..c5c152a 100644
--- a/tests/__main__.py
+++ b/tests/__main__.py
@@ -13,7 +13,8 @@ except ImportError:
 suite = unittest.TestLoader().loadTestsFromNames(
     ['tests.test_debug.suite',
      'tests.type.__main__.suite',
-     'tests.codec.__main__.suite']
+     'tests.codec.__main__.suite',
+     'tests.compat.__main__.suite']
 )
 
 
diff --git a/tests/codec/der/test_encoder.py b/tests/codec/der/test_encoder.py
index 0611dc8..a148763 100644
--- a/tests/codec/der/test_encoder.py
+++ b/tests/codec/der/test_encoder.py
@@ -47,19 +47,22 @@ class SetWithChoiceEncoderTestCase(unittest.TestCase):
     def setUp(self):
         c = univ.Choice(componentType=namedtype.NamedTypes(
             namedtype.NamedType('name', univ.OctetString()),
-            namedtype.NamedType('amount', univ.Integer(0)))
+            namedtype.NamedType('amount', univ.Boolean()))
         )
         self.s = univ.Set(componentType=namedtype.NamedTypes(
-            namedtype.NamedType('place-holder', univ.Null()),
+            namedtype.NamedType('value', univ.Integer(5)),
             namedtype.NamedType('status', c))
         )
 
-    def testDefMode(self):
-        self.s.setComponentByPosition(0, '')
+    def testComponentsOrdering1(self):
         self.s.setComponentByName('status')
-        self.s.getComponentByName('status').setComponentByPosition(0, 'ann')
-        assert encoder.encode(self.s) == ints2octs((49, 7, 4, 3, 97, 110, 110, 5, 0))
+        self.s.getComponentByName('status').setComponentByPosition(0, 'A')
+        assert encoder.encode(self.s) == ints2octs((49, 6, 2, 1, 5, 4, 1, 65))
 
+    def testComponentsOrdering2(self):
+        self.s.setComponentByName('status')
+        self.s.getComponentByName('status').setComponentByPosition(1, True)
+        assert encoder.encode(self.s) == ints2octs((49, 6, 1, 1, 255, 2, 1, 5))
 
 suite = unittest.TestLoader().loadTestsFromModule(sys.modules[__name__])
 
diff --git a/tests/compat/__init__.py b/tests/compat/__init__.py
new file mode 100644
index 0000000..8c3066b
--- /dev/null
+++ b/tests/compat/__init__.py
@@ -0,0 +1 @@
+# This file is necessary to make this directory a package.
diff --git a/tests/compat/__main__.py b/tests/compat/__main__.py
new file mode 100644
index 0000000..ce26562
--- /dev/null
+++ b/tests/compat/__main__.py
@@ -0,0 +1,21 @@
+#
+# This file is part of pyasn1 software.
+#
+# Copyright (c) 2005-2017, Ilya Etingof <etingof@gmail.com>
+# License: http://pyasn1.sf.net/license.html
+#
+try:
+    import unittest2 as unittest
+
+except ImportError:
+    import unittest
+
+suite = unittest.TestLoader().loadTestsFromNames(
+    ['tests.compat.test_binary.suite',
+     'tests.compat.test_integer.suite',
+     'tests.compat.test_octets.suite']
+)
+
+
+if __name__ == '__main__':
+    unittest.TextTestRunner(verbosity=2).run(suite)
diff --git a/tests/compat/test_binary.py b/tests/compat/test_binary.py
new file mode 100644
index 0000000..7660206
--- /dev/null
+++ b/tests/compat/test_binary.py
@@ -0,0 +1,53 @@
+#
+# This file is part of pyasn1 software.
+#
+# Copyright (c) 2005-2017, Ilya Etingof <etingof@gmail.com>
+# License: http://pyasn1.sf.net/license.html
+#
+import sys
+from pyasn1.compat import binary
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
+
+
+class BinaryTestCase(unittest.TestCase):
+
+    def test_bin_zero(self):
+        assert '0b0' == binary.bin(0)
+
+
+    def test_bin_noarg(self):
+        try:
+            binary.bin()
+
+        except TypeError:
+            pass
+
+        except:
+            assert 0, 'bin() tolerates no arguments'
+
+
+    def test_bin_allones(self):
+        assert '0b1111111111111111111111111111111111111111111111111111111111111111' == binary.bin(0xffffffffffffffff)
+
+
+    def test_bin_allzeros(self):
+        assert '0b0' == binary.bin(0x0000000)
+
+
+
+    def test_bin_pos(self):
+        assert '0b1000000010000000100000001' == binary.bin(0x01010101)
+
+
+    def test_bin_neg(self):
+        assert '-0b1000000010000000100000001' == binary.bin(-0x01010101)
+
+
+suite = unittest.TestLoader().loadTestsFromModule(sys.modules[__name__])
+
+if __name__ == '__main__':
+    unittest.TextTestRunner(verbosity=2).run(suite)
diff --git a/tests/compat/test_integer.py b/tests/compat/test_integer.py
new file mode 100644
index 0000000..9179641
--- /dev/null
+++ b/tests/compat/test_integer.py
@@ -0,0 +1,50 @@
+#
+# This file is part of pyasn1 software.
+#
+# Copyright (c) 2005-2017, Ilya Etingof <etingof@gmail.com>
+# License: http://pyasn1.sf.net/license.html
+#
+import sys
+from pyasn1.compat import integer
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
+
+
+class IntegerTestCase(unittest.TestCase):
+
+    if sys.version_info[0] > 2:
+
+        def test_from_bytes_zero(self):
+            assert 0 == integer.from_bytes(bytes([0]), signed=False)
+
+        def test_from_bytes_unsigned(self):
+            assert -66051 == integer.from_bytes(bytes([254, 253, 253]), signed=True)
+
+        def test_from_bytes_signed(self):
+            assert 66051 == integer.from_bytes(bytes([0, 1, 2, 3]), signed=False)
+
+        def test_from_bytes_empty(self):
+            assert 0 == integer.from_bytes(bytes([]))
+
+    else:
+
+        def test_from_bytes_zero(self):
+            assert 0 == integer.from_bytes('\x00', signed=False)
+
+        def test_from_bytes_unsigned(self):
+            assert -66051 == integer.from_bytes('\xfe\xfd\xfd', signed=True)
+
+        def test_from_bytes_signed(self):
+            assert 66051 == integer.from_bytes('\x01\x02\x03', signed=False)
+
+        def test_from_bytes_empty(self):
+            assert 0 == integer.from_bytes('')
+
+
+suite = unittest.TestLoader().loadTestsFromModule(sys.modules[__name__])
+
+if __name__ == '__main__':
+    unittest.TextTestRunner(verbosity=2).run(suite)
diff --git a/tests/compat/test_octets.py b/tests/compat/test_octets.py
new file mode 100644
index 0000000..4652b33
--- /dev/null
+++ b/tests/compat/test_octets.py
@@ -0,0 +1,114 @@
+#
+# This file is part of pyasn1 software.
+#
+# Copyright (c) 2005-2017, Ilya Etingof <etingof@gmail.com>
+# License: http://pyasn1.sf.net/license.html
+#
+import sys
+from pyasn1.compat import octets
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
+
+
+class OctetsTestCase(unittest.TestCase):
+
+    if sys.version_info[0] > 2:
+
+        def test_ints2octs(self):
+            assert [1, 2, 3] == list(octets.ints2octs([1, 2, 3]))
+
+        def test_ints2octs_empty(self):
+            assert not octets.ints2octs([])
+
+        def test_int2oct(self):
+            assert [12] == list(octets.int2oct(12))
+
+        def test_octs2ints(self):
+            assert [1, 2, 3] == list(octets.octs2ints(bytes([1, 2, 3])))
+
+        def test_octs2ints_empty(self):
+            assert not octets.octs2ints(bytes([]))
+
+        def test_oct2int(self):
+            assert 12 == octets.oct2int(bytes([12]))[0]
+
+        def test_str2octs(self):
+            assert bytes([1, 2, 3]) == octets.str2octs('\x01\x02\x03')
+
+        def test_str2octs_empty(self):
+            assert not octets.str2octs('')
+
+        def test_octs2str(self):
+            assert '\x01\x02\x03' == octets.octs2str(bytes([1, 2, 3]))
+
+        def test_octs2str_empty(self):
+            assert not octets.octs2str(bytes([]))
+
+        def test_isOctetsType(self):
+            assert octets.isOctetsType('abc') == False
+            assert octets.isOctetsType(123) == False
+            assert octets.isOctetsType(bytes()) == True
+
+        def test_isStringType(self):
+            assert octets.isStringType('abc') == True
+            assert octets.isStringType(123) == False
+            assert octets.isStringType(bytes()) == False
+
+        def test_ensureString(self):
+            assert 'abc'.encode() == octets.ensureString('abc'.encode())
+            assert bytes([1, 2, 3]) == octets.ensureString([1, 2, 3])
+
+    else:
+
+        def test_ints2octs(self):
+            assert '\x01\x02\x03' == octets.ints2octs([1, 2, 3])
+
+        def test_ints2octs_empty(self):
+            assert not octets.ints2octs([])
+
+        def test_int2oct(self):
+            assert '\x0c' == octets.int2oct(12)
+
+        def test_octs2ints(self):
+            assert [1, 2, 3] == octets.octs2ints('\x01\x02\x03')
+
+        def test_octs2ints_empty(self):
+            assert not octets.octs2ints('')
+
+        def test_oct2int(self):
+            assert 12 == octets.oct2int('\x0c')
+
+        def test_str2octs(self):
+            assert '\x01\x02\x03' == octets.str2octs('\x01\x02\x03')
+
+        def test_str2octs_empty(self):
+            assert not octets.str2octs('')
+
+        def test_octs2str(self):
+            assert '\x01\x02\x03' == octets.octs2str('\x01\x02\x03')
+
+        def test_octs2str_empty(self):
+            assert not octets.octs2str('')
+
+        def test_isOctetsType(self):
+            assert octets.isOctetsType('abc') == True
+            assert octets.isOctetsType(123) == False
+            assert octets.isOctetsType(unicode('abc')) == False
+
+        def test_isStringType(self):
+            assert octets.isStringType('abc') == True
+            assert octets.isStringType(123) == False
+            assert octets.isStringType(unicode('abc')) == True
+
+        def test_ensureString(self):
+            assert 'abc' == octets.ensureString('abc')
+            assert '123' == octets.ensureString(123)
+
+
+suite = unittest.TestLoader().loadTestsFromModule(sys.modules[__name__])
+
+if __name__ == '__main__':
+    unittest.TextTestRunner(verbosity=2).run(suite)