Refactor BER decoder into a suspendable coroutine

The goal of this change is to make the decoder stopping on input
data starvation and resuming from where it stopped whenever the
caller decides to try again (hopefully making sure that some more
input becomes available).

This change makes it possible for the decoder to operate on streams
of data (meaning that the entire DER blob might not be immediately
available on input).

On top of that, the decoder yields partially reconstructed ASN.1
object on input starvation making it possible for the caller to
inspect what has been decoded so far and possibly consume partial
ASN.1 data.

All these new feature are natively available through
`StreamingDecoder` class. Previously published API is implemented
as a thin wrapper on top of that ensuring backward compatibility.

1 files changed, 89 insertions, 70 deletions

diff --git a/pyasn1/codec/native/decoder.py b/pyasn1/codec/native/decoder.py
index 104b92e..ecb1b16 100644
--- a/pyasn1/codec/native/decoder.py
+++ b/pyasn1/codec/native/decoder.py
@@ -17,17 +17,17 @@ __all__ = ['decode']
 LOG = debug.registerLoggee(__name__, flags=debug.DEBUG_DECODER)
 
 
-class AbstractScalarDecoder(object):
+class AbstractScalarPayloadDecoder(object):
     def __call__(self, pyObject, asn1Spec, decodeFun=None, **options):
         return asn1Spec.clone(pyObject)
 
 
-class BitStringDecoder(AbstractScalarDecoder):
+class BitStringPayloadDecoder(AbstractScalarPayloadDecoder):
     def __call__(self, pyObject, asn1Spec, decodeFun=None, **options):
         return asn1Spec.clone(univ.BitString.fromBinaryString(pyObject))
 
 
-class SequenceOrSetDecoder(object):
+class SequenceOrSetPayloadDecoder(object):
     def __call__(self, pyObject, asn1Spec, decodeFun=None, **options):
         asn1Value = asn1Spec.clone()
 
@@ -40,7 +40,7 @@ class SequenceOrSetDecoder(object):
         return asn1Value
 
 
-class SequenceOfOrSetOfDecoder(object):
+class SequenceOfOrSetOfPayloadDecoder(object):
     def __call__(self, pyObject, asn1Spec, decodeFun=None, **options):
         asn1Value = asn1Spec.clone()
 
@@ -50,7 +50,7 @@ class SequenceOfOrSetOfDecoder(object):
         return asn1Value
 
 
-class ChoiceDecoder(object):
+class ChoicePayloadDecoder(object):
     def __call__(self, pyObject, asn1Spec, decodeFun=None, **options):
         asn1Value = asn1Spec.clone()
 
@@ -64,87 +64,92 @@ class ChoiceDecoder(object):
         return asn1Value
 
 
-tagMap = {
-    univ.Integer.tagSet: AbstractScalarDecoder(),
-    univ.Boolean.tagSet: AbstractScalarDecoder(),
-    univ.BitString.tagSet: BitStringDecoder(),
-    univ.OctetString.tagSet: AbstractScalarDecoder(),
-    univ.Null.tagSet: AbstractScalarDecoder(),
-    univ.ObjectIdentifier.tagSet: AbstractScalarDecoder(),
-    univ.Enumerated.tagSet: AbstractScalarDecoder(),
-    univ.Real.tagSet: AbstractScalarDecoder(),
-    univ.Sequence.tagSet: SequenceOrSetDecoder(),  # conflicts with SequenceOf
-    univ.Set.tagSet: SequenceOrSetDecoder(),  # conflicts with SetOf
-    univ.Choice.tagSet: ChoiceDecoder(),  # conflicts with Any
+TAG_MAP = {
+    univ.Integer.tagSet: AbstractScalarPayloadDecoder(),
+    univ.Boolean.tagSet: AbstractScalarPayloadDecoder(),
+    univ.BitString.tagSet: BitStringPayloadDecoder(),
+    univ.OctetString.tagSet: AbstractScalarPayloadDecoder(),
+    univ.Null.tagSet: AbstractScalarPayloadDecoder(),
+    univ.ObjectIdentifier.tagSet: AbstractScalarPayloadDecoder(),
+    univ.Enumerated.tagSet: AbstractScalarPayloadDecoder(),
+    univ.Real.tagSet: AbstractScalarPayloadDecoder(),
+    univ.Sequence.tagSet: SequenceOrSetPayloadDecoder(),  # conflicts with SequenceOf
+    univ.Set.tagSet: SequenceOrSetPayloadDecoder(),  # conflicts with SetOf
+    univ.Choice.tagSet: ChoicePayloadDecoder(),  # conflicts with Any
     # character string types
-    char.UTF8String.tagSet: AbstractScalarDecoder(),
-    char.NumericString.tagSet: AbstractScalarDecoder(),
-    char.PrintableString.tagSet: AbstractScalarDecoder(),
-    char.TeletexString.tagSet: AbstractScalarDecoder(),
-    char.VideotexString.tagSet: AbstractScalarDecoder(),
-    char.IA5String.tagSet: AbstractScalarDecoder(),
-    char.GraphicString.tagSet: AbstractScalarDecoder(),
-    char.VisibleString.tagSet: AbstractScalarDecoder(),
-    char.GeneralString.tagSet: AbstractScalarDecoder(),
-    char.UniversalString.tagSet: AbstractScalarDecoder(),
-    char.BMPString.tagSet: AbstractScalarDecoder(),
+    char.UTF8String.tagSet: AbstractScalarPayloadDecoder(),
+    char.NumericString.tagSet: AbstractScalarPayloadDecoder(),
+    char.PrintableString.tagSet: AbstractScalarPayloadDecoder(),
+    char.TeletexString.tagSet: AbstractScalarPayloadDecoder(),
+    char.VideotexString.tagSet: AbstractScalarPayloadDecoder(),
+    char.IA5String.tagSet: AbstractScalarPayloadDecoder(),
+    char.GraphicString.tagSet: AbstractScalarPayloadDecoder(),
+    char.VisibleString.tagSet: AbstractScalarPayloadDecoder(),
+    char.GeneralString.tagSet: AbstractScalarPayloadDecoder(),
+    char.UniversalString.tagSet: AbstractScalarPayloadDecoder(),
+    char.BMPString.tagSet: AbstractScalarPayloadDecoder(),
     # useful types
-    useful.ObjectDescriptor.tagSet: AbstractScalarDecoder(),
-    useful.GeneralizedTime.tagSet: AbstractScalarDecoder(),
-    useful.UTCTime.tagSet: AbstractScalarDecoder()
+    useful.ObjectDescriptor.tagSet: AbstractScalarPayloadDecoder(),
+    useful.GeneralizedTime.tagSet: AbstractScalarPayloadDecoder(),
+    useful.UTCTime.tagSet: AbstractScalarPayloadDecoder()
 }
 
 # Put in ambiguous & non-ambiguous types for faster codec lookup
-typeMap = {
-    univ.Integer.typeId: AbstractScalarDecoder(),
-    univ.Boolean.typeId: AbstractScalarDecoder(),
-    univ.BitString.typeId: BitStringDecoder(),
-    univ.OctetString.typeId: AbstractScalarDecoder(),
-    univ.Null.typeId: AbstractScalarDecoder(),
-    univ.ObjectIdentifier.typeId: AbstractScalarDecoder(),
-    univ.Enumerated.typeId: AbstractScalarDecoder(),
-    univ.Real.typeId: AbstractScalarDecoder(),
+TYPE_MAP = {
+    univ.Integer.typeId: AbstractScalarPayloadDecoder(),
+    univ.Boolean.typeId: AbstractScalarPayloadDecoder(),
+    univ.BitString.typeId: BitStringPayloadDecoder(),
+    univ.OctetString.typeId: AbstractScalarPayloadDecoder(),
+    univ.Null.typeId: AbstractScalarPayloadDecoder(),
+    univ.ObjectIdentifier.typeId: AbstractScalarPayloadDecoder(),
+    univ.Enumerated.typeId: AbstractScalarPayloadDecoder(),
+    univ.Real.typeId: AbstractScalarPayloadDecoder(),
     # ambiguous base types
-    univ.Set.typeId: SequenceOrSetDecoder(),
-    univ.SetOf.typeId: SequenceOfOrSetOfDecoder(),
-    univ.Sequence.typeId: SequenceOrSetDecoder(),
-    univ.SequenceOf.typeId: SequenceOfOrSetOfDecoder(),
-    univ.Choice.typeId: ChoiceDecoder(),
-    univ.Any.typeId: AbstractScalarDecoder(),
+    univ.Set.typeId: SequenceOrSetPayloadDecoder(),
+    univ.SetOf.typeId: SequenceOfOrSetOfPayloadDecoder(),
+    univ.Sequence.typeId: SequenceOrSetPayloadDecoder(),
+    univ.SequenceOf.typeId: SequenceOfOrSetOfPayloadDecoder(),
+    univ.Choice.typeId: ChoicePayloadDecoder(),
+    univ.Any.typeId: AbstractScalarPayloadDecoder(),
     # character string types
-    char.UTF8String.typeId: AbstractScalarDecoder(),
-    char.NumericString.typeId: AbstractScalarDecoder(),
-    char.PrintableString.typeId: AbstractScalarDecoder(),
-    char.TeletexString.typeId: AbstractScalarDecoder(),
-    char.VideotexString.typeId: AbstractScalarDecoder(),
-    char.IA5String.typeId: AbstractScalarDecoder(),
-    char.GraphicString.typeId: AbstractScalarDecoder(),
-    char.VisibleString.typeId: AbstractScalarDecoder(),
-    char.GeneralString.typeId: AbstractScalarDecoder(),
-    char.UniversalString.typeId: AbstractScalarDecoder(),
-    char.BMPString.typeId: AbstractScalarDecoder(),
+    char.UTF8String.typeId: AbstractScalarPayloadDecoder(),
+    char.NumericString.typeId: AbstractScalarPayloadDecoder(),
+    char.PrintableString.typeId: AbstractScalarPayloadDecoder(),
+    char.TeletexString.typeId: AbstractScalarPayloadDecoder(),
+    char.VideotexString.typeId: AbstractScalarPayloadDecoder(),
+    char.IA5String.typeId: AbstractScalarPayloadDecoder(),
+    char.GraphicString.typeId: AbstractScalarPayloadDecoder(),
+    char.VisibleString.typeId: AbstractScalarPayloadDecoder(),
+    char.GeneralString.typeId: AbstractScalarPayloadDecoder(),
+    char.UniversalString.typeId: AbstractScalarPayloadDecoder(),
+    char.BMPString.typeId: AbstractScalarPayloadDecoder(),
     # useful types
-    useful.ObjectDescriptor.typeId: AbstractScalarDecoder(),
-    useful.GeneralizedTime.typeId: AbstractScalarDecoder(),
-    useful.UTCTime.typeId: AbstractScalarDecoder()
+    useful.ObjectDescriptor.typeId: AbstractScalarPayloadDecoder(),
+    useful.GeneralizedTime.typeId: AbstractScalarPayloadDecoder(),
+    useful.UTCTime.typeId: AbstractScalarPayloadDecoder()
 }
 
 
-class Decoder(object):
+class SingleItemDecoder(object):
+
+    TAG_MAP = TAG_MAP
+    TYPE_MAP = TYPE_MAP
 
-    # noinspection PyDefaultArgument
-    def __init__(self, tagMap, typeMap):
-        self.__tagMap = tagMap
-        self.__typeMap = typeMap
+    def __init__(self, tagMap=None, typeMap=None):
+        self.__tagMap = tagMap or self.TAG_MAP
+        self.__typeMap = typeMap or self.TYPE_MAP
 
     def __call__(self, pyObject, asn1Spec, **options):
 
         if LOG:
             debug.scope.push(type(pyObject).__name__)
-            LOG('decoder called at scope %s, working with type %s' % (debug.scope, type(pyObject).__name__))
+            LOG('decoder called at scope %s, working with '
+                'type %s' % (debug.scope, type(pyObject).__name__))
 
         if asn1Spec is None or not isinstance(asn1Spec, base.Asn1Item):
-            raise error.PyAsn1Error('asn1Spec is not valid (should be an instance of an ASN.1 Item, not %s)' % asn1Spec.__class__.__name__)
+            raise error.PyAsn1Error(
+                'asn1Spec is not valid (should be an instance of an ASN.1 '
+                'Item, not %s)' % asn1Spec.__class__.__name__)
 
         try:
             valueDecoder = self.__typeMap[asn1Spec.typeId]
@@ -155,21 +160,35 @@ class Decoder(object):
 
             try:
                 valueDecoder = self.__tagMap[baseTagSet]
+
             except KeyError:
                 raise error.PyAsn1Error('Unknown ASN.1 tag %s' % asn1Spec.tagSet)
 
         if LOG:
-            LOG('calling decoder %s on Python type %s <%s>' % (type(valueDecoder).__name__, type(pyObject).__name__, repr(pyObject)))
+            LOG('calling decoder %s on Python type %s '
+                '<%s>' % (type(valueDecoder).__name__,
+                          type(pyObject).__name__, repr(pyObject)))
 
         value = valueDecoder(pyObject, asn1Spec, self, **options)
 
         if LOG:
-            LOG('decoder %s produced ASN.1 type %s <%s>' % (type(valueDecoder).__name__, type(value).__name__, repr(value)))
+            LOG('decoder %s produced ASN.1 type %s '
+                '<%s>' % (type(valueDecoder).__name__,
+                          type(value).__name__, repr(value)))
             debug.scope.pop()
 
         return value
 
 
+class Decoder(object):
+    SINGLE_ITEM_DECODER = SingleItemDecoder
+
+    @classmethod
+    def __call__(cls, pyObject, asn1Spec=None, **kwargs):
+        singleItemDecoder = cls.SINGLE_ITEM_DECODER()
+        return singleItemDecoder(pyObject, asn1Spec=asn1Spec, **kwargs)
+
+
 #: Turns Python objects of built-in types into ASN.1 objects.
 #:
 #: Takes Python objects of built-in types and turns them into a tree of
@@ -210,4 +229,4 @@ class Decoder(object):
 #:    SequenceOf:
 #:     1 2 3
 #:
-decode = Decoder(tagMap, typeMap)
+decode = Decoder()