diff options
Diffstat (limited to 'pyasn1/codec')
| -rw-r--r-- | pyasn1/codec/ber/decoder.py | 1188 | ||||
| -rw-r--r-- | pyasn1/codec/ber/encoder.py | 54 | ||||
| -rw-r--r-- | pyasn1/codec/cer/decoder.py | 72 | ||||
| -rw-r--r-- | pyasn1/codec/cer/encoder.py | 24 | ||||
| -rw-r--r-- | pyasn1/codec/der/decoder.py | 48 | ||||
| -rw-r--r-- | pyasn1/codec/der/encoder.py | 25 | ||||
| -rw-r--r-- | pyasn1/codec/native/decoder.py | 164 | ||||
| -rw-r--r-- | pyasn1/codec/native/encoder.py | 48 | ||||
| -rw-r--r-- | pyasn1/codec/streaming.py | 243 |
9 files changed, 1309 insertions, 557 deletions
diff --git a/pyasn1/codec/ber/decoder.py b/pyasn1/codec/ber/decoder.py index 5ff485f..0755adc 100644 --- a/pyasn1/codec/ber/decoder.py +++ b/pyasn1/codec/ber/decoder.py @@ -4,11 +4,18 @@ # Copyright (c) 2005-2019, Ilya Etingof <etingof@gmail.com> # License: http://snmplabs.com/pyasn1/license.html # +import os + from pyasn1 import debug from pyasn1 import error from pyasn1.codec.ber import eoo +from pyasn1.codec.streaming import asSeekableStream +from pyasn1.codec.streaming import isEndOfStream +from pyasn1.codec.streaming import peekIntoStream +from pyasn1.codec.streaming import readFromStream from pyasn1.compat.integer import from_bytes from pyasn1.compat.octets import oct2int, octs2ints, ints2octs, null +from pyasn1.error import PyAsn1Error from pyasn1.type import base from pyasn1.type import char from pyasn1.type import tag @@ -16,33 +23,51 @@ from pyasn1.type import tagmap from pyasn1.type import univ from pyasn1.type import useful -__all__ = ['decode'] +__all__ = ['StreamingDecoder', 'Decoder', 'decode'] LOG = debug.registerLoggee(__name__, flags=debug.DEBUG_DECODER) noValue = base.noValue +SubstrateUnderrunError = error.SubstrateUnderrunError + -class AbstractDecoder(object): +class AbstractPayloadDecoder(object): protoComponent = None def valueDecoder(self, substrate, asn1Spec, tagSet=None, length=None, state=None, decodeFun=None, substrateFun=None, **options): - raise error.PyAsn1Error('Decoder not implemented for %s' % (tagSet,)) + """Decode value with fixed byte length. + + The decoder is allowed to consume as many bytes as necessary. + """ + raise error.PyAsn1Error('SingleItemDecoder not implemented for %s' % (tagSet,)) # TODO: Seems more like an NotImplementedError? def indefLenValueDecoder(self, substrate, asn1Spec, tagSet=None, length=None, state=None, decodeFun=None, substrateFun=None, **options): - raise error.PyAsn1Error('Indefinite length mode decoder not implemented for %s' % (tagSet,)) + """Decode value with undefined length. + + The decoder is allowed to consume as many bytes as necessary. + """ + raise error.PyAsn1Error('Indefinite length mode decoder not implemented for %s' % (tagSet,)) # TODO: Seems more like an NotImplementedError? + + @staticmethod + def _passAsn1Object(asn1Object, options): + if 'asn1Object' not in options: + options['asn1Object'] = asn1Object + + return options -class AbstractSimpleDecoder(AbstractDecoder): +class AbstractSimplePayloadDecoder(AbstractPayloadDecoder): @staticmethod - def substrateCollector(asn1Object, substrate, length): - return substrate[:length], substrate[length:] + def substrateCollector(asn1Object, substrate, length, options): + for chunk in readFromStream(substrate, length, options): + yield chunk def _createComponent(self, asn1Spec, tagSet, value, **options): if options.get('native'): @@ -55,7 +80,7 @@ class AbstractSimpleDecoder(AbstractDecoder): return asn1Spec.clone(value) -class ExplicitTagDecoder(AbstractSimpleDecoder): +class RawPayloadDecoder(AbstractSimplePayloadDecoder): protoComponent = univ.Any('') def valueDecoder(self, substrate, asn1Spec, @@ -63,45 +88,43 @@ class ExplicitTagDecoder(AbstractSimpleDecoder): decodeFun=None, substrateFun=None, **options): if substrateFun: - return substrateFun( - self._createComponent(asn1Spec, tagSet, '', **options), - substrate, length - ) - - head, tail = substrate[:length], substrate[length:] + asn1Object = self._createComponent(asn1Spec, tagSet, '', **options) - value, _ = decodeFun(head, asn1Spec, tagSet, length, **options) + for chunk in substrateFun(asn1Object, substrate, length, options): + yield chunk - if LOG: - LOG('explicit tag container carries %d octets of trailing payload ' - '(will be lost!): %s' % (len(_), debug.hexdump(_))) + return - return value, tail + for value in decodeFun(substrate, asn1Spec, tagSet, length, **options): + yield value def indefLenValueDecoder(self, substrate, asn1Spec, tagSet=None, length=None, state=None, decodeFun=None, substrateFun=None, **options): if substrateFun: - return substrateFun( - self._createComponent(asn1Spec, tagSet, '', **options), - substrate, length - ) + asn1Object = self._createComponent(asn1Spec, tagSet, '', **options) - value, substrate = decodeFun(substrate, asn1Spec, tagSet, length, **options) + for chunk in substrateFun(asn1Object, substrate, length, options): + yield chunk - eooMarker, substrate = decodeFun(substrate, allowEoo=True, **options) + return - if eooMarker is eoo.endOfOctets: - return value, substrate - else: - raise error.PyAsn1Error('Missing end-of-octets terminator') + while True: + for value in decodeFun( + substrate, asn1Spec, tagSet, length, + allowEoo=True, **options): + + if value is eoo.endOfOctets: + return + + yield value -explicitTagDecoder = ExplicitTagDecoder() +rawPayloadDecoder = RawPayloadDecoder() -class IntegerDecoder(AbstractSimpleDecoder): +class IntegerPayloadDecoder(AbstractSimplePayloadDecoder): protoComponent = univ.Integer(0) def valueDecoder(self, substrate, asn1Spec, @@ -112,25 +135,28 @@ class IntegerDecoder(AbstractSimpleDecoder): if tagSet[0].tagFormat != tag.tagFormatSimple: raise error.PyAsn1Error('Simple tag format expected') - head, tail = substrate[:length], substrate[length:] + for chunk in readFromStream(substrate, length, options): + if isinstance(chunk, SubstrateUnderrunError): + yield chunk - if not head: - return self._createComponent(asn1Spec, tagSet, 0, **options), tail + if chunk: + value = from_bytes(chunk, signed=True) - value = from_bytes(head, signed=True) + else: + value = 0 - return self._createComponent(asn1Spec, tagSet, value, **options), tail + yield self._createComponent(asn1Spec, tagSet, value, **options) -class BooleanDecoder(IntegerDecoder): +class BooleanPayloadDecoder(IntegerPayloadDecoder): protoComponent = univ.Boolean(0) def _createComponent(self, asn1Spec, tagSet, value, **options): - return IntegerDecoder._createComponent( + return IntegerPayloadDecoder._createComponent( self, asn1Spec, tagSet, value and 1 or 0, **options) -class BitStringDecoder(AbstractSimpleDecoder): +class BitStringPayloadDecoder(AbstractSimplePayloadDecoder): protoComponent = univ.BitString(()) supportConstructedForm = True @@ -138,27 +164,47 @@ class BitStringDecoder(AbstractSimpleDecoder): tagSet=None, length=None, state=None, decodeFun=None, substrateFun=None, **options): - head, tail = substrate[:length], substrate[length:] if substrateFun: - return substrateFun(self._createComponent( - asn1Spec, tagSet, noValue, **options), substrate, length) + asn1Object = self._createComponent(asn1Spec, tagSet, noValue, **options) + + for chunk in substrateFun(asn1Object, substrate, length, options): + yield chunk + + return + + if not length: + raise error.PyAsn1Error('Empty BIT STRING substrate') + + for chunk in isEndOfStream(substrate): + if isinstance(chunk, SubstrateUnderrunError): + yield chunk - if not head: + if chunk: raise error.PyAsn1Error('Empty BIT STRING substrate') if tagSet[0].tagFormat == tag.tagFormatSimple: # XXX what tag to check? - trailingBits = oct2int(head[0]) + for trailingBits in readFromStream(substrate, 1, options): + if isinstance(trailingBits, SubstrateUnderrunError): + yield trailingBits + + trailingBits = ord(trailingBits) if trailingBits > 7: raise error.PyAsn1Error( 'Trailing bits overflow %s' % trailingBits ) + for chunk in readFromStream(substrate, length - 1, options): + if isinstance(chunk, SubstrateUnderrunError): + yield chunk + value = self.protoComponent.fromOctetString( - head[1:], internalFormat=True, padding=trailingBits) + chunk, internalFormat=True, padding=trailingBits) + + yield self._createComponent(asn1Spec, tagSet, value, **options) - return self._createComponent(asn1Spec, tagSet, value, **options), tail + return if not self.supportConstructedForm: raise error.PyAsn1Error('Constructed encoding form prohibited ' @@ -172,9 +218,14 @@ class BitStringDecoder(AbstractSimpleDecoder): bitString = self.protoComponent.fromOctetString(null, internalFormat=True) - while head: - component, head = decodeFun(head, self.protoComponent, - substrateFun=substrateFun, **options) + current_position = substrate.tell() + + while substrate.tell() - current_position < length: + for component in decodeFun( + substrate, self.protoComponent, substrateFun=substrateFun, + **options): + if isinstance(component, SubstrateUnderrunError): + yield component trailingBits = oct2int(component[0]) if trailingBits > 7: @@ -187,7 +238,7 @@ class BitStringDecoder(AbstractSimpleDecoder): prepend=bitString, padding=trailingBits ) - return self._createComponent(asn1Spec, tagSet, bitString, **options), tail + yield self._createComponent(asn1Spec, tagSet, bitString, **options) def indefLenValueDecoder(self, substrate, asn1Spec, tagSet=None, length=None, state=None, @@ -195,17 +246,30 @@ class BitStringDecoder(AbstractSimpleDecoder): **options): if substrateFun: - return substrateFun(self._createComponent(asn1Spec, tagSet, noValue, **options), substrate, length) + asn1Object = self._createComponent(asn1Spec, tagSet, noValue, **options) + + for chunk in substrateFun(asn1Object, substrate, length, options): + yield chunk + + return # All inner fragments are of the same type, treat them as octet string substrateFun = self.substrateCollector bitString = self.protoComponent.fromOctetString(null, internalFormat=True) - while substrate: - component, substrate = decodeFun(substrate, self.protoComponent, - substrateFun=substrateFun, - allowEoo=True, **options) + while True: # loop over fragments + + for component in decodeFun( + substrate, self.protoComponent, substrateFun=substrateFun, + allowEoo=True, **options): + + if component is eoo.endOfOctets: + break + + if isinstance(component, SubstrateUnderrunError): + yield component + if component is eoo.endOfOctets: break @@ -220,13 +284,10 @@ class BitStringDecoder(AbstractSimpleDecoder): prepend=bitString, padding=trailingBits ) - else: - raise error.SubstrateUnderrunError('No EOO seen before substrate ends') - - return self._createComponent(asn1Spec, tagSet, bitString, **options), substrate + yield self._createComponent(asn1Spec, tagSet, bitString, **options) -class OctetStringDecoder(AbstractSimpleDecoder): +class OctetStringPayloadDecoder(AbstractSimplePayloadDecoder): protoComponent = univ.OctetString('') supportConstructedForm = True @@ -234,14 +295,22 @@ class OctetStringDecoder(AbstractSimpleDecoder): tagSet=None, length=None, state=None, decodeFun=None, substrateFun=None, **options): - head, tail = substrate[:length], substrate[length:] - if substrateFun: - return substrateFun(self._createComponent(asn1Spec, tagSet, noValue, **options), - substrate, length) + asn1Object = self._createComponent(asn1Spec, tagSet, noValue, **options) + + for chunk in substrateFun(asn1Object, substrate, length, options): + yield chunk + + return if tagSet[0].tagFormat == tag.tagFormatSimple: # XXX what tag to check? - return self._createComponent(asn1Spec, tagSet, head, **options), tail + for chunk in readFromStream(substrate, length, options): + if isinstance(chunk, SubstrateUnderrunError): + yield chunk + + yield self._createComponent(asn1Spec, tagSet, chunk, **options) + + return if not self.supportConstructedForm: raise error.PyAsn1Error('Constructed encoding form prohibited at %s' % self.__class__.__name__) @@ -254,13 +323,18 @@ class OctetStringDecoder(AbstractSimpleDecoder): header = null - while head: - component, head = decodeFun(head, self.protoComponent, - substrateFun=substrateFun, - **options) + original_position = substrate.tell() + # head = popSubstream(substrate, length) + while substrate.tell() - original_position < length: + for component in decodeFun( + substrate, self.protoComponent, substrateFun=substrateFun, + **options): + if isinstance(component, SubstrateUnderrunError): + yield component + header += component - return self._createComponent(asn1Spec, tagSet, header, **options), tail + yield self._createComponent(asn1Spec, tagSet, header, **options) def indefLenValueDecoder(self, substrate, asn1Spec, tagSet=None, length=None, state=None, @@ -268,32 +342,38 @@ class OctetStringDecoder(AbstractSimpleDecoder): **options): if substrateFun and substrateFun is not self.substrateCollector: asn1Object = self._createComponent(asn1Spec, tagSet, noValue, **options) - return substrateFun(asn1Object, substrate, length) + + for chunk in substrateFun(asn1Object, substrate, length, options): + yield chunk + + return # All inner fragments are of the same type, treat them as octet string substrateFun = self.substrateCollector header = null - while substrate: - component, substrate = decodeFun(substrate, - self.protoComponent, - substrateFun=substrateFun, - allowEoo=True, **options) + while True: # loop over fragments + + for component in decodeFun( + substrate, self.protoComponent, substrateFun=substrateFun, + allowEoo=True, **options): + + if isinstance(component, SubstrateUnderrunError): + yield component + + if component is eoo.endOfOctets: + break + if component is eoo.endOfOctets: break header += component - else: - raise error.SubstrateUnderrunError( - 'No EOO seen before substrate ends' - ) + yield self._createComponent(asn1Spec, tagSet, header, **options) - return self._createComponent(asn1Spec, tagSet, header, **options), substrate - -class NullDecoder(AbstractSimpleDecoder): +class NullPayloadDecoder(AbstractSimplePayloadDecoder): protoComponent = univ.Null('') def valueDecoder(self, substrate, asn1Spec, @@ -304,17 +384,19 @@ class NullDecoder(AbstractSimpleDecoder): if tagSet[0].tagFormat != tag.tagFormatSimple: raise error.PyAsn1Error('Simple tag format expected') - head, tail = substrate[:length], substrate[length:] + for chunk in readFromStream(substrate, length, options): + if isinstance(chunk, SubstrateUnderrunError): + yield chunk component = self._createComponent(asn1Spec, tagSet, '', **options) - if head: + if chunk: raise error.PyAsn1Error('Unexpected %d-octet substrate for Null' % length) - return component, tail + yield component -class ObjectIdentifierDecoder(AbstractSimpleDecoder): +class ObjectIdentifierPayloadDecoder(AbstractSimplePayloadDecoder): protoComponent = univ.ObjectIdentifier(()) def valueDecoder(self, substrate, asn1Spec, @@ -324,17 +406,20 @@ class ObjectIdentifierDecoder(AbstractSimpleDecoder): if tagSet[0].tagFormat != tag.tagFormatSimple: raise error.PyAsn1Error('Simple tag format expected') - head, tail = substrate[:length], substrate[length:] - if not head: + for chunk in readFromStream(substrate, length, options): + if isinstance(chunk, SubstrateUnderrunError): + yield chunk + + if not chunk: raise error.PyAsn1Error('Empty substrate') - head = octs2ints(head) + chunk = octs2ints(chunk) oid = () index = 0 - substrateLen = len(head) + substrateLen = len(chunk) while index < substrateLen: - subId = head[index] + subId = chunk[index] index += 1 if subId < 128: oid += (subId,) @@ -348,7 +433,7 @@ class ObjectIdentifierDecoder(AbstractSimpleDecoder): raise error.SubstrateUnderrunError( 'Short substrate for sub-OID past %s' % (oid,) ) - nextSubId = head[index] + nextSubId = chunk[index] index += 1 oid += ((subId << 7) + nextSubId,) elif subId == 128: @@ -366,12 +451,12 @@ class ObjectIdentifierDecoder(AbstractSimpleDecoder): elif oid[0] >= 80: oid = (2, oid[0] - 80) + oid[1:] else: - raise error.PyAsn1Error('Malformed first OID octet: %s' % head[0]) + raise error.PyAsn1Error('Malformed first OID octet: %s' % chunk[0]) - return self._createComponent(asn1Spec, tagSet, oid, **options), tail + yield self._createComponent(asn1Spec, tagSet, oid, **options) -class RealDecoder(AbstractSimpleDecoder): +class RealPayloadDecoder(AbstractSimplePayloadDecoder): protoComponent = univ.Real() def valueDecoder(self, substrate, asn1Spec, @@ -381,15 +466,18 @@ class RealDecoder(AbstractSimpleDecoder): if tagSet[0].tagFormat != tag.tagFormatSimple: raise error.PyAsn1Error('Simple tag format expected') - head, tail = substrate[:length], substrate[length:] + for chunk in readFromStream(substrate, length, options): + if isinstance(chunk, SubstrateUnderrunError): + yield chunk - if not head: - return self._createComponent(asn1Spec, tagSet, 0.0, **options), tail + if not chunk: + yield self._createComponent(asn1Spec, tagSet, 0.0, **options) + return - fo = oct2int(head[0]) - head = head[1:] + fo = oct2int(chunk[0]) + chunk = chunk[1:] if fo & 0x80: # binary encoding - if not head: + if not chunk: raise error.PyAsn1Error("Incomplete floating-point value") if LOG: @@ -398,12 +486,12 @@ class RealDecoder(AbstractSimpleDecoder): n = (fo & 0x03) + 1 if n == 4: - n = oct2int(head[0]) - head = head[1:] + n = oct2int(chunk[0]) + chunk = chunk[1:] - eo, head = head[:n], head[n:] + eo, chunk = chunk[:n], chunk[n:] - if not eo or not head: + if not eo or not chunk: raise error.PyAsn1Error('Real exponent screwed') e = oct2int(eo[0]) & 0x80 and -1 or 0 @@ -425,10 +513,10 @@ class RealDecoder(AbstractSimpleDecoder): e *= 4 p = 0 - while head: # value + while chunk: # value p <<= 8 - p |= oct2int(head[0]) - head = head[1:] + p |= oct2int(chunk[0]) + chunk = chunk[1:] if fo & 0x40: # sign bit p = -p @@ -444,7 +532,7 @@ class RealDecoder(AbstractSimpleDecoder): value = fo & 0x01 and '-inf' or 'inf' elif fo & 0xc0 == 0: # character encoding - if not head: + if not chunk: raise error.PyAsn1Error("Incomplete floating-point value") if LOG: @@ -452,13 +540,13 @@ class RealDecoder(AbstractSimpleDecoder): try: if fo & 0x3 == 0x1: # NR1 - value = (int(head), 10, 0) + value = (int(chunk), 10, 0) elif fo & 0x3 == 0x2: # NR2 - value = float(head) + value = float(chunk) elif fo & 0x3 == 0x3: # NR3 - value = float(head) + value = float(chunk) else: raise error.SubstrateUnderrunError( @@ -475,14 +563,14 @@ class RealDecoder(AbstractSimpleDecoder): 'Unknown encoding (tag %s)' % fo ) - return self._createComponent(asn1Spec, tagSet, value, **options), tail + yield self._createComponent(asn1Spec, tagSet, value, **options) -class AbstractConstructedDecoder(AbstractDecoder): +class AbstractConstructedPayloadDecoder(AbstractPayloadDecoder): protoComponent = None -class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): +class ConstructedPayloadDecoderBase(AbstractConstructedPayloadDecoder): protoRecordComponent = None protoSequenceComponent = None @@ -492,33 +580,43 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): def _getComponentPositionByType(self, asn1Object, tagSet, idx): raise NotImplementedError() - def _decodeComponents(self, substrate, tagSet=None, decodeFun=None, **options): + def _decodeComponentsSchemaless( + self, substrate, tagSet=None, decodeFun=None, + length=None, **options): + + asn1Object = None + components = [] componentTypes = set() - while substrate: - component, substrate = decodeFun(substrate, **options) - if component is eoo.endOfOctets: + original_position = substrate.tell() + + while length == -1 or substrate.tell() < original_position + length: + for component in decodeFun(substrate, **options): + if isinstance(component, SubstrateUnderrunError): + yield component + + if length == -1 and component is eoo.endOfOctets: break components.append(component) componentTypes.add(component.tagSet) - # Now we have to guess is it SEQUENCE/SET or SEQUENCE OF/SET OF - # The heuristics is: - # * 1+ components of different types -> likely SEQUENCE/SET - # * otherwise -> likely SEQUENCE OF/SET OF - if len(componentTypes) > 1: - protoComponent = self.protoRecordComponent + # Now we have to guess is it SEQUENCE/SET or SEQUENCE OF/SET OF + # The heuristics is: + # * 1+ components of different types -> likely SEQUENCE/SET + # * otherwise -> likely SEQUENCE OF/SET OF + if len(componentTypes) > 1: + protoComponent = self.protoRecordComponent - else: - protoComponent = self.protoSequenceComponent + else: + protoComponent = self.protoSequenceComponent - asn1Object = protoComponent.clone( - # construct tagSet from base tag from prototype ASN.1 object - # and additional tags recovered from the substrate - tagSet=tag.TagSet(protoComponent.tagSet.baseTag, *tagSet.superTags) - ) + asn1Object = protoComponent.clone( + # construct tagSet from base tag from prototype ASN.1 object + # and additional tags recovered from the substrate + tagSet=tag.TagSet(protoComponent.tagSet.baseTag, *tagSet.superTags) + ) if LOG: LOG('guessed %r container type (pass `asn1Spec` to guide the ' @@ -531,7 +629,7 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): matchTags=False, matchConstraints=False ) - return asn1Object, substrate + yield asn1Object def valueDecoder(self, substrate, asn1Spec, tagSet=None, length=None, state=None, @@ -540,9 +638,9 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): if tagSet[0].tagFormat != tag.tagFormatConstructed: raise error.PyAsn1Error('Constructed tag format expected') - head, tail = substrate[:length], substrate[length:] + original_position = substrate.tell() - if substrateFun is not None: + if substrateFun: if asn1Spec is not None: asn1Object = asn1Spec.clone() @@ -552,23 +650,36 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): else: asn1Object = self.protoRecordComponent, self.protoSequenceComponent - return substrateFun(asn1Object, substrate, length) + for chunk in substrateFun(asn1Object, substrate, length, options): + yield chunk + + return if asn1Spec is None: - asn1Object, trailing = self._decodeComponents( - head, tagSet=tagSet, decodeFun=decodeFun, **options - ) + for asn1Object in self._decodeComponentsSchemaless( + substrate, tagSet=tagSet, decodeFun=decodeFun, + length=length, **options): + if isinstance(asn1Object, SubstrateUnderrunError): + yield asn1Object - if trailing: + if substrate.tell() < original_position + length: if LOG: + for trailing in readFromStream(substrate, context=options): + if isinstance(trailing, SubstrateUnderrunError): + yield trailing + LOG('Unused trailing %d octets encountered: %s' % ( len(trailing), debug.hexdump(trailing))) - return asn1Object, tail + yield asn1Object + + return asn1Object = asn1Spec.clone() asn1Object.clear() + options = self._passAsn1Object(asn1Object, options) + if asn1Spec.typeId in (univ.Sequence.typeId, univ.Set.typeId): namedTypes = asn1Spec.componentType @@ -583,7 +694,7 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): seenIndices = set() idx = 0 - while head: + while substrate.tell() - original_position < length: if not namedTypes: componentType = None @@ -606,7 +717,9 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): 'Excessive components decoded at %r' % (asn1Spec,) ) - component, head = decodeFun(head, componentType, **options) + for component in decodeFun(substrate, componentType, **options): + if isinstance(component, SubstrateUnderrunError): + yield component if not isDeterministic and namedTypes: if isSetType: @@ -693,18 +806,20 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): for pos, containerElement in enumerate( containerValue): - component, rest = decodeFun( - containerValue[pos].asOctets(), - asn1Spec=openType, **options - ) + stream = asSeekableStream(containerValue[pos].asOctets()) + + for component in decodeFun(stream, asn1Spec=openType, **options): + if isinstance(component, SubstrateUnderrunError): + yield component containerValue[pos] = component else: - component, rest = decodeFun( - asn1Object.getComponentByPosition(idx).asOctets(), - asn1Spec=openType, **options - ) + stream = asSeekableStream(asn1Object.getComponentByPosition(idx).asOctets()) + + for component in decodeFun(stream, asn1Spec=openType, **options): + if isinstance(component, SubstrateUnderrunError): + yield component asn1Object.setComponentByPosition(idx, component) @@ -714,9 +829,6 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): raise inconsistency else: - asn1Object = asn1Spec.clone() - asn1Object.clear() - componentType = asn1Spec.componentType if LOG: @@ -724,8 +836,11 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): idx = 0 - while head: - component, head = decodeFun(head, componentType, **options) + while substrate.tell() - original_position < length: + for component in decodeFun(substrate, componentType, **options): + if isinstance(component, SubstrateUnderrunError): + yield component + asn1Object.setComponentByPosition( idx, component, verifyConstraints=False, @@ -734,7 +849,7 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): idx += 1 - return asn1Object, tail + yield asn1Object def indefLenValueDecoder(self, substrate, asn1Spec, tagSet=None, length=None, state=None, @@ -753,17 +868,27 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): else: asn1Object = self.protoRecordComponent, self.protoSequenceComponent - return substrateFun(asn1Object, substrate, length) + for chunk in substrateFun(asn1Object, substrate, length, options): + yield chunk + + return if asn1Spec is None: - return self._decodeComponents( - substrate, tagSet=tagSet, decodeFun=decodeFun, - **dict(options, allowEoo=True) - ) + for asn1Object in self._decodeComponentsSchemaless( + substrate, tagSet=tagSet, decodeFun=decodeFun, + length=length, **dict(options, allowEoo=True)): + if isinstance(asn1Object, SubstrateUnderrunError): + yield asn1Object + + yield asn1Object + + return asn1Object = asn1Spec.clone() asn1Object.clear() + options = self._passAsn1Object(asn1Object, options) + if asn1Spec.typeId in (univ.Sequence.typeId, univ.Set.typeId): namedTypes = asn1Object.componentType @@ -777,8 +902,10 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): asn1Spec)) seenIndices = set() + idx = 0 - while substrate: + + while True: # loop over components if len(namedTypes) <= idx: asn1Spec = None @@ -801,13 +928,21 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): 'Excessive components decoded at %r' % (asn1Object,) ) - component, substrate = decodeFun(substrate, asn1Spec, allowEoo=True, **options) + for component in decodeFun(substrate, asn1Spec, allowEoo=True, **options): + + if isinstance(component, SubstrateUnderrunError): + yield component + + if component is eoo.endOfOctets: + break + if component is eoo.endOfOctets: break if not isDeterministic and namedTypes: if isSetType: idx = namedTypes.getPositionByType(component.effectiveTagSet) + elif namedTypes[idx].isOptional or namedTypes[idx].isDefaulted: idx = namedTypes.getPositionNearType(component.effectiveTagSet, idx) @@ -820,17 +955,14 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): seenIndices.add(idx) idx += 1 - else: - raise error.SubstrateUnderrunError( - 'No EOO seen before substrate ends' - ) - if LOG: LOG('seen component indices %s' % seenIndices) if namedTypes: if not namedTypes.requiredComponents.issubset(seenIndices): - raise error.PyAsn1Error('ASN.1 object %s has uninitialized components' % asn1Object.__class__.__name__) + raise error.PyAsn1Error( + 'ASN.1 object %s has uninitialized ' + 'components' % asn1Object.__class__.__name__) if namedTypes.hasOpenTypes: @@ -892,20 +1024,28 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): for pos, containerElement in enumerate( containerValue): - component, rest = decodeFun( - containerValue[pos].asOctets(), - asn1Spec=openType, **dict(options, allowEoo=True) - ) + stream = asSeekableStream(containerValue[pos].asOctets()) + + for component in decodeFun(stream, asn1Spec=openType, + **dict(options, allowEoo=True)): + if isinstance(component, SubstrateUnderrunError): + yield component + + if component is eoo.endOfOctets: + break containerValue[pos] = component else: - component, rest = decodeFun( - asn1Object.getComponentByPosition(idx).asOctets(), - asn1Spec=openType, **dict(options, allowEoo=True) - ) + stream = asSeekableStream(asn1Object.getComponentByPosition(idx).asOctets()) + for component in decodeFun(stream, asn1Spec=openType, + **dict(options, allowEoo=True)): + if isinstance(component, SubstrateUnderrunError): + yield component + + if component is eoo.endOfOctets: + break - if component is not eoo.endOfOctets: asn1Object.setComponentByPosition(idx, component) else: @@ -914,9 +1054,6 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): raise inconsistency else: - asn1Object = asn1Spec.clone() - asn1Object.clear() - componentType = asn1Spec.componentType if LOG: @@ -924,8 +1061,16 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): idx = 0 - while substrate: - component, substrate = decodeFun(substrate, componentType, allowEoo=True, **options) + while True: + + for component in decodeFun( + substrate, componentType, allowEoo=True, **options): + + if isinstance(component, SubstrateUnderrunError): + yield component + + if component is eoo.endOfOctets: + break if component is eoo.endOfOctets: break @@ -938,50 +1083,42 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder): idx += 1 - else: - raise error.SubstrateUnderrunError( - 'No EOO seen before substrate ends' - ) - - return asn1Object, substrate + yield asn1Object -class SequenceOrSequenceOfDecoder(UniversalConstructedTypeDecoder): +class SequenceOrSequenceOfPayloadDecoder(ConstructedPayloadDecoderBase): protoRecordComponent = univ.Sequence() protoSequenceComponent = univ.SequenceOf() -class SequenceDecoder(SequenceOrSequenceOfDecoder): +class SequencePayloadDecoder(SequenceOrSequenceOfPayloadDecoder): protoComponent = univ.Sequence() -class SequenceOfDecoder(SequenceOrSequenceOfDecoder): +class SequenceOfPayloadDecoder(SequenceOrSequenceOfPayloadDecoder): protoComponent = univ.SequenceOf() -class SetOrSetOfDecoder(UniversalConstructedTypeDecoder): +class SetOrSetOfPayloadDecoder(ConstructedPayloadDecoderBase): protoRecordComponent = univ.Set() protoSequenceComponent = univ.SetOf() -class SetDecoder(SetOrSetOfDecoder): +class SetPayloadDecoder(SetOrSetOfPayloadDecoder): protoComponent = univ.Set() - -class SetOfDecoder(SetOrSetOfDecoder): +class SetOfPayloadDecoder(SetOrSetOfPayloadDecoder): protoComponent = univ.SetOf() -class ChoiceDecoder(AbstractConstructedDecoder): +class ChoicePayloadDecoder(ConstructedPayloadDecoderBase): protoComponent = univ.Choice() def valueDecoder(self, substrate, asn1Spec, tagSet=None, length=None, state=None, decodeFun=None, substrateFun=None, **options): - head, tail = substrate[:length], substrate[length:] - if asn1Spec is None: asn1Object = self.protoComponent.clone(tagSet=tagSet) @@ -989,24 +1126,31 @@ class ChoiceDecoder(AbstractConstructedDecoder): asn1Object = asn1Spec.clone() if substrateFun: - return substrateFun(asn1Object, substrate, length) + for chunk in substrateFun(asn1Object, substrate, length, options): + yield chunk + + return + + options = self._passAsn1Object(asn1Object, options) if asn1Object.tagSet == tagSet: if LOG: LOG('decoding %s as explicitly tagged CHOICE' % (tagSet,)) - component, head = decodeFun( - head, asn1Object.componentTagMap, **options - ) + for component in decodeFun( + substrate, asn1Object.componentTagMap, **options): + if isinstance(component, SubstrateUnderrunError): + yield component else: if LOG: LOG('decoding %s as untagged CHOICE' % (tagSet,)) - component, head = decodeFun( - head, asn1Object.componentTagMap, - tagSet, length, state, **options - ) + for component in decodeFun( + substrate, asn1Object.componentTagMap, tagSet, length, + state, **options): + if isinstance(component, SubstrateUnderrunError): + yield component effectiveTagSet = component.effectiveTagSet @@ -1020,7 +1164,7 @@ class ChoiceDecoder(AbstractConstructedDecoder): innerFlag=False ) - return asn1Object, tail + yield asn1Object def indefLenValueDecoder(self, substrate, asn1Spec, tagSet=None, length=None, state=None, @@ -1028,53 +1172,67 @@ class ChoiceDecoder(AbstractConstructedDecoder): **options): if asn1Spec is None: asn1Object = self.protoComponent.clone(tagSet=tagSet) + else: asn1Object = asn1Spec.clone() if substrateFun: - return substrateFun(asn1Object, substrate, length) + for chunk in substrateFun(asn1Object, substrate, length, options): + yield chunk - if asn1Object.tagSet == tagSet: - if LOG: - LOG('decoding %s as explicitly tagged CHOICE' % (tagSet,)) + return - component, substrate = decodeFun( - substrate, asn1Object.componentType.tagMapUnique, **options - ) + options = self._passAsn1Object(asn1Object, options) - # eat up EOO marker - eooMarker, substrate = decodeFun( - substrate, allowEoo=True, **options - ) + isTagged = asn1Object.tagSet == tagSet - if eooMarker is not eoo.endOfOctets: - raise error.PyAsn1Error('No EOO seen before substrate ends') + if LOG: + LOG('decoding %s as %stagged CHOICE' % ( + tagSet, isTagged and 'explicitly ' or 'un')) - else: - if LOG: - LOG('decoding %s as untagged CHOICE' % (tagSet,)) + while True: - component, substrate = decodeFun( - substrate, asn1Object.componentType.tagMapUnique, - tagSet, length, state, **options - ) + if isTagged: + iterator = decodeFun( + substrate, asn1Object.componentType.tagMapUnique, + **dict(options, allowEoo=True)) - effectiveTagSet = component.effectiveTagSet + else: + iterator = decodeFun( + substrate, asn1Object.componentType.tagMapUnique, + tagSet, length, state, **dict(options, allowEoo=True)) - if LOG: - LOG('decoded component %s, effective tag set %s' % (component, effectiveTagSet)) + for component in iterator: - asn1Object.setComponentByType( - effectiveTagSet, component, - verifyConstraints=False, - matchTags=False, matchConstraints=False, - innerFlag=False - ) + if isinstance(component, SubstrateUnderrunError): + yield component + + if component is eoo.endOfOctets: + break + + effectiveTagSet = component.effectiveTagSet - return asn1Object, substrate + if LOG: + LOG('decoded component %s, effective tag set ' + '%s' % (component, effectiveTagSet)) + + asn1Object.setComponentByType( + effectiveTagSet, component, + verifyConstraints=False, + matchTags=False, matchConstraints=False, + innerFlag=False + ) + if not isTagged: + break + + if not isTagged or component is eoo.endOfOctets: + break -class AnyDecoder(AbstractSimpleDecoder): + yield asn1Object + + +class AnyPayloadDecoder(AbstractSimplePayloadDecoder): protoComponent = univ.Any() def valueDecoder(self, substrate, asn1Spec, @@ -1091,22 +1249,32 @@ class AnyDecoder(AbstractSimpleDecoder): isUntagged = tagSet != asn1Spec.tagSet if isUntagged: - fullSubstrate = options['fullSubstrate'] + fullPosition = substrate.markedPosition + currentPosition = substrate.tell() - # untagged Any container, recover inner header substrate - length += len(fullSubstrate) - len(substrate) - substrate = fullSubstrate + substrate.seek(fullPosition, os.SEEK_SET) + length += currentPosition - fullPosition if LOG: - LOG('decoding as untagged ANY, substrate %s' % debug.hexdump(substrate)) + for chunk in peekIntoStream(substrate, length): + if isinstance(chunk, SubstrateUnderrunError): + yield chunk + LOG('decoding as untagged ANY, substrate ' + '%s' % debug.hexdump(chunk)) if substrateFun: - return substrateFun(self._createComponent(asn1Spec, tagSet, noValue, **options), - substrate, length) + for chunk in substrateFun( + self._createComponent(asn1Spec, tagSet, noValue, **options), + substrate, length, options): + yield chunk - head, tail = substrate[:length], substrate[length:] + return - return self._createComponent(asn1Spec, tagSet, head, **options), tail + for chunk in readFromStream(substrate, length, options): + if isinstance(chunk, SubstrateUnderrunError): + yield chunk + + yield self._createComponent(asn1Spec, tagSet, chunk, **options) def indefLenValueDecoder(self, substrate, asn1Spec, tagSet=None, length=None, state=None, @@ -1123,26 +1291,36 @@ class AnyDecoder(AbstractSimpleDecoder): if isTagged: # tagged Any type -- consume header substrate - header = null + chunk = null if LOG: LOG('decoding as tagged ANY') else: - fullSubstrate = options['fullSubstrate'] + # TODO: Seems not to be tested + fullPosition = substrate.markedPosition + currentPosition = substrate.tell() - # untagged Any, recover header substrate - header = fullSubstrate[:-len(substrate)] + substrate.seek(fullPosition, os.SEEK_SET) + for chunk in readFromStream(substrate, currentPosition - fullPosition, options): + if isinstance(chunk, SubstrateUnderrunError): + yield chunk if LOG: - LOG('decoding as untagged ANY, header substrate %s' % debug.hexdump(header)) + LOG('decoding as untagged ANY, header substrate %s' % debug.hexdump(chunk)) # Any components do not inherit initial tag asn1Spec = self.protoComponent if substrateFun and substrateFun is not self.substrateCollector: - asn1Object = self._createComponent(asn1Spec, tagSet, noValue, **options) - return substrateFun(asn1Object, header + substrate, length + len(header)) + asn1Object = self._createComponent( + asn1Spec, tagSet, noValue, **options) + + for chunk in substrateFun( + asn1Object, chunk + substrate, length + len(chunk), options): + yield chunk + + return if LOG: LOG('assembling constructed serialization') @@ -1150,131 +1328,134 @@ class AnyDecoder(AbstractSimpleDecoder): # All inner fragments are of the same type, treat them as octet string substrateFun = self.substrateCollector - while substrate: - component, substrate = decodeFun(substrate, asn1Spec, - substrateFun=substrateFun, - allowEoo=True, **options) + while True: # loop over fragments + + for component in decodeFun( + substrate, asn1Spec, substrateFun=substrateFun, + allowEoo=True, **options): + + if isinstance(component, SubstrateUnderrunError): + yield component + + if component is eoo.endOfOctets: + break + if component is eoo.endOfOctets: break - header += component - - else: - raise error.SubstrateUnderrunError( - 'No EOO seen before substrate ends' - ) + chunk += component if substrateFun: - return header, substrate + yield chunk # TODO: Weird else: - return self._createComponent(asn1Spec, tagSet, header, **options), substrate + yield self._createComponent(asn1Spec, tagSet, chunk, **options) # character string types -class UTF8StringDecoder(OctetStringDecoder): +class UTF8StringPayloadDecoder(OctetStringPayloadDecoder): protoComponent = char.UTF8String() -class NumericStringDecoder(OctetStringDecoder): +class NumericStringPayloadDecoder(OctetStringPayloadDecoder): protoComponent = char.NumericString() -class PrintableStringDecoder(OctetStringDecoder): +class PrintableStringPayloadDecoder(OctetStringPayloadDecoder): protoComponent = char.PrintableString() -class TeletexStringDecoder(OctetStringDecoder): +class TeletexStringPayloadDecoder(OctetStringPayloadDecoder): protoComponent = char.TeletexString() -class VideotexStringDecoder(OctetStringDecoder): +class VideotexStringPayloadDecoder(OctetStringPayloadDecoder): protoComponent = char.VideotexString() -class IA5StringDecoder(OctetStringDecoder): +class IA5StringPayloadDecoder(OctetStringPayloadDecoder): protoComponent = char.IA5String() -class GraphicStringDecoder(OctetStringDecoder): +class GraphicStringPayloadDecoder(OctetStringPayloadDecoder): protoComponent = char.GraphicString() -class VisibleStringDecoder(OctetStringDecoder): +class VisibleStringPayloadDecoder(OctetStringPayloadDecoder): protoComponent = char.VisibleString() -class GeneralStringDecoder(OctetStringDecoder): +class GeneralStringPayloadDecoder(OctetStringPayloadDecoder): protoComponent = char.GeneralString() -class UniversalStringDecoder(OctetStringDecoder): +class UniversalStringPayloadDecoder(OctetStringPayloadDecoder): protoComponent = char.UniversalString() -class BMPStringDecoder(OctetStringDecoder): +class BMPStringPayloadDecoder(OctetStringPayloadDecoder): protoComponent = char.BMPString() # "useful" types -class ObjectDescriptorDecoder(OctetStringDecoder): +class ObjectDescriptorPayloadDecoder(OctetStringPayloadDecoder): protoComponent = useful.ObjectDescriptor() -class GeneralizedTimeDecoder(OctetStringDecoder): +class GeneralizedTimePayloadDecoder(OctetStringPayloadDecoder): protoComponent = useful.GeneralizedTime() -class UTCTimeDecoder(OctetStringDecoder): +class UTCTimePayloadDecoder(OctetStringPayloadDecoder): protoComponent = useful.UTCTime() -tagMap = { - univ.Integer.tagSet: IntegerDecoder(), - univ.Boolean.tagSet: BooleanDecoder(), - univ.BitString.tagSet: BitStringDecoder(), - univ.OctetString.tagSet: OctetStringDecoder(), - univ.Null.tagSet: NullDecoder(), - univ.ObjectIdentifier.tagSet: ObjectIdentifierDecoder(), - univ.Enumerated.tagSet: IntegerDecoder(), - univ.Real.tagSet: RealDecoder(), - univ.Sequence.tagSet: SequenceOrSequenceOfDecoder(), # conflicts with SequenceOf - univ.Set.tagSet: SetOrSetOfDecoder(), # conflicts with SetOf - univ.Choice.tagSet: ChoiceDecoder(), # conflicts with Any +TAG_MAP = { + univ.Integer.tagSet: IntegerPayloadDecoder(), + univ.Boolean.tagSet: BooleanPayloadDecoder(), + univ.BitString.tagSet: BitStringPayloadDecoder(), + univ.OctetString.tagSet: OctetStringPayloadDecoder(), + univ.Null.tagSet: NullPayloadDecoder(), + univ.ObjectIdentifier.tagSet: ObjectIdentifierPayloadDecoder(), + univ.Enumerated.tagSet: IntegerPayloadDecoder(), + univ.Real.tagSet: RealPayloadDecoder(), + univ.Sequence.tagSet: SequenceOrSequenceOfPayloadDecoder(), # conflicts with SequenceOf + univ.Set.tagSet: SetOrSetOfPayloadDecoder(), # conflicts with SetOf + univ.Choice.tagSet: ChoicePayloadDecoder(), # conflicts with Any # character string types - char.UTF8String.tagSet: UTF8StringDecoder(), - char.NumericString.tagSet: NumericStringDecoder(), - char.PrintableString.tagSet: PrintableStringDecoder(), - char.TeletexString.tagSet: TeletexStringDecoder(), - char.VideotexString.tagSet: VideotexStringDecoder(), - char.IA5String.tagSet: IA5StringDecoder(), - char.GraphicString.tagSet: GraphicStringDecoder(), - char.VisibleString.tagSet: VisibleStringDecoder(), - char.GeneralString.tagSet: GeneralStringDecoder(), - char.UniversalString.tagSet: UniversalStringDecoder(), - char.BMPString.tagSet: BMPStringDecoder(), + char.UTF8String.tagSet: UTF8StringPayloadDecoder(), + char.NumericString.tagSet: NumericStringPayloadDecoder(), + char.PrintableString.tagSet: PrintableStringPayloadDecoder(), + char.TeletexString.tagSet: TeletexStringPayloadDecoder(), + char.VideotexString.tagSet: VideotexStringPayloadDecoder(), + char.IA5String.tagSet: IA5StringPayloadDecoder(), + char.GraphicString.tagSet: GraphicStringPayloadDecoder(), + char.VisibleString.tagSet: VisibleStringPayloadDecoder(), + char.GeneralString.tagSet: GeneralStringPayloadDecoder(), + char.UniversalString.tagSet: UniversalStringPayloadDecoder(), + char.BMPString.tagSet: BMPStringPayloadDecoder(), # useful types - useful.ObjectDescriptor.tagSet: ObjectDescriptorDecoder(), - useful.GeneralizedTime.tagSet: GeneralizedTimeDecoder(), - useful.UTCTime.tagSet: UTCTimeDecoder() + useful.ObjectDescriptor.tagSet: ObjectDescriptorPayloadDecoder(), + useful.GeneralizedTime.tagSet: GeneralizedTimePayloadDecoder(), + useful.UTCTime.tagSet: UTCTimePayloadDecoder() } # Type-to-codec map for ambiguous ASN.1 types -typeMap = { - univ.Set.typeId: SetDecoder(), - univ.SetOf.typeId: SetOfDecoder(), - univ.Sequence.typeId: SequenceDecoder(), - univ.SequenceOf.typeId: SequenceOfDecoder(), - univ.Choice.typeId: ChoiceDecoder(), - univ.Any.typeId: AnyDecoder() +TYPE_MAP = { + univ.Set.typeId: SetPayloadDecoder(), + univ.SetOf.typeId: SetOfPayloadDecoder(), + univ.Sequence.typeId: SequencePayloadDecoder(), + univ.SequenceOf.typeId: SequenceOfPayloadDecoder(), + univ.Choice.typeId: ChoicePayloadDecoder(), + univ.Any.typeId: AnyPayloadDecoder() } # Put in non-ambiguous types for faster codec lookup -for typeDecoder in tagMap.values(): +for typeDecoder in TAG_MAP.values(): if typeDecoder.protoComponent is not None: typeId = typeDecoder.protoComponent.__class__.typeId - if typeId is not None and typeId not in typeMap: - typeMap[typeId] = typeDecoder + if typeId is not None and typeId not in TYPE_MAP: + TYPE_MAP[typeId] = typeDecoder (stDecodeTag, @@ -1289,65 +1470,81 @@ for typeDecoder in tagMap.values(): stStop) = [x for x in range(10)] -class Decoder(object): +EOO_SENTINEL = ints2octs((0, 0)) + + +class SingleItemDecoder(object): defaultErrorState = stErrorCondition #defaultErrorState = stDumpRawValue - defaultRawDecoder = AnyDecoder() + defaultRawDecoder = AnyPayloadDecoder() + supportIndefLength = True - # noinspection PyDefaultArgument - def __init__(self, tagMap, typeMap={}): - self.__tagMap = tagMap - self.__typeMap = typeMap + TAG_MAP = TAG_MAP + TYPE_MAP = TYPE_MAP + + def __init__(self, **options): + self._tagMap = options.get('tagMap', self.TAG_MAP) + self._typeMap = options.get('typeMap', self.TYPE_MAP) + # Tag & TagSet objects caches - self.__tagCache = {} - self.__tagSetCache = {} - self.__eooSentinel = ints2octs((0, 0)) + self._tagCache = {} + self._tagSetCache = {} def __call__(self, substrate, asn1Spec=None, tagSet=None, length=None, state=stDecodeTag, decodeFun=None, substrateFun=None, **options): - if LOG: - LOG('decoder called at scope %s with state %d, working with up to %d octets of substrate: %s' % (debug.scope, state, len(substrate), debug.hexdump(substrate))) - allowEoo = options.pop('allowEoo', False) + if LOG: + LOG('decoder called at scope %s with state %d, working with up ' + 'to %s octets of substrate: ' + '%s' % (debug.scope, state, length, substrate)) + # Look for end-of-octets sentinel if allowEoo and self.supportIndefLength: - if substrate[:2] == self.__eooSentinel: + + for eoo_candidate in readFromStream(substrate, 2, options): + if isinstance(eoo_candidate, SubstrateUnderrunError): + yield eoo_candidate + + if eoo_candidate == EOO_SENTINEL: if LOG: LOG('end-of-octets sentinel found') - return eoo.endOfOctets, substrate[2:] + yield eoo.endOfOctets + return - value = noValue + else: + substrate.seek(-2, os.SEEK_CUR) - tagMap = self.__tagMap - typeMap = self.__typeMap - tagCache = self.__tagCache - tagSetCache = self.__tagSetCache + tagMap = self._tagMap + typeMap = self._typeMap + tagCache = self._tagCache + tagSetCache = self._tagSetCache - fullSubstrate = substrate + value = noValue + + substrate.markedPosition = substrate.tell() while state is not stStop: if state is stDecodeTag: - if not substrate: - raise error.SubstrateUnderrunError( - 'Short octet stream on tag decoding' - ) - # Decode tag isShortTag = True - firstOctet = substrate[0] - substrate = substrate[1:] + + for firstByte in readFromStream(substrate, 1, options): + if isinstance(firstByte, SubstrateUnderrunError): + yield firstByte + + firstOctet = ord(firstByte) try: lastTag = tagCache[firstOctet] except KeyError: - integerTag = oct2int(firstOctet) + integerTag = firstOctet tagClass = integerTag & 0xC0 tagFormat = integerTag & 0x20 tagId = integerTag & 0x1F @@ -1357,21 +1554,23 @@ class Decoder(object): lengthOctetIdx = 0 tagId = 0 - try: - while True: - integerTag = oct2int(substrate[lengthOctetIdx]) - lengthOctetIdx += 1 - tagId <<= 7 - tagId |= (integerTag & 0x7F) - if not integerTag & 0x80: - break - - substrate = substrate[lengthOctetIdx:] - - except IndexError: - raise error.SubstrateUnderrunError( - 'Short octet stream on long tag decoding' - ) + while True: + for integerByte in readFromStream(substrate, 1, options): + if isinstance(integerByte, SubstrateUnderrunError): + yield integerByte + + if not integerByte: + raise error.SubstrateUnderrunError( + 'Short octet stream on long tag decoding' + ) + + integerTag = ord(integerByte) + lengthOctetIdx += 1 + tagId <<= 7 + tagId |= (integerTag & 0x7F) + + if not integerTag & 0x80: + break lastTag = tag.Tag( tagClass=tagClass, tagFormat=tagFormat, tagId=tagId @@ -1403,21 +1602,22 @@ class Decoder(object): if state is stDecodeLength: # Decode length - if not substrate: - raise error.SubstrateUnderrunError( - 'Short octet stream on length decoding' - ) + for firstOctet in readFromStream(substrate, 1, options): + if isinstance(firstOctet, SubstrateUnderrunError): + yield firstOctet - firstOctet = oct2int(substrate[0]) + firstOctet = ord(firstOctet) if firstOctet < 128: - size = 1 length = firstOctet elif firstOctet > 128: size = firstOctet & 0x7F # encoded in size bytes - encodedLength = octs2ints(substrate[1:size + 1]) + for encodedLength in readFromStream(substrate, size, options): + if isinstance(encodedLength, SubstrateUnderrunError): + yield encodedLength + encodedLength = list(encodedLength) # missing check on maximum size, which shouldn't be a # problem, we can handle more than is possible if len(encodedLength) != size: @@ -1428,27 +1628,19 @@ class Decoder(object): length = 0 for lengthOctet in encodedLength: length <<= 8 - length |= lengthOctet + length |= oct2int(lengthOctet) size += 1 - else: - size = 1 + else: # 128 means indefinite length = -1 - substrate = substrate[size:] - - if length == -1: - if not self.supportIndefLength: - raise error.PyAsn1Error('Indefinite length encoding not supported by this codec') - - else: - if len(substrate) < length: - raise error.SubstrateUnderrunError('%d-octet short' % (length - len(substrate))) + if length == -1 and not self.supportIndefLength: + raise error.PyAsn1Error('Indefinite length encoding not supported by this codec') state = stGetValueDecoder if LOG: - LOG('value length decoded into %d, payload substrate is: %s' % (length, debug.hexdump(length == -1 and substrate or substrate[:length]))) + LOG('value length decoded into %d' % length) if state is stGetValueDecoder: if asn1Spec is None: @@ -1567,26 +1759,33 @@ class Decoder(object): if not options.get('recursiveFlag', True) and not substrateFun: # deprecate this substrateFun = lambda a, b, c: (a, b[:c]) - options.update(fullSubstrate=fullSubstrate) + original_position = substrate.tell() if length == -1: # indef length - value, substrate = concreteDecoder.indefLenValueDecoder( - substrate, asn1Spec, - tagSet, length, stGetValueDecoder, - self, substrateFun, - **options - ) + for value in concreteDecoder.indefLenValueDecoder( + substrate, asn1Spec, + tagSet, length, stGetValueDecoder, + self, substrateFun, **options): + if isinstance(value, SubstrateUnderrunError): + yield value else: - value, substrate = concreteDecoder.valueDecoder( - substrate, asn1Spec, - tagSet, length, stGetValueDecoder, - self, substrateFun, - **options - ) + for value in concreteDecoder.valueDecoder( + substrate, asn1Spec, + tagSet, length, stGetValueDecoder, + self, substrateFun, **options): + if isinstance(value, SubstrateUnderrunError): + yield value + + bytesRead = substrate.tell() - original_position + if bytesRead != length: + raise PyAsn1Error( + "Read %s bytes instead of expected %s." % (bytesRead, length)) if LOG: - LOG('codec %s yields type %s, value:\n%s\n...remaining substrate is: %s' % (concreteDecoder.__class__.__name__, value.__class__.__name__, isinstance(value, base.Asn1Item) and value.prettyPrint() or value, substrate and debug.hexdump(substrate) or '<none>')) + LOG('codec %s yields type %s, value:\n%s\n...' % ( + concreteDecoder.__class__.__name__, value.__class__.__name__, + isinstance(value, base.Asn1Item) and value.prettyPrint() or value)) state = stStop break @@ -1596,7 +1795,7 @@ class Decoder(object): tagSet[0].tagFormat == tag.tagFormatConstructed and tagSet[0].tagClass != tag.tagClassUniversal): # Assume explicit tagging - concreteDecoder = explicitTagDecoder + concreteDecoder = rawPayloadDecoder state = stDecodeValue else: @@ -1623,7 +1822,190 @@ class Decoder(object): debug.scope.pop() LOG('decoder left scope %s, call completed' % debug.scope) - return value, substrate + yield value + + +class StreamingDecoder(object): + """Create an iterator that turns BER/CER/DER byte stream into ASN.1 objects. + + On each iteration, consume whatever BER/CER/DER serialization is + available in the `substrate` stream-like object and turns it into + one or more, possibly nested, ASN.1 objects. + + Parameters + ---------- + substrate: :py:class:`file`, :py:class:`io.BytesIO` + BER/CER/DER serialization in form of a byte stream + + Keyword Args + ------------ + asn1Spec: :py:class:`~pyasn1.type.base.PyAsn1Item` + A pyasn1 type object to act as a template guiding the decoder. + Depending on the ASN.1 structure being decoded, `asn1Spec` may + or may not be required. One of the reasons why `asn1Spec` may + me required is that ASN.1 structure is encoded in the *IMPLICIT* + tagging mode. + + Yields + ------ + : :py:class:`~pyasn1.type.base.PyAsn1Item`, :py:class:`~pyasn1.error.SubstrateUnderrunError` + Decoded ASN.1 object (possibly, nested) or + :py:class:`~pyasn1.error.SubstrateUnderrunError` object indicating + insufficient BER/CER/DER serialization on input to fully recover ASN.1 + objects from it. + + In the latter case the caller is advised to ensure some more data in + the input stream, then call the iterator again. The decoder will resume + the decoding process using the newly arrived data. + + The `context` property of :py:class:`~pyasn1.error.SubstrateUnderrunError` + object might hold a reference to the partially populated ASN.1 object + being reconstructed. + + Raises + ------ + ~pyasn1.error.PyAsn1Error, ~pyasn1.error.EndOfStreamError + `PyAsn1Error` on deserialization error, `EndOfStreamError` on + premature stream closure. + + Examples + -------- + Decode BER serialisation without ASN.1 schema + + .. code-block:: pycon + + >>> stream = io.BytesIO( + ... b'0\t\x02\x01\x01\x02\x01\x02\x02\x01\x03') + >>> + >>> for asn1Object in StreamingDecoder(stream): + ... print(asn1Object) + >>> + SequenceOf: + 1 2 3 + + Decode BER serialisation with ASN.1 schema + + .. code-block:: pycon + + >>> stream = io.BytesIO( + ... b'0\t\x02\x01\x01\x02\x01\x02\x02\x01\x03') + >>> + >>> schema = SequenceOf(componentType=Integer()) + >>> + >>> decoder = StreamingDecoder(stream, asn1Spec=schema) + >>> for asn1Object in decoder: + ... print(asn1Object) + >>> + SequenceOf: + 1 2 3 + """ + + SINGLE_ITEM_DECODER = SingleItemDecoder + + def __init__(self, substrate, asn1Spec=None, **options): + self._singleItemDecoder = self.SINGLE_ITEM_DECODER(**options) + self._substrate = asSeekableStream(substrate) + self._asn1Spec = asn1Spec + self._options = options + + def __iter__(self): + while True: + for asn1Object in self._singleItemDecoder( + self._substrate, self._asn1Spec, **self._options): + yield asn1Object + + for chunk in isEndOfStream(self._substrate): + if isinstance(chunk, SubstrateUnderrunError): + yield + + break + + if chunk: + break + + +class Decoder(object): + """Create a BER decoder object. + + Parse BER/CER/DER octet-stream into one, possibly nested, ASN.1 object. + """ + STREAMING_DECODER = StreamingDecoder + + @classmethod + def __call__(cls, substrate, asn1Spec=None, **options): + """Turns BER/CER/DER octet stream into an ASN.1 object. + + Takes BER/CER/DER octet-stream in form of :py:class:`bytes` (Python 3) + or :py:class:`str` (Python 2) and decode it into an ASN.1 object + (e.g. :py:class:`~pyasn1.type.base.PyAsn1Item` derivative) which + may be a scalar or an arbitrary nested structure. + + Parameters + ---------- + substrate: :py:class:`bytes` (Python 3) or :py:class:`str` (Python 2) + BER/CER/DER octet-stream to parse + + Keyword Args + ------------ + asn1Spec: :py:class:`~pyasn1.type.base.PyAsn1Item` + A pyasn1 type object (:py:class:`~pyasn1.type.base.PyAsn1Item` + derivative) to act as a template guiding the decoder. + Depending on the ASN.1 structure being decoded, `asn1Spec` may or + may not be required. Most common reason for it to require is that + ASN.1 structure is encoded in *IMPLICIT* tagging mode. + + Returns + ------- + : :py:class:`tuple` + A tuple of :py:class:`~pyasn1.type.base.PyAsn1Item` object + recovered from BER/CER/DER substrate and the unprocessed trailing + portion of the `substrate` (may be empty) + + Raises + ------ + : :py:class:`~pyasn1.error.PyAsn1Error` + :py:class:`~pyasn1.error.SubstrateUnderrunError` on insufficient + input or :py:class:`~pyasn1.error.PyAsn1Error` on decoding error. + + Examples + -------- + Decode BER/CER/DER serialisation without ASN.1 schema + + .. code-block:: pycon + + >>> s, unprocessed = decode(b'0\t\x02\x01\x01\x02\x01\x02\x02\x01\x03') + >>> str(s) + SequenceOf: + 1 2 3 + + Decode BER/CER/DER serialisation with ASN.1 schema + + .. code-block:: pycon + + >>> seq = SequenceOf(componentType=Integer()) + >>> s, unprocessed = decode( + b'0\t\x02\x01\x01\x02\x01\x02\x02\x01\x03', asn1Spec=seq) + >>> str(s) + SequenceOf: + 1 2 3 + + """ + substrate = asSeekableStream(substrate) + + streamingDecoder = cls.STREAMING_DECODER( + substrate, asn1Spec, **options) + + for asn1Object in streamingDecoder: + if isinstance(asn1Object, SubstrateUnderrunError): + raise error.SubstrateUnderrunError('Short substrate on input') + + try: + tail = next(readFromStream(substrate)) + + except error.EndOfStreamError: + tail = null + + return asn1Object, tail #: Turns BER octet stream into an ASN.1 object. @@ -1655,6 +2037,11 @@ class Decoder(object): #: ~pyasn1.error.PyAsn1Error, ~pyasn1.error.SubstrateUnderrunError #: On decoding errors #: +#: Notes +#: ----- +#: This function is deprecated. Please use :py:class:`Decoder` or +#: :py:class:`StreamingDecoder` class instance. +#: #: Examples #: -------- #: Decode BER serialisation without ASN.1 schema @@ -1676,7 +2063,4 @@ class Decoder(object): #: SequenceOf: #: 1 2 3 #: -decode = Decoder(tagMap, typeMap) - -# XXX -# non-recursive decoding; return position rather than substrate +decode = Decoder() diff --git a/pyasn1/codec/ber/encoder.py b/pyasn1/codec/ber/encoder.py index 778aa86..826ea73 100644 --- a/pyasn1/codec/ber/encoder.py +++ b/pyasn1/codec/ber/encoder.py @@ -17,7 +17,7 @@ from pyasn1.type import tag from pyasn1.type import univ from pyasn1.type import useful -__all__ = ['encode'] +__all__ = ['Encoder', 'encode'] LOG = debug.registerLoggee(__name__, flags=debug.DEBUG_ENCODER) @@ -706,7 +706,7 @@ class AnyEncoder(OctetStringEncoder): return value, not options.get('defMode', True), True -tagMap = { +TAG_MAP = { eoo.endOfOctets.tagSet: EndOfOctetsEncoder(), univ.Boolean.tagSet: BooleanEncoder(), univ.Integer.tagSet: IntegerEncoder(), @@ -739,7 +739,7 @@ tagMap = { } # Put in ambiguous & non-ambiguous types for faster codec lookup -typeMap = { +TYPE_MAP = { univ.Boolean.typeId: BooleanEncoder(), univ.Integer.typeId: IntegerEncoder(), univ.BitString.typeId: BitStringEncoder(), @@ -774,14 +774,16 @@ typeMap = { } -class Encoder(object): +class SingleItemEncoder(object): fixedDefLengthMode = None fixedChunkSize = None - # noinspection PyDefaultArgument - def __init__(self, tagMap, typeMap={}): - self.__tagMap = tagMap - self.__typeMap = typeMap + TAG_MAP = TAG_MAP + TYPE_MAP = TYPE_MAP + + def __init__(self, **options): + self._tagMap = options.get('tagMap', self.TAG_MAP) + self._typeMap = options.get('typeMap', self.TYPE_MAP) def __call__(self, value, asn1Spec=None, **options): try: @@ -795,8 +797,11 @@ class Encoder(object): 'and "asn1Spec" not given' % (value,)) if LOG: - LOG('encoder called in %sdef mode, chunk size %s for ' - 'type %s, value:\n%s' % (not options.get('defMode', True) and 'in' or '', options.get('maxChunkSize', 0), asn1Spec is None and value.prettyPrintType() or asn1Spec.prettyPrintType(), value)) + LOG('encoder called in %sdef mode, chunk size %s for type %s, ' + 'value:\n%s' % (not options.get('defMode', True) and 'in' or '', + options.get('maxChunkSize', 0), + asn1Spec is None and value.prettyPrintType() or + asn1Spec.prettyPrintType(), value)) if self.fixedDefLengthMode is not None: options.update(defMode=self.fixedDefLengthMode) @@ -804,12 +809,12 @@ class Encoder(object): if self.fixedChunkSize is not None: options.update(maxChunkSize=self.fixedChunkSize) - try: - concreteEncoder = self.__typeMap[typeId] + concreteEncoder = self._typeMap[typeId] if LOG: - LOG('using value codec %s chosen by type ID %s' % (concreteEncoder.__class__.__name__, typeId)) + LOG('using value codec %s chosen by type ID ' + '%s' % (concreteEncoder.__class__.__name__, typeId)) except KeyError: if asn1Spec is None: @@ -821,21 +826,36 @@ class Encoder(object): baseTagSet = tag.TagSet(tagSet.baseTag, tagSet.baseTag) try: - concreteEncoder = self.__tagMap[baseTagSet] + concreteEncoder = self._tagMap[baseTagSet] except KeyError: raise error.PyAsn1Error('No encoder for %r (%s)' % (value, tagSet)) if LOG: - LOG('using value codec %s chosen by tagSet %s' % (concreteEncoder.__class__.__name__, tagSet)) + LOG('using value codec %s chosen by tagSet ' + '%s' % (concreteEncoder.__class__.__name__, tagSet)) substrate = concreteEncoder.encode(value, asn1Spec, self, **options) if LOG: - LOG('codec %s built %s octets of substrate: %s\nencoder completed' % (concreteEncoder, len(substrate), debug.hexdump(substrate))) + LOG('codec %s built %s octets of substrate: %s\nencoder ' + 'completed' % (concreteEncoder, len(substrate), + debug.hexdump(substrate))) return substrate + +class Encoder(object): + SINGLE_ITEM_ENCODER = SingleItemEncoder + + def __init__(self, **options): + self._singleItemEncoder = self.SINGLE_ITEM_ENCODER(**options) + + def __call__(self, pyObject, asn1Spec=None, **options): + return self._singleItemEncoder( + pyObject, asn1Spec=asn1Spec, **options) + + #: Turns ASN.1 object into BER octet stream. #: #: Takes any ASN.1 object (e.g. :py:class:`~pyasn1.type.base.PyAsn1Item` derivative) @@ -887,4 +907,4 @@ class Encoder(object): #: >>> encode(seq) #: b'0\t\x02\x01\x01\x02\x01\x02\x02\x01\x03' #: -encode = Encoder(tagMap, typeMap) +encode = Encoder() diff --git a/pyasn1/codec/cer/decoder.py b/pyasn1/codec/cer/decoder.py index 3e86fd0..0a92b26 100644 --- a/pyasn1/codec/cer/decoder.py +++ b/pyasn1/codec/cer/decoder.py @@ -5,60 +5,88 @@ # License: http://snmplabs.com/pyasn1/license.html # from pyasn1 import error +from pyasn1.codec.streaming import readFromStream from pyasn1.codec.ber import decoder from pyasn1.compat.octets import oct2int from pyasn1.type import univ -__all__ = ['decode'] +__all__ = ['decode', 'StreamingDecoder'] +SubstrateUnderrunError = error.SubstrateUnderrunError -class BooleanDecoder(decoder.AbstractSimpleDecoder): + +class BooleanPayloadDecoder(decoder.AbstractSimplePayloadDecoder): protoComponent = univ.Boolean(0) def valueDecoder(self, substrate, asn1Spec, tagSet=None, length=None, state=None, decodeFun=None, substrateFun=None, **options): - head, tail = substrate[:length], substrate[length:] - if not head or length != 1: + + if length != 1: raise error.PyAsn1Error('Not single-octet Boolean payload') - byte = oct2int(head[0]) + + for chunk in readFromStream(substrate, length, options): + if isinstance(chunk, SubstrateUnderrunError): + yield chunk + + byte = oct2int(chunk[0]) + # CER/DER specifies encoding of TRUE as 0xFF and FALSE as 0x0, while # BER allows any non-zero value as TRUE; cf. sections 8.2.2. and 11.1 # in https://www.itu.int/ITU-T/studygroups/com17/languages/X.690-0207.pdf if byte == 0xff: value = 1 + elif byte == 0x00: value = 0 + else: raise error.PyAsn1Error('Unexpected Boolean payload: %s' % byte) - return self._createComponent(asn1Spec, tagSet, value, **options), tail + + yield self._createComponent(asn1Spec, tagSet, value, **options) + # TODO: prohibit non-canonical encoding -BitStringDecoder = decoder.BitStringDecoder -OctetStringDecoder = decoder.OctetStringDecoder -RealDecoder = decoder.RealDecoder - -tagMap = decoder.tagMap.copy() -tagMap.update( - {univ.Boolean.tagSet: BooleanDecoder(), - univ.BitString.tagSet: BitStringDecoder(), - univ.OctetString.tagSet: OctetStringDecoder(), - univ.Real.tagSet: RealDecoder()} +BitStringPayloadDecoder = decoder.BitStringPayloadDecoder +OctetStringPayloadDecoder = decoder.OctetStringPayloadDecoder +RealPayloadDecoder = decoder.RealPayloadDecoder + +TAG_MAP = decoder.TAG_MAP.copy() +TAG_MAP.update( + {univ.Boolean.tagSet: BooleanPayloadDecoder(), + univ.BitString.tagSet: BitStringPayloadDecoder(), + univ.OctetString.tagSet: OctetStringPayloadDecoder(), + univ.Real.tagSet: RealPayloadDecoder()} ) -typeMap = decoder.typeMap.copy() +TYPE_MAP = decoder.TYPE_MAP.copy() # Put in non-ambiguous types for faster codec lookup -for typeDecoder in tagMap.values(): +for typeDecoder in TAG_MAP.values(): if typeDecoder.protoComponent is not None: typeId = typeDecoder.protoComponent.__class__.typeId - if typeId is not None and typeId not in typeMap: - typeMap[typeId] = typeDecoder + if typeId is not None and typeId not in TYPE_MAP: + TYPE_MAP[typeId] = typeDecoder + + +class SingleItemDecoder(decoder.SingleItemDecoder): + __doc__ = decoder.SingleItemDecoder.__doc__ + + TAG_MAP = TAG_MAP + TYPE_MAP = TYPE_MAP + + +class StreamingDecoder(decoder.StreamingDecoder): + __doc__ = decoder.StreamingDecoder.__doc__ + + SINGLE_ITEM_DECODER = SingleItemDecoder class Decoder(decoder.Decoder): - pass + __doc__ = decoder.Decoder.__doc__ + + STREAMING_DECODER = StreamingDecoder #: Turns CER octet stream into an ASN.1 object. @@ -111,4 +139,4 @@ class Decoder(decoder.Decoder): #: SequenceOf: #: 1 2 3 #: -decode = Decoder(tagMap, decoder.typeMap) +decode = Decoder() diff --git a/pyasn1/codec/cer/encoder.py b/pyasn1/codec/cer/encoder.py index 935b696..9e6cdac 100644 --- a/pyasn1/codec/cer/encoder.py +++ b/pyasn1/codec/cer/encoder.py @@ -10,7 +10,7 @@ from pyasn1.compat.octets import str2octs, null from pyasn1.type import univ from pyasn1.type import useful -__all__ = ['encode'] +__all__ = ['Encoder', 'encode'] class BooleanEncoder(encoder.IntegerEncoder): @@ -234,8 +234,9 @@ class SequenceEncoder(encoder.SequenceEncoder): omitEmptyOptionals = True -tagMap = encoder.tagMap.copy() -tagMap.update({ +TAG_MAP = encoder.TAG_MAP.copy() + +TAG_MAP.update({ univ.Boolean.tagSet: BooleanEncoder(), univ.Real.tagSet: RealEncoder(), useful.GeneralizedTime.tagSet: GeneralizedTimeEncoder(), @@ -245,8 +246,9 @@ tagMap.update({ univ.Sequence.typeId: SequenceEncoder() }) -typeMap = encoder.typeMap.copy() -typeMap.update({ +TYPE_MAP = encoder.TYPE_MAP.copy() + +TYPE_MAP.update({ univ.Boolean.typeId: BooleanEncoder(), univ.Real.typeId: RealEncoder(), useful.GeneralizedTime.typeId: GeneralizedTimeEncoder(), @@ -259,10 +261,18 @@ typeMap.update({ }) -class Encoder(encoder.Encoder): +class SingleItemEncoder(encoder.SingleItemEncoder): fixedDefLengthMode = False fixedChunkSize = 1000 + TAG_MAP = TAG_MAP + TYPE_MAP = TYPE_MAP + + +class Encoder(encoder.Encoder): + SINGLE_ITEM_ENCODER = SingleItemEncoder + + #: Turns ASN.1 object into CER octet stream. #: #: Takes any ASN.1 object (e.g. :py:class:`~pyasn1.type.base.PyAsn1Item` derivative) @@ -308,6 +318,6 @@ class Encoder(encoder.Encoder): #: >>> encode(seq) #: b'0\x80\x02\x01\x01\x02\x01\x02\x02\x01\x03\x00\x00' #: -encode = Encoder(tagMap, typeMap) +encode = Encoder() # EncoderFactory queries class instance and builds a map of tags -> encoders diff --git a/pyasn1/codec/der/decoder.py b/pyasn1/codec/der/decoder.py index 1a13fdb..b9526c3 100644 --- a/pyasn1/codec/der/decoder.py +++ b/pyasn1/codec/der/decoder.py @@ -7,40 +7,58 @@ from pyasn1.codec.cer import decoder from pyasn1.type import univ -__all__ = ['decode'] +__all__ = ['decode', 'StreamingDecoder'] -class BitStringDecoder(decoder.BitStringDecoder): +class BitStringPayloadDecoder(decoder.BitStringPayloadDecoder): supportConstructedForm = False -class OctetStringDecoder(decoder.OctetStringDecoder): +class OctetStringPayloadDecoder(decoder.OctetStringPayloadDecoder): supportConstructedForm = False + # TODO: prohibit non-canonical encoding -RealDecoder = decoder.RealDecoder +RealPayloadDecoder = decoder.RealPayloadDecoder -tagMap = decoder.tagMap.copy() -tagMap.update( - {univ.BitString.tagSet: BitStringDecoder(), - univ.OctetString.tagSet: OctetStringDecoder(), - univ.Real.tagSet: RealDecoder()} +TAG_MAP = decoder.TAG_MAP.copy() +TAG_MAP.update( + {univ.BitString.tagSet: BitStringPayloadDecoder(), + univ.OctetString.tagSet: OctetStringPayloadDecoder(), + univ.Real.tagSet: RealPayloadDecoder()} ) -typeMap = decoder.typeMap.copy() +TYPE_MAP = decoder.TYPE_MAP.copy() # Put in non-ambiguous types for faster codec lookup -for typeDecoder in tagMap.values(): +for typeDecoder in TAG_MAP.values(): if typeDecoder.protoComponent is not None: typeId = typeDecoder.protoComponent.__class__.typeId - if typeId is not None and typeId not in typeMap: - typeMap[typeId] = typeDecoder + if typeId is not None and typeId not in TYPE_MAP: + TYPE_MAP[typeId] = typeDecoder -class Decoder(decoder.Decoder): +class SingleItemDecoder(decoder.SingleItemDecoder): + __doc__ = decoder.SingleItemDecoder.__doc__ + + TAG_MAP = TAG_MAP + TYPE_MAP = TYPE_MAP + supportIndefLength = False +class StreamingDecoder(decoder.StreamingDecoder): + __doc__ = decoder.StreamingDecoder.__doc__ + + SINGLE_ITEM_DECODER = SingleItemDecoder + + +class Decoder(decoder.Decoder): + __doc__ = decoder.Decoder.__doc__ + + STREAMING_DECODER = StreamingDecoder + + #: Turns DER octet stream into an ASN.1 object. #: #: Takes DER octet-stream and decode it into an ASN.1 object @@ -91,4 +109,4 @@ class Decoder(decoder.Decoder): #: SequenceOf: #: 1 2 3 #: -decode = Decoder(tagMap, typeMap) +decode = Decoder() diff --git a/pyasn1/codec/der/encoder.py b/pyasn1/codec/der/encoder.py index 90e982d..1a6af82 100644 --- a/pyasn1/codec/der/encoder.py +++ b/pyasn1/codec/der/encoder.py @@ -8,7 +8,7 @@ from pyasn1 import error from pyasn1.codec.cer import encoder from pyasn1.type import univ -__all__ = ['encode'] +__all__ = ['Encoder', 'encode'] class SetEncoder(encoder.SetEncoder): @@ -42,23 +42,34 @@ class SetEncoder(encoder.SetEncoder): else: return compType.tagSet -tagMap = encoder.tagMap.copy() -tagMap.update({ + +TAG_MAP = encoder.TAG_MAP.copy() + +TAG_MAP.update({ # Set & SetOf have same tags univ.Set.tagSet: SetEncoder() }) -typeMap = encoder.typeMap.copy() -typeMap.update({ +TYPE_MAP = encoder.TYPE_MAP.copy() + +TYPE_MAP.update({ # Set & SetOf have same tags univ.Set.typeId: SetEncoder() }) -class Encoder(encoder.Encoder): +class SingleItemEncoder(encoder.SingleItemEncoder): fixedDefLengthMode = True fixedChunkSize = 0 + TAG_MAP = TAG_MAP + TYPE_MAP = TYPE_MAP + + +class Encoder(encoder.Encoder): + SINGLE_ITEM_ENCODER = SingleItemEncoder + + #: Turns ASN.1 object into DER octet stream. #: #: Takes any ASN.1 object (e.g. :py:class:`~pyasn1.type.base.PyAsn1Item` derivative) @@ -104,4 +115,4 @@ class Encoder(encoder.Encoder): #: >>> encode(seq) #: b'0\t\x02\x01\x01\x02\x01\x02\x02\x01\x03' #: -encode = Encoder(tagMap, typeMap) +encode = Encoder() diff --git a/pyasn1/codec/native/decoder.py b/pyasn1/codec/native/decoder.py index 104b92e..1838b7d 100644 --- a/pyasn1/codec/native/decoder.py +++ b/pyasn1/codec/native/decoder.py @@ -17,17 +17,17 @@ __all__ = ['decode'] LOG = debug.registerLoggee(__name__, flags=debug.DEBUG_DECODER) -class AbstractScalarDecoder(object): +class AbstractScalarPayloadDecoder(object): def __call__(self, pyObject, asn1Spec, decodeFun=None, **options): return asn1Spec.clone(pyObject) -class BitStringDecoder(AbstractScalarDecoder): +class BitStringPayloadDecoder(AbstractScalarPayloadDecoder): def __call__(self, pyObject, asn1Spec, decodeFun=None, **options): return asn1Spec.clone(univ.BitString.fromBinaryString(pyObject)) -class SequenceOrSetDecoder(object): +class SequenceOrSetPayloadDecoder(object): def __call__(self, pyObject, asn1Spec, decodeFun=None, **options): asn1Value = asn1Spec.clone() @@ -40,7 +40,7 @@ class SequenceOrSetDecoder(object): return asn1Value -class SequenceOfOrSetOfDecoder(object): +class SequenceOfOrSetOfPayloadDecoder(object): def __call__(self, pyObject, asn1Spec, decodeFun=None, **options): asn1Value = asn1Spec.clone() @@ -50,7 +50,7 @@ class SequenceOfOrSetOfDecoder(object): return asn1Value -class ChoiceDecoder(object): +class ChoicePayloadDecoder(object): def __call__(self, pyObject, asn1Spec, decodeFun=None, **options): asn1Value = asn1Spec.clone() @@ -64,112 +64,132 @@ class ChoiceDecoder(object): return asn1Value -tagMap = { - univ.Integer.tagSet: AbstractScalarDecoder(), - univ.Boolean.tagSet: AbstractScalarDecoder(), - univ.BitString.tagSet: BitStringDecoder(), - univ.OctetString.tagSet: AbstractScalarDecoder(), - univ.Null.tagSet: AbstractScalarDecoder(), - univ.ObjectIdentifier.tagSet: AbstractScalarDecoder(), - univ.Enumerated.tagSet: AbstractScalarDecoder(), - univ.Real.tagSet: AbstractScalarDecoder(), - univ.Sequence.tagSet: SequenceOrSetDecoder(), # conflicts with SequenceOf - univ.Set.tagSet: SequenceOrSetDecoder(), # conflicts with SetOf - univ.Choice.tagSet: ChoiceDecoder(), # conflicts with Any +TAG_MAP = { + univ.Integer.tagSet: AbstractScalarPayloadDecoder(), + univ.Boolean.tagSet: AbstractScalarPayloadDecoder(), + univ.BitString.tagSet: BitStringPayloadDecoder(), + univ.OctetString.tagSet: AbstractScalarPayloadDecoder(), + univ.Null.tagSet: AbstractScalarPayloadDecoder(), + univ.ObjectIdentifier.tagSet: AbstractScalarPayloadDecoder(), + univ.Enumerated.tagSet: AbstractScalarPayloadDecoder(), + univ.Real.tagSet: AbstractScalarPayloadDecoder(), + univ.Sequence.tagSet: SequenceOrSetPayloadDecoder(), # conflicts with SequenceOf + univ.Set.tagSet: SequenceOrSetPayloadDecoder(), # conflicts with SetOf + univ.Choice.tagSet: ChoicePayloadDecoder(), # conflicts with Any # character string types - char.UTF8String.tagSet: AbstractScalarDecoder(), - char.NumericString.tagSet: AbstractScalarDecoder(), - char.PrintableString.tagSet: AbstractScalarDecoder(), - char.TeletexString.tagSet: AbstractScalarDecoder(), - char.VideotexString.tagSet: AbstractScalarDecoder(), - char.IA5String.tagSet: AbstractScalarDecoder(), - char.GraphicString.tagSet: AbstractScalarDecoder(), - char.VisibleString.tagSet: AbstractScalarDecoder(), - char.GeneralString.tagSet: AbstractScalarDecoder(), - char.UniversalString.tagSet: AbstractScalarDecoder(), - char.BMPString.tagSet: AbstractScalarDecoder(), + char.UTF8String.tagSet: AbstractScalarPayloadDecoder(), + char.NumericString.tagSet: AbstractScalarPayloadDecoder(), + char.PrintableString.tagSet: AbstractScalarPayloadDecoder(), + char.TeletexString.tagSet: AbstractScalarPayloadDecoder(), + char.VideotexString.tagSet: AbstractScalarPayloadDecoder(), + char.IA5String.tagSet: AbstractScalarPayloadDecoder(), + char.GraphicString.tagSet: AbstractScalarPayloadDecoder(), + char.VisibleString.tagSet: AbstractScalarPayloadDecoder(), + char.GeneralString.tagSet: AbstractScalarPayloadDecoder(), + char.UniversalString.tagSet: AbstractScalarPayloadDecoder(), + char.BMPString.tagSet: AbstractScalarPayloadDecoder(), # useful types - useful.ObjectDescriptor.tagSet: AbstractScalarDecoder(), - useful.GeneralizedTime.tagSet: AbstractScalarDecoder(), - useful.UTCTime.tagSet: AbstractScalarDecoder() + useful.ObjectDescriptor.tagSet: AbstractScalarPayloadDecoder(), + useful.GeneralizedTime.tagSet: AbstractScalarPayloadDecoder(), + useful.UTCTime.tagSet: AbstractScalarPayloadDecoder() } # Put in ambiguous & non-ambiguous types for faster codec lookup -typeMap = { - univ.Integer.typeId: AbstractScalarDecoder(), - univ.Boolean.typeId: AbstractScalarDecoder(), - univ.BitString.typeId: BitStringDecoder(), - univ.OctetString.typeId: AbstractScalarDecoder(), - univ.Null.typeId: AbstractScalarDecoder(), - univ.ObjectIdentifier.typeId: AbstractScalarDecoder(), - univ.Enumerated.typeId: AbstractScalarDecoder(), - univ.Real.typeId: AbstractScalarDecoder(), +TYPE_MAP = { + univ.Integer.typeId: AbstractScalarPayloadDecoder(), + univ.Boolean.typeId: AbstractScalarPayloadDecoder(), + univ.BitString.typeId: BitStringPayloadDecoder(), + univ.OctetString.typeId: AbstractScalarPayloadDecoder(), + univ.Null.typeId: AbstractScalarPayloadDecoder(), + univ.ObjectIdentifier.typeId: AbstractScalarPayloadDecoder(), + univ.Enumerated.typeId: AbstractScalarPayloadDecoder(), + univ.Real.typeId: AbstractScalarPayloadDecoder(), # ambiguous base types - univ.Set.typeId: SequenceOrSetDecoder(), - univ.SetOf.typeId: SequenceOfOrSetOfDecoder(), - univ.Sequence.typeId: SequenceOrSetDecoder(), - univ.SequenceOf.typeId: SequenceOfOrSetOfDecoder(), - univ.Choice.typeId: ChoiceDecoder(), - univ.Any.typeId: AbstractScalarDecoder(), + univ.Set.typeId: SequenceOrSetPayloadDecoder(), + univ.SetOf.typeId: SequenceOfOrSetOfPayloadDecoder(), + univ.Sequence.typeId: SequenceOrSetPayloadDecoder(), + univ.SequenceOf.typeId: SequenceOfOrSetOfPayloadDecoder(), + univ.Choice.typeId: ChoicePayloadDecoder(), + univ.Any.typeId: AbstractScalarPayloadDecoder(), # character string types - char.UTF8String.typeId: AbstractScalarDecoder(), - char.NumericString.typeId: AbstractScalarDecoder(), - char.PrintableString.typeId: AbstractScalarDecoder(), - char.TeletexString.typeId: AbstractScalarDecoder(), - char.VideotexString.typeId: AbstractScalarDecoder(), - char.IA5String.typeId: AbstractScalarDecoder(), - char.GraphicString.typeId: AbstractScalarDecoder(), - char.VisibleString.typeId: AbstractScalarDecoder(), - char.GeneralString.typeId: AbstractScalarDecoder(), - char.UniversalString.typeId: AbstractScalarDecoder(), - char.BMPString.typeId: AbstractScalarDecoder(), + char.UTF8String.typeId: AbstractScalarPayloadDecoder(), + char.NumericString.typeId: AbstractScalarPayloadDecoder(), + char.PrintableString.typeId: AbstractScalarPayloadDecoder(), + char.TeletexString.typeId: AbstractScalarPayloadDecoder(), + char.VideotexString.typeId: AbstractScalarPayloadDecoder(), + char.IA5String.typeId: AbstractScalarPayloadDecoder(), + char.GraphicString.typeId: AbstractScalarPayloadDecoder(), + char.VisibleString.typeId: AbstractScalarPayloadDecoder(), + char.GeneralString.typeId: AbstractScalarPayloadDecoder(), + char.UniversalString.typeId: AbstractScalarPayloadDecoder(), + char.BMPString.typeId: AbstractScalarPayloadDecoder(), # useful types - useful.ObjectDescriptor.typeId: AbstractScalarDecoder(), - useful.GeneralizedTime.typeId: AbstractScalarDecoder(), - useful.UTCTime.typeId: AbstractScalarDecoder() + useful.ObjectDescriptor.typeId: AbstractScalarPayloadDecoder(), + useful.GeneralizedTime.typeId: AbstractScalarPayloadDecoder(), + useful.UTCTime.typeId: AbstractScalarPayloadDecoder() } -class Decoder(object): +class SingleItemDecoder(object): + + TAG_MAP = TAG_MAP + TYPE_MAP = TYPE_MAP - # noinspection PyDefaultArgument - def __init__(self, tagMap, typeMap): - self.__tagMap = tagMap - self.__typeMap = typeMap + def __init__(self, **options): + self._tagMap = options.get('tagMap', self.TAG_MAP) + self._typeMap = options.get('typeMap', self.TYPE_MAP) def __call__(self, pyObject, asn1Spec, **options): if LOG: debug.scope.push(type(pyObject).__name__) - LOG('decoder called at scope %s, working with type %s' % (debug.scope, type(pyObject).__name__)) + LOG('decoder called at scope %s, working with ' + 'type %s' % (debug.scope, type(pyObject).__name__)) if asn1Spec is None or not isinstance(asn1Spec, base.Asn1Item): - raise error.PyAsn1Error('asn1Spec is not valid (should be an instance of an ASN.1 Item, not %s)' % asn1Spec.__class__.__name__) + raise error.PyAsn1Error( + 'asn1Spec is not valid (should be an instance of an ASN.1 ' + 'Item, not %s)' % asn1Spec.__class__.__name__) try: - valueDecoder = self.__typeMap[asn1Spec.typeId] + valueDecoder = self._typeMap[asn1Spec.typeId] except KeyError: # use base type for codec lookup to recover untagged types baseTagSet = tag.TagSet(asn1Spec.tagSet.baseTag, asn1Spec.tagSet.baseTag) try: - valueDecoder = self.__tagMap[baseTagSet] + valueDecoder = self._tagMap[baseTagSet] + except KeyError: raise error.PyAsn1Error('Unknown ASN.1 tag %s' % asn1Spec.tagSet) if LOG: - LOG('calling decoder %s on Python type %s <%s>' % (type(valueDecoder).__name__, type(pyObject).__name__, repr(pyObject))) + LOG('calling decoder %s on Python type %s ' + '<%s>' % (type(valueDecoder).__name__, + type(pyObject).__name__, repr(pyObject))) value = valueDecoder(pyObject, asn1Spec, self, **options) if LOG: - LOG('decoder %s produced ASN.1 type %s <%s>' % (type(valueDecoder).__name__, type(value).__name__, repr(value))) + LOG('decoder %s produced ASN.1 type %s ' + '<%s>' % (type(valueDecoder).__name__, + type(value).__name__, repr(value))) debug.scope.pop() return value +class Decoder(object): + SINGLE_ITEM_DECODER = SingleItemDecoder + + def __init__(self, **options): + self._singleItemDecoder = self.SINGLE_ITEM_DECODER(**options) + + def __call__(self, pyObject, asn1Spec=None, **kwargs): + return self._singleItemDecoder(pyObject, asn1Spec=asn1Spec, **kwargs) + + #: Turns Python objects of built-in types into ASN.1 objects. #: #: Takes Python objects of built-in types and turns them into a tree of @@ -210,4 +230,4 @@ class Decoder(object): #: SequenceOf: #: 1 2 3 #: -decode = Decoder(tagMap, typeMap) +decode = Decoder() diff --git a/pyasn1/codec/native/encoder.py b/pyasn1/codec/native/encoder.py index a83d00a..6b8ca6d 100644 --- a/pyasn1/codec/native/encoder.py +++ b/pyasn1/codec/native/encoder.py @@ -103,7 +103,7 @@ class AnyEncoder(AbstractItemEncoder): return value.asOctets() -tagMap = { +TAG_MAP = { univ.Boolean.tagSet: BooleanEncoder(), univ.Integer.tagSet: IntegerEncoder(), univ.BitString.tagSet: BitStringEncoder(), @@ -136,7 +136,7 @@ tagMap = { # Put in ambiguous & non-ambiguous types for faster codec lookup -typeMap = { +TYPE_MAP = { univ.Boolean.typeId: BooleanEncoder(), univ.Integer.typeId: IntegerEncoder(), univ.BitString.typeId: BitStringEncoder(), @@ -171,48 +171,66 @@ typeMap = { } -class Encoder(object): +class SingleItemEncoder(object): + + TAG_MAP = TAG_MAP + TYPE_MAP = TYPE_MAP - # noinspection PyDefaultArgument - def __init__(self, tagMap, typeMap={}): - self.__tagMap = tagMap - self.__typeMap = typeMap + def __init__(self, **options): + self._tagMap = options.get('tagMap', self.TAG_MAP) + self._typeMap = options.get('typeMap', self.TYPE_MAP) def __call__(self, value, **options): if not isinstance(value, base.Asn1Item): - raise error.PyAsn1Error('value is not valid (should be an instance of an ASN.1 Item)') + raise error.PyAsn1Error( + 'value is not valid (should be an instance of an ASN.1 Item)') if LOG: debug.scope.push(type(value).__name__) - LOG('encoder called for type %s <%s>' % (type(value).__name__, value.prettyPrint())) + LOG('encoder called for type %s ' + '<%s>' % (type(value).__name__, value.prettyPrint())) tagSet = value.tagSet try: - concreteEncoder = self.__typeMap[value.typeId] + concreteEncoder = self._typeMap[value.typeId] except KeyError: # use base type for codec lookup to recover untagged types - baseTagSet = tag.TagSet(value.tagSet.baseTag, value.tagSet.baseTag) + baseTagSet = tag.TagSet( + value.tagSet.baseTag, value.tagSet.baseTag) try: - concreteEncoder = self.__tagMap[baseTagSet] + concreteEncoder = self._tagMap[baseTagSet] except KeyError: raise error.PyAsn1Error('No encoder for %s' % (value,)) if LOG: - LOG('using value codec %s chosen by %s' % (concreteEncoder.__class__.__name__, tagSet)) + LOG('using value codec %s chosen by ' + '%s' % (concreteEncoder.__class__.__name__, tagSet)) pyObject = concreteEncoder.encode(value, self, **options) if LOG: - LOG('encoder %s produced: %s' % (type(concreteEncoder).__name__, repr(pyObject))) + LOG('encoder %s produced: ' + '%s' % (type(concreteEncoder).__name__, repr(pyObject))) debug.scope.pop() return pyObject +class Encoder(object): + SINGLE_ITEM_ENCODER = SingleItemEncoder + + def __init__(self, **options): + self._singleItemEncoder = self.SINGLE_ITEM_ENCODER(**options) + + def __call__(self, pyObject, asn1Spec=None, **options): + return self._singleItemEncoder( + pyObject, asn1Spec=asn1Spec, **options) + + #: Turns ASN.1 object into a Python built-in type object(s). #: #: Takes any ASN.1 object (e.g. :py:class:`~pyasn1.type.base.PyAsn1Item` derivative) @@ -248,4 +266,4 @@ class Encoder(object): #: >>> encode(seq) #: [1, 2, 3] #: -encode = Encoder(tagMap, typeMap) +encode = SingleItemEncoder() diff --git a/pyasn1/codec/streaming.py b/pyasn1/codec/streaming.py new file mode 100644 index 0000000..6d0146b --- /dev/null +++ b/pyasn1/codec/streaming.py @@ -0,0 +1,243 @@ +# +# This file is part of pyasn1 software. +# +# Copyright (c) 2005-2019, Ilya Etingof <etingof@gmail.com> +# License: http://snmplabs.com/pyasn1/license.html +# +import io +import os +import sys + +from pyasn1 import error +from pyasn1.type import univ + +_PY2 = sys.version_info < (3,) + + +class CachingStreamWrapper(io.IOBase): + """Wrapper around non-seekable streams. + + Note that the implementation is tied to the decoder, + not checking for dangerous arguments for the sake + of performance. + + The read bytes are kept in an internal cache until + setting _markedPosition which may reset the cache. + """ + def __init__(self, raw): + self._raw = raw + self._cache = io.BytesIO() + self._markedPosition = 0 + + def peek(self, n): + result = self.read(n) + self._cache.seek(-len(result), os.SEEK_CUR) + return result + + def seekable(self): + return True + + def seek(self, n=-1, whence=os.SEEK_SET): + # Note that this not safe for seeking forward. + return self._cache.seek(n, whence) + + def read(self, n=-1): + read_from_cache = self._cache.read(n) + if n != -1: + n -= len(read_from_cache) + if not n: # 0 bytes left to read + return read_from_cache + + read_from_raw = self._raw.read(n) + + self._cache.write(read_from_raw) + + return read_from_cache + read_from_raw + + @property + def markedPosition(self): + """Position where the currently processed element starts. + + This is used for back-tracking in SingleItemDecoder.__call__ + and (indefLen)ValueDecoder and should not be used for other purposes. + The client is not supposed to ever seek before this position. + """ + return self._markedPosition + + @markedPosition.setter + def markedPosition(self, value): + # By setting the value, we ensure we won't seek back before it. + # `value` should be the same as the current position + # We don't check for this for performance reasons. + self._markedPosition = value + + # Whenever we set _marked_position, we know for sure + # that we will not return back, and thus it is + # safe to drop all cached data. + if self._cache.tell() > io.DEFAULT_BUFFER_SIZE: + self._cache = io.BytesIO(self._cache.read()) + self._markedPosition = 0 + + def tell(self): + return self._cache.tell() + + +def asSeekableStream(substrate): + """Convert object to seekable byte-stream. + + Parameters + ---------- + substrate: :py:class:`bytes` or :py:class:`io.IOBase` or :py:class:`univ.OctetString` + + Returns + ------- + : :py:class:`io.IOBase` + + Raises + ------ + : :py:class:`~pyasn1.error.PyAsn1Error` + If the supplied substrate cannot be converted to a seekable stream. + """ + if isinstance(substrate, io.BytesIO): + return substrate + + elif isinstance(substrate, bytes): + return io.BytesIO(substrate) + + elif isinstance(substrate, univ.OctetString): + return io.BytesIO(substrate.asOctets()) + + try: + # Special case: impossible to set attributes on `file` built-in + if _PY2 and isinstance(substrate, file): + return io.BufferedReader(substrate) + + elif substrate.seekable(): # Will fail for most invalid types + return substrate + + else: + return CachingStreamWrapper(substrate) + + except AttributeError: + raise error.UnsupportedSubstrateError( + "Cannot convert " + substrate.__class__.__name__ + + " to a seekable bit stream.") + + +def isEndOfStream(substrate): + """Check whether we have reached the end of a stream. + + Although it is more effective to read and catch exceptions, this + function + + Parameters + ---------- + substrate: :py:class:`IOBase` + Stream to check + + Returns + ------- + : :py:class:`bool` + """ + if isinstance(substrate, io.BytesIO): + cp = substrate.tell() + substrate.seek(0, os.SEEK_END) + result = substrate.tell() == cp + substrate.seek(cp, os.SEEK_SET) + yield result + + else: + received = substrate.read(1) + if received is None: + yield + + if received: + substrate.seek(-1, os.SEEK_CUR) + + yield not received + + +def peekIntoStream(substrate, size=-1): + """Peek into stream. + + Parameters + ---------- + substrate: :py:class:`IOBase` + Stream to read from. + + size: :py:class:`int` + How many bytes to peek (-1 = all available) + + Returns + ------- + : :py:class:`bytes` or :py:class:`str` + The return type depends on Python major version + """ + if hasattr(substrate, "peek"): + received = substrate.peek(size) + if received is None: + yield + + while len(received) < size: + yield + + yield received + + else: + current_position = substrate.tell() + try: + for chunk in readFromStream(substrate, size): + yield chunk + + finally: + substrate.seek(current_position) + + +def readFromStream(substrate, size=-1, context=None): + """Read from the stream. + + Parameters + ---------- + substrate: :py:class:`IOBase` + Stream to read from. + + Keyword parameters + ------------------ + size: :py:class:`int` + How many bytes to read (-1 = all available) + + context: :py:class:`dict` + Opaque caller context will be attached to exception objects created + by this function. + + Yields + ------ + : :py:class:`bytes` or :py:class:`str` or :py:class:`SubstrateUnderrunError` + Read data or :py:class:`~pyasn1.error.SubstrateUnderrunError` + object if no `size` bytes is readily available in the stream. The + data type depends on Python major version + + Raises + ------ + : :py:class:`~pyasn1.error.EndOfStreamError` + Input stream is exhausted + """ + while True: + # this will block unless stream is non-blocking + received = substrate.read(size) + if received is None: # non-blocking stream can do this + yield error.SubstrateUnderrunError(context=context) + + elif not received and size != 0: # end-of-stream + raise error.EndOfStreamError(context=context) + + elif len(received) < size: + substrate.seek(-len(received), os.SEEK_CUR) + + # behave like a non-blocking stream + yield error.SubstrateUnderrunError(context=context) + + else: + break + + yield received |