Add `SET|SEQUENCE OF ANY` encoding support (#165)

For example:

  AttributeTypeAndValues ::= SEQUENCE {
    type OBJECT IDENTIFIER,
    values SET OF ANY DEFINED BY type
  }

This patch adds support of the above ASN.1 syntax to BER/DER/CER
codecs.

It appears that to implement this feature properly, `SetOf`/`SequenceOf`
pyasn1 types need to have `.componentType` wrapped into something
similar to `NamedType` that `Set`/`Sequence` have. That additional
layer would then carry the open type meta information. Without it,
`Sequence`/`Set` codec needs to signal `SetOf`/`SequenceOf` codec
of the open type being processed, which is a slight hack.

A other inconvenience is that when `SetOf`/`SequenceOf` deal with
an open type component, they should not verify types on component
assignment. Without open type property in `SetOf`/`SequenceOf`,
the code checks for `Any` component type which is another hack.

The above shortcomings should be addressed in the follow up patch.

3 files changed, 124 insertions, 38 deletions

diff --git a/pyasn1/codec/ber/decoder.py b/pyasn1/codec/ber/decoder.py
index 8c556fc..655af04 100644
--- a/pyasn1/codec/ber/decoder.py
+++ b/pyasn1/codec/ber/decoder.py
@@ -671,12 +671,28 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder):
                                 LOG('resolved open type %r by governing '
                                     'value %r' % (openType, governingValue))
 
-                            component, rest = decodeFun(
-                                asn1Object.getComponentByPosition(idx).asOctets(),
-                                asn1Spec=openType
-                            )
+                            containerValue = asn1Object.getComponentByPosition(idx)
+
+                            if containerValue.typeId in (
+                                    univ.SetOf.typeId, univ.SequenceOf.typeId):
+
+                                for pos, containerElement in enumerate(
+                                        containerValue):
+
+                                    component, rest = decodeFun(
+                                        containerValue[pos].asOctets(),
+                                        asn1Spec=openType
+                                    )
 
-                            asn1Object.setComponentByPosition(idx, component)
+                                    containerValue[pos] = component
+
+                            else:
+                                component, rest = decodeFun(
+                                    asn1Object.getComponentByPosition(idx).asOctets(),
+                                    asn1Spec=openType
+                                )
+
+                                asn1Object.setComponentByPosition(idx, component)
 
             else:
                 asn1Object.verifySizeSpec()
@@ -799,7 +815,7 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder):
                 if not namedTypes.requiredComponents.issubset(seenIndices):
                     raise error.PyAsn1Error('ASN.1 object %s has uninitialized components' % asn1Object.__class__.__name__)
 
-                if  namedTypes.hasOpenTypes:
+                if namedTypes.hasOpenTypes:
 
                     openTypes = options.get('openTypes', {})
 
@@ -837,13 +853,29 @@ class UniversalConstructedTypeDecoder(AbstractConstructedDecoder):
                                 LOG('resolved open type %r by governing '
                                     'value %r' % (openType, governingValue))
 
-                            component, rest = decodeFun(
-                                asn1Object.getComponentByPosition(idx).asOctets(),
-                                asn1Spec=openType, allowEoo=True
-                            )
+                            containerValue = asn1Object.getComponentByPosition(idx)
 
-                            if component is not eoo.endOfOctets:
-                                asn1Object.setComponentByPosition(idx, component)
+                            if containerValue.typeId in (
+                                    univ.SetOf.typeId, univ.SequenceOf.typeId):
+
+                                for pos, containerElement in enumerate(
+                                        containerValue):
+
+                                    component, rest = decodeFun(
+                                        containerValue[pos].asOctets(),
+                                        asn1Spec=openType, allowEoo=True
+                                    )
+
+                                    containerValue[pos] = component
+
+                            else:
+                                component, rest = decodeFun(
+                                    asn1Object.getComponentByPosition(idx).asOctets(),
+                                    asn1Spec=openType, allowEoo=True
+                                )
+
+                                if component is not eoo.endOfOctets:
+                                    asn1Object.setComponentByPosition(idx, component)
 
                 else:
                     asn1Object.verifySizeSpec()
diff --git a/pyasn1/codec/ber/encoder.py b/pyasn1/codec/ber/encoder.py
index 325ed46..b17fca8 100644
--- a/pyasn1/codec/ber/encoder.py
+++ b/pyasn1/codec/ber/encoder.py
@@ -89,6 +89,8 @@ class AbstractItemEncoder(object):
 
         defMode = options.get('defMode', True)
 
+        substrate = null
+
         for idx, singleTag in enumerate(tagSet.superTags):
 
             defModeOverride = defMode
@@ -556,18 +558,32 @@ class SequenceEncoder(AbstractItemEncoder):
                     if omitEmptyOptionals:
                         options.update(ifNotEmpty=namedType.isOptional)
 
-                chunk = encodeFun(component, asn1Spec, **options)
-
                 # wrap open type blob if needed
-                if namedTypes and namedType.openType:
-                    wrapType = namedType.asn1Object
-                    if wrapType.tagSet and not wrapType.isSameTypeWith(component):
-                        chunk = encodeFun(chunk, wrapType, **options)
+                if (namedTypes and namedType.openType
+                        and namedType.asn1Object.tagSet):
 
-                        if LOG:
-                            LOG('wrapped open type with wrap type %r' % (wrapType,))
+                    if component.typeId in (
+                        univ.SetOf.typeId, univ.SequenceOf.typeId):
+                            substrate += encodeFun(
+                                component, asn1Spec,
+                                **dict(options, openType=True))
+
+                    else:
+                        chunk = encodeFun(component, asn1Spec, **options)
+
+                        wrapType = namedType.asn1Object
 
-                substrate += chunk
+                        if wrapType.isSameTypeWith(component):
+                            substrate += chunk
+
+                        else:
+                            substrate += encodeFun(chunk, wrapType, **options)
+
+                            if LOG:
+                                LOG('wrapped with wrap type %r' % (wrapType,))
+
+                else:
+                    substrate += encodeFun(component, asn1Spec, **options)
 
         else:
             # bare Python value + ASN.1 schema
@@ -593,18 +609,31 @@ class SequenceEncoder(AbstractItemEncoder):
                 if omitEmptyOptionals:
                     options.update(ifNotEmpty=namedType.isOptional)
 
-                chunk = encodeFun(component, asn1Spec[idx], **options)
-
                 # wrap open type blob if needed
-                if namedType.openType:
-                    wrapType = namedType.asn1Object
-                    if wrapType.tagSet and not wrapType.isSameTypeWith(component):
-                        chunk = encodeFun(chunk, wrapType, **options)
+                if namedType.openType and namedType.asn1Object.tagSet:
 
-                        if LOG:
-                            LOG('wrapped open type with wrap type %r' % (wrapType,))
+                    if component.typeId in (
+                        univ.SetOf.typeId, univ.SequenceOf.typeId):
+                            substrate += encodeFun(
+                                component, asn1Spec[idx],
+                                **dict(options, openType=True))
+
+                    else:
+                        chunk = encodeFun(component, asn1Spec[idx], **options)
+
+                        wrapType = namedType.asn1Object
 
-                substrate += chunk
+                        if wrapType.isSameTypeWith(component):
+                            substrate += chunk
+
+                        else:
+                            substrate += encodeFun(chunk, wrapType, **options)
+
+                            if LOG:
+                                LOG('wrapped with wrap type %r' % (wrapType,))
+
+                else:
+                    substrate += encodeFun(component, asn1Spec[idx], **options)
 
         return substrate, True, True
 
@@ -613,13 +642,32 @@ class SequenceOfEncoder(AbstractItemEncoder):
     def encodeValue(self, value, asn1Spec, encodeFun, **options):
         if asn1Spec is None:
             value.verifySizeSpec()
+
+            wrapType = value.componentType
+
         else:
-            asn1Spec = asn1Spec.componentType
+            asn1Spec = wrapType = asn1Spec.componentType
+
+        openType = options.pop('openType', False)
 
         substrate = null
 
         for idx, component in enumerate(value):
-            substrate += encodeFun(value[idx], asn1Spec, **options)
+            if openType:
+                # do not use asn1Spec even if given because it's a wrapper
+                chunk = encodeFun(component, **options)
+
+                if not wrapType.isSameTypeWith(component):
+                    # wrap encoded value with wrapper container (e.g. ANY)
+                    chunk = encodeFun(chunk, wrapType, **options)
+
+                    if LOG:
+                        LOG('wrapped with wrap type %r' % (wrapType,))
+
+            else:
+                chunk = encodeFun(component, asn1Spec, **options)
+
+            substrate += chunk
 
         return substrate, True, True
 
diff --git a/pyasn1/type/univ.py b/pyasn1/type/univ.py
index 96623e7..5634f94 100644
--- a/pyasn1/type/univ.py
+++ b/pyasn1/type/univ.py
@@ -1854,13 +1854,19 @@ class SequenceOfAndSetOfBase(base.AbstractConstructedAsn1Item):
                     'Non-ASN.1 value %r and undefined component'
                     ' type at %r' % (value, self))
 
-        elif componentType is not None:
-            if self.strictConstraints:
-                if not componentType.isSameTypeWith(
-                        value, matchTags, matchConstraints):
+        elif componentType is not None and (matchTags or matchConstraints):
+            subtypeChecker = (
+                    self.strictConstraints and
+                    componentType.isSameTypeWith or
+                    componentType.isSuperTypeOf)
+
+            if not subtypeChecker(value, matchTags, matchConstraints):
+                # TODO: we should wrap componentType with UnnamedType to carry
+                # additional properties associated with componentType
+                if componentType.typeId != Any.typeId:
                     raise error.PyAsn1Error(
-                        'Component value is tag-incompatible: %r '
-                        'vs %r' % (value, componentType))
+                        'Component value is tag-incompatible: %r vs '
+                        '%r' % (value, componentType))
 
             else:
                 if not componentType.isSuperTypeOf(