diff options
Diffstat (limited to 'libgo/go/encoding')
32 files changed, 2148 insertions, 628 deletions
diff --git a/libgo/go/encoding/asn1/asn1.go b/libgo/go/encoding/asn1/asn1.go index 2b5ad08551e..044f74ab469 100644 --- a/libgo/go/encoding/asn1/asn1.go +++ b/libgo/go/encoding/asn1/asn1.go @@ -841,6 +841,13 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam case reflect.Struct: structType := fieldType + for i := 0; i < structType.NumField(); i++ { + if structType.Field(i).PkgPath != "" { + err = StructuralError{"struct contains unexported fields"} + return + } + } + if structType.NumField() > 0 && structType.Field(0).Type == rawContentsType { bytes := bytes[initOffset:offset] @@ -969,7 +976,7 @@ func setDefaultValue(v reflect.Value, params fieldParameters) (ok bool) { // The following tags on struct fields have special meaning to Unmarshal: // // application specifies that a APPLICATION tag is used -// default:x sets the default value for optional integer fields +// default:x sets the default value for optional integer fields (only used if optional is also present) // explicit specifies that an additional, explicit tag wraps the implicit one // optional marks the field as ASN.1 OPTIONAL // set causes a SET, rather than a SEQUENCE type to be expected diff --git a/libgo/go/encoding/asn1/asn1_test.go b/libgo/go/encoding/asn1/asn1_test.go index f8623fa9a21..9976656df89 100644 --- a/libgo/go/encoding/asn1/asn1_test.go +++ b/libgo/go/encoding/asn1/asn1_test.go @@ -132,9 +132,13 @@ func TestParseBigInt(t *testing.T) { if ret.String() != test.base10 { t.Errorf("#%d: bad result from %x, got %s want %s", i, test.in, ret.String(), test.base10) } - fw := newForkableWriter() - marshalBigInt(fw, ret) - result := fw.Bytes() + e, err := makeBigInt(ret) + if err != nil { + t.Errorf("%d: err=%q", i, err) + continue + } + result := make([]byte, e.Len()) + e.Encode(result) if !bytes.Equal(result, test.in) { t.Errorf("#%d: got %x from marshaling %s, want %x", i, result, ret, test.in) } @@ -963,7 +967,7 @@ func TestUnmarshalInvalidUTF8(t *testing.T) { func TestMarshalNilValue(t *testing.T) { nilValueTestData := []interface{}{ nil, - struct{ v interface{} }{}, + struct{ V interface{} }{}, } for i, test := range nilValueTestData { if _, err := Marshal(test); err == nil { @@ -971,3 +975,32 @@ func TestMarshalNilValue(t *testing.T) { } } } + +type unexported struct { + X int + y int +} + +type exported struct { + X int + Y int +} + +func TestUnexportedStructField(t *testing.T) { + want := StructuralError{"struct contains unexported fields"} + + _, err := Marshal(unexported{X: 5, y: 1}) + if err != want { + t.Errorf("got %v, want %v", err, want) + } + + bs, err := Marshal(exported{X: 5, Y: 1}) + if err != nil { + t.Fatal(err) + } + var u unexported + _, err = Unmarshal(bs, &u) + if err != want { + t.Errorf("got %v, want %v", err, want) + } +} diff --git a/libgo/go/encoding/asn1/marshal.go b/libgo/go/encoding/asn1/marshal.go index 30797ef0996..225fd0849c6 100644 --- a/libgo/go/encoding/asn1/marshal.go +++ b/libgo/go/encoding/asn1/marshal.go @@ -5,77 +5,125 @@ package asn1 import ( - "bytes" "errors" "fmt" - "io" "math/big" "reflect" "time" "unicode/utf8" ) -// A forkableWriter is an in-memory buffer that can be -// 'forked' to create new forkableWriters that bracket the -// original. After -// pre, post := w.fork() -// the overall sequence of bytes represented is logically w+pre+post. -type forkableWriter struct { - *bytes.Buffer - pre, post *forkableWriter +var ( + byte00Encoder encoder = byteEncoder(0x00) + byteFFEncoder encoder = byteEncoder(0xff) +) + +// encoder represents a ASN.1 element that is waiting to be marshaled. +type encoder interface { + // Len returns the number of bytes needed to marshal this element. + Len() int + // Encode encodes this element by writing Len() bytes to dst. + Encode(dst []byte) +} + +type byteEncoder byte + +func (c byteEncoder) Len() int { + return 1 } -func newForkableWriter() *forkableWriter { - return &forkableWriter{new(bytes.Buffer), nil, nil} +func (c byteEncoder) Encode(dst []byte) { + dst[0] = byte(c) } -func (f *forkableWriter) fork() (pre, post *forkableWriter) { - if f.pre != nil || f.post != nil { - panic("have already forked") +type bytesEncoder []byte + +func (b bytesEncoder) Len() int { + return len(b) +} + +func (b bytesEncoder) Encode(dst []byte) { + if copy(dst, b) != len(b) { + panic("internal error") } - f.pre = newForkableWriter() - f.post = newForkableWriter() - return f.pre, f.post } -func (f *forkableWriter) Len() (l int) { - l += f.Buffer.Len() - if f.pre != nil { - l += f.pre.Len() +type stringEncoder string + +func (s stringEncoder) Len() int { + return len(s) +} + +func (s stringEncoder) Encode(dst []byte) { + if copy(dst, s) != len(s) { + panic("internal error") } - if f.post != nil { - l += f.post.Len() +} + +type multiEncoder []encoder + +func (m multiEncoder) Len() int { + var size int + for _, e := range m { + size += e.Len() } - return + return size } -func (f *forkableWriter) writeTo(out io.Writer) (n int, err error) { - n, err = out.Write(f.Bytes()) - if err != nil { - return +func (m multiEncoder) Encode(dst []byte) { + var off int + for _, e := range m { + e.Encode(dst[off:]) + off += e.Len() } +} - var nn int +type taggedEncoder struct { + // scratch contains temporary space for encoding the tag and length of + // an element in order to avoid extra allocations. + scratch [8]byte + tag encoder + body encoder +} - if f.pre != nil { - nn, err = f.pre.writeTo(out) - n += nn - if err != nil { - return - } +func (t *taggedEncoder) Len() int { + return t.tag.Len() + t.body.Len() +} + +func (t *taggedEncoder) Encode(dst []byte) { + t.tag.Encode(dst) + t.body.Encode(dst[t.tag.Len():]) +} + +type int64Encoder int64 + +func (i int64Encoder) Len() int { + n := 1 + + for i > 127 { + n++ + i >>= 8 } - if f.post != nil { - nn, err = f.post.writeTo(out) - n += nn + for i < -128 { + n++ + i >>= 8 } - return + + return n } -func marshalBase128Int(out *forkableWriter, n int64) (err error) { +func (i int64Encoder) Encode(dst []byte) { + n := i.Len() + + for j := 0; j < n; j++ { + dst[j] = byte(i >> uint((n-1-j)*8)) + } +} + +func base128IntLength(n int64) int { if n == 0 { - err = out.WriteByte(0) - return + return 1 } l := 0 @@ -83,54 +131,33 @@ func marshalBase128Int(out *forkableWriter, n int64) (err error) { l++ } + return l +} + +func appendBase128Int(dst []byte, n int64) []byte { + l := base128IntLength(n) + for i := l - 1; i >= 0; i-- { o := byte(n >> uint(i*7)) o &= 0x7f if i != 0 { o |= 0x80 } - err = out.WriteByte(o) - if err != nil { - return - } - } - - return nil -} -func marshalInt64(out *forkableWriter, i int64) (err error) { - n := int64Length(i) - - for ; n > 0; n-- { - err = out.WriteByte(byte(i >> uint((n-1)*8))) - if err != nil { - return - } + dst = append(dst, o) } - return nil + return dst } -func int64Length(i int64) (numBytes int) { - numBytes = 1 - - for i > 127 { - numBytes++ - i >>= 8 +func makeBigInt(n *big.Int) (encoder, error) { + if n == nil { + return nil, StructuralError{"empty integer"} } - for i < -128 { - numBytes++ - i >>= 8 - } - - return -} - -func marshalBigInt(out *forkableWriter, n *big.Int) (err error) { if n.Sign() < 0 { // A negative number has to be converted to two's-complement - // form. So we'll subtract 1 and invert. If the + // form. So we'll invert and subtract 1. If the // most-significant-bit isn't set then we'll need to pad the // beginning with 0xff in order to keep the number negative. nMinus1 := new(big.Int).Neg(n) @@ -140,41 +167,31 @@ func marshalBigInt(out *forkableWriter, n *big.Int) (err error) { bytes[i] ^= 0xff } if len(bytes) == 0 || bytes[0]&0x80 == 0 { - err = out.WriteByte(0xff) - if err != nil { - return - } + return multiEncoder([]encoder{byteFFEncoder, bytesEncoder(bytes)}), nil } - _, err = out.Write(bytes) + return bytesEncoder(bytes), nil } else if n.Sign() == 0 { // Zero is written as a single 0 zero rather than no bytes. - err = out.WriteByte(0x00) + return byte00Encoder, nil } else { bytes := n.Bytes() if len(bytes) > 0 && bytes[0]&0x80 != 0 { // We'll have to pad this with 0x00 in order to stop it // looking like a negative number. - err = out.WriteByte(0) - if err != nil { - return - } + return multiEncoder([]encoder{byte00Encoder, bytesEncoder(bytes)}), nil } - _, err = out.Write(bytes) + return bytesEncoder(bytes), nil } - return } -func marshalLength(out *forkableWriter, i int) (err error) { +func appendLength(dst []byte, i int) []byte { n := lengthLength(i) for ; n > 0; n-- { - err = out.WriteByte(byte(i >> uint((n-1)*8))) - if err != nil { - return - } + dst = append(dst, byte(i>>uint((n-1)*8))) } - return nil + return dst } func lengthLength(i int) (numBytes int) { @@ -186,123 +203,104 @@ func lengthLength(i int) (numBytes int) { return } -func marshalTagAndLength(out *forkableWriter, t tagAndLength) (err error) { +func appendTagAndLength(dst []byte, t tagAndLength) []byte { b := uint8(t.class) << 6 if t.isCompound { b |= 0x20 } if t.tag >= 31 { b |= 0x1f - err = out.WriteByte(b) - if err != nil { - return - } - err = marshalBase128Int(out, int64(t.tag)) - if err != nil { - return - } + dst = append(dst, b) + dst = appendBase128Int(dst, int64(t.tag)) } else { b |= uint8(t.tag) - err = out.WriteByte(b) - if err != nil { - return - } + dst = append(dst, b) } if t.length >= 128 { l := lengthLength(t.length) - err = out.WriteByte(0x80 | byte(l)) - if err != nil { - return - } - err = marshalLength(out, t.length) - if err != nil { - return - } + dst = append(dst, 0x80|byte(l)) + dst = appendLength(dst, t.length) } else { - err = out.WriteByte(byte(t.length)) - if err != nil { - return - } + dst = append(dst, byte(t.length)) } - return nil + return dst } -func marshalBitString(out *forkableWriter, b BitString) (err error) { - paddingBits := byte((8 - b.BitLength%8) % 8) - err = out.WriteByte(paddingBits) - if err != nil { - return - } - _, err = out.Write(b.Bytes) - return +type bitStringEncoder BitString + +func (b bitStringEncoder) Len() int { + return len(b.Bytes) + 1 } -func marshalObjectIdentifier(out *forkableWriter, oid []int) (err error) { - if len(oid) < 2 || oid[0] > 2 || (oid[0] < 2 && oid[1] >= 40) { - return StructuralError{"invalid object identifier"} +func (b bitStringEncoder) Encode(dst []byte) { + dst[0] = byte((8 - b.BitLength%8) % 8) + if copy(dst[1:], b.Bytes) != len(b.Bytes) { + panic("internal error") } +} - err = marshalBase128Int(out, int64(oid[0]*40+oid[1])) - if err != nil { - return +type oidEncoder []int + +func (oid oidEncoder) Len() int { + l := base128IntLength(int64(oid[0]*40 + oid[1])) + for i := 2; i < len(oid); i++ { + l += base128IntLength(int64(oid[i])) } + return l +} + +func (oid oidEncoder) Encode(dst []byte) { + dst = appendBase128Int(dst[:0], int64(oid[0]*40+oid[1])) for i := 2; i < len(oid); i++ { - err = marshalBase128Int(out, int64(oid[i])) - if err != nil { - return - } + dst = appendBase128Int(dst, int64(oid[i])) } +} - return +func makeObjectIdentifier(oid []int) (e encoder, err error) { + if len(oid) < 2 || oid[0] > 2 || (oid[0] < 2 && oid[1] >= 40) { + return nil, StructuralError{"invalid object identifier"} + } + + return oidEncoder(oid), nil } -func marshalPrintableString(out *forkableWriter, s string) (err error) { - b := []byte(s) - for _, c := range b { - if !isPrintable(c) { - return StructuralError{"PrintableString contains invalid character"} +func makePrintableString(s string) (e encoder, err error) { + for i := 0; i < len(s); i++ { + if !isPrintable(s[i]) { + return nil, StructuralError{"PrintableString contains invalid character"} } } - _, err = out.Write(b) - return + return stringEncoder(s), nil } -func marshalIA5String(out *forkableWriter, s string) (err error) { - b := []byte(s) - for _, c := range b { - if c > 127 { - return StructuralError{"IA5String contains invalid character"} +func makeIA5String(s string) (e encoder, err error) { + for i := 0; i < len(s); i++ { + if s[i] > 127 { + return nil, StructuralError{"IA5String contains invalid character"} } } - _, err = out.Write(b) - return + return stringEncoder(s), nil } -func marshalUTF8String(out *forkableWriter, s string) (err error) { - _, err = out.Write([]byte(s)) - return +func makeUTF8String(s string) encoder { + return stringEncoder(s) } -func marshalTwoDigits(out *forkableWriter, v int) (err error) { - err = out.WriteByte(byte('0' + (v/10)%10)) - if err != nil { - return - } - return out.WriteByte(byte('0' + v%10)) +func appendTwoDigits(dst []byte, v int) []byte { + return append(dst, byte('0'+(v/10)%10), byte('0'+v%10)) } -func marshalFourDigits(out *forkableWriter, v int) (err error) { +func appendFourDigits(dst []byte, v int) []byte { var bytes [4]byte for i := range bytes { bytes[3-i] = '0' + byte(v%10) v /= 10 } - _, err = out.Write(bytes[:]) - return + return append(dst, bytes[:]...) } func outsideUTCRange(t time.Time) bool { @@ -310,80 +308,75 @@ func outsideUTCRange(t time.Time) bool { return year < 1950 || year >= 2050 } -func marshalUTCTime(out *forkableWriter, t time.Time) (err error) { +func makeUTCTime(t time.Time) (e encoder, err error) { + dst := make([]byte, 0, 18) + + dst, err = appendUTCTime(dst, t) + if err != nil { + return nil, err + } + + return bytesEncoder(dst), nil +} + +func makeGeneralizedTime(t time.Time) (e encoder, err error) { + dst := make([]byte, 0, 20) + + dst, err = appendGeneralizedTime(dst, t) + if err != nil { + return nil, err + } + + return bytesEncoder(dst), nil +} + +func appendUTCTime(dst []byte, t time.Time) (ret []byte, err error) { year := t.Year() switch { case 1950 <= year && year < 2000: - err = marshalTwoDigits(out, year-1900) + dst = appendTwoDigits(dst, year-1900) case 2000 <= year && year < 2050: - err = marshalTwoDigits(out, year-2000) + dst = appendTwoDigits(dst, year-2000) default: - return StructuralError{"cannot represent time as UTCTime"} - } - if err != nil { - return + return nil, StructuralError{"cannot represent time as UTCTime"} } - return marshalTimeCommon(out, t) + return appendTimeCommon(dst, t), nil } -func marshalGeneralizedTime(out *forkableWriter, t time.Time) (err error) { +func appendGeneralizedTime(dst []byte, t time.Time) (ret []byte, err error) { year := t.Year() if year < 0 || year > 9999 { - return StructuralError{"cannot represent time as GeneralizedTime"} - } - if err = marshalFourDigits(out, year); err != nil { - return + return nil, StructuralError{"cannot represent time as GeneralizedTime"} } - return marshalTimeCommon(out, t) + dst = appendFourDigits(dst, year) + + return appendTimeCommon(dst, t), nil } -func marshalTimeCommon(out *forkableWriter, t time.Time) (err error) { +func appendTimeCommon(dst []byte, t time.Time) []byte { _, month, day := t.Date() - err = marshalTwoDigits(out, int(month)) - if err != nil { - return - } - - err = marshalTwoDigits(out, day) - if err != nil { - return - } + dst = appendTwoDigits(dst, int(month)) + dst = appendTwoDigits(dst, day) hour, min, sec := t.Clock() - err = marshalTwoDigits(out, hour) - if err != nil { - return - } - - err = marshalTwoDigits(out, min) - if err != nil { - return - } - - err = marshalTwoDigits(out, sec) - if err != nil { - return - } + dst = appendTwoDigits(dst, hour) + dst = appendTwoDigits(dst, min) + dst = appendTwoDigits(dst, sec) _, offset := t.Zone() switch { case offset/60 == 0: - err = out.WriteByte('Z') - return + return append(dst, 'Z') case offset > 0: - err = out.WriteByte('+') + dst = append(dst, '+') case offset < 0: - err = out.WriteByte('-') - } - - if err != nil { - return + dst = append(dst, '-') } offsetMinutes := offset / 60 @@ -391,13 +384,10 @@ func marshalTimeCommon(out *forkableWriter, t time.Time) (err error) { offsetMinutes = -offsetMinutes } - err = marshalTwoDigits(out, offsetMinutes/60) - if err != nil { - return - } + dst = appendTwoDigits(dst, offsetMinutes/60) + dst = appendTwoDigits(dst, offsetMinutes%60) - err = marshalTwoDigits(out, offsetMinutes%60) - return + return dst } func stripTagAndLength(in []byte) []byte { @@ -408,114 +398,130 @@ func stripTagAndLength(in []byte) []byte { return in[offset:] } -func marshalBody(out *forkableWriter, value reflect.Value, params fieldParameters) (err error) { +func makeBody(value reflect.Value, params fieldParameters) (e encoder, err error) { switch value.Type() { case flagType: - return nil + return bytesEncoder(nil), nil case timeType: t := value.Interface().(time.Time) if params.timeType == TagGeneralizedTime || outsideUTCRange(t) { - return marshalGeneralizedTime(out, t) - } else { - return marshalUTCTime(out, t) + return makeGeneralizedTime(t) } + return makeUTCTime(t) case bitStringType: - return marshalBitString(out, value.Interface().(BitString)) + return bitStringEncoder(value.Interface().(BitString)), nil case objectIdentifierType: - return marshalObjectIdentifier(out, value.Interface().(ObjectIdentifier)) + return makeObjectIdentifier(value.Interface().(ObjectIdentifier)) case bigIntType: - return marshalBigInt(out, value.Interface().(*big.Int)) + return makeBigInt(value.Interface().(*big.Int)) } switch v := value; v.Kind() { case reflect.Bool: if v.Bool() { - return out.WriteByte(255) - } else { - return out.WriteByte(0) + return byteFFEncoder, nil } + return byte00Encoder, nil case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - return marshalInt64(out, v.Int()) + return int64Encoder(v.Int()), nil case reflect.Struct: t := v.Type() + for i := 0; i < t.NumField(); i++ { + if t.Field(i).PkgPath != "" { + return nil, StructuralError{"struct contains unexported fields"} + } + } + startingField := 0 + n := t.NumField() + if n == 0 { + return bytesEncoder(nil), nil + } + // If the first element of the structure is a non-empty // RawContents, then we don't bother serializing the rest. - if t.NumField() > 0 && t.Field(0).Type == rawContentsType { + if t.Field(0).Type == rawContentsType { s := v.Field(0) if s.Len() > 0 { - bytes := make([]byte, s.Len()) - for i := 0; i < s.Len(); i++ { - bytes[i] = uint8(s.Index(i).Uint()) - } + bytes := s.Bytes() /* The RawContents will contain the tag and * length fields but we'll also be writing * those ourselves, so we strip them out of * bytes */ - _, err = out.Write(stripTagAndLength(bytes)) - return - } else { - startingField = 1 + return bytesEncoder(stripTagAndLength(bytes)), nil } + + startingField = 1 } - for i := startingField; i < t.NumField(); i++ { - var pre *forkableWriter - pre, out = out.fork() - err = marshalField(pre, v.Field(i), parseFieldParameters(t.Field(i).Tag.Get("asn1"))) - if err != nil { - return + switch n1 := n - startingField; n1 { + case 0: + return bytesEncoder(nil), nil + case 1: + return makeField(v.Field(startingField), parseFieldParameters(t.Field(startingField).Tag.Get("asn1"))) + default: + m := make([]encoder, n1) + for i := 0; i < n1; i++ { + m[i], err = makeField(v.Field(i+startingField), parseFieldParameters(t.Field(i+startingField).Tag.Get("asn1"))) + if err != nil { + return nil, err + } } + + return multiEncoder(m), nil } - return case reflect.Slice: sliceType := v.Type() if sliceType.Elem().Kind() == reflect.Uint8 { - bytes := make([]byte, v.Len()) - for i := 0; i < v.Len(); i++ { - bytes[i] = uint8(v.Index(i).Uint()) - } - _, err = out.Write(bytes) - return + return bytesEncoder(v.Bytes()), nil } var fp fieldParameters - for i := 0; i < v.Len(); i++ { - var pre *forkableWriter - pre, out = out.fork() - err = marshalField(pre, v.Index(i), fp) - if err != nil { - return + + switch l := v.Len(); l { + case 0: + return bytesEncoder(nil), nil + case 1: + return makeField(v.Index(0), fp) + default: + m := make([]encoder, l) + + for i := 0; i < l; i++ { + m[i], err = makeField(v.Index(i), fp) + if err != nil { + return nil, err + } } + + return multiEncoder(m), nil } - return case reflect.String: switch params.stringType { case TagIA5String: - return marshalIA5String(out, v.String()) + return makeIA5String(v.String()) case TagPrintableString: - return marshalPrintableString(out, v.String()) + return makePrintableString(v.String()) default: - return marshalUTF8String(out, v.String()) + return makeUTF8String(v.String()), nil } } - return StructuralError{"unknown Go type"} + return nil, StructuralError{"unknown Go type"} } -func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) (err error) { +func makeField(v reflect.Value, params fieldParameters) (e encoder, err error) { if !v.IsValid() { - return fmt.Errorf("asn1: cannot marshal nil value") + return nil, fmt.Errorf("asn1: cannot marshal nil value") } // If the field is an interface{} then recurse into it. if v.Kind() == reflect.Interface && v.Type().NumMethod() == 0 { - return marshalField(out, v.Elem(), params) + return makeField(v.Elem(), params) } if v.Kind() == reflect.Slice && v.Len() == 0 && params.omitEmpty { - return + return bytesEncoder(nil), nil } if params.optional && params.defaultValue != nil && canHaveDefaultValue(v.Kind()) { @@ -523,46 +529,45 @@ func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) defaultValue.SetInt(*params.defaultValue) if reflect.DeepEqual(v.Interface(), defaultValue.Interface()) { - return + return bytesEncoder(nil), nil } } // If no default value is given then the zero value for the type is // assumed to be the default value. This isn't obviously the correct - // behaviour, but it's what Go has traditionally done. + // behavior, but it's what Go has traditionally done. if params.optional && params.defaultValue == nil { if reflect.DeepEqual(v.Interface(), reflect.Zero(v.Type()).Interface()) { - return + return bytesEncoder(nil), nil } } if v.Type() == rawValueType { rv := v.Interface().(RawValue) if len(rv.FullBytes) != 0 { - _, err = out.Write(rv.FullBytes) - } else { - err = marshalTagAndLength(out, tagAndLength{rv.Class, rv.Tag, len(rv.Bytes), rv.IsCompound}) - if err != nil { - return - } - _, err = out.Write(rv.Bytes) + return bytesEncoder(rv.FullBytes), nil } - return + + t := new(taggedEncoder) + + t.tag = bytesEncoder(appendTagAndLength(t.scratch[:0], tagAndLength{rv.Class, rv.Tag, len(rv.Bytes), rv.IsCompound})) + t.body = bytesEncoder(rv.Bytes) + + return t, nil } tag, isCompound, ok := getUniversalType(v.Type()) if !ok { - err = StructuralError{fmt.Sprintf("unknown Go type: %v", v.Type())} - return + return nil, StructuralError{fmt.Sprintf("unknown Go type: %v", v.Type())} } class := ClassUniversal if params.timeType != 0 && tag != TagUTCTime { - return StructuralError{"explicit time type given to non-time member"} + return nil, StructuralError{"explicit time type given to non-time member"} } if params.stringType != 0 && tag != TagPrintableString { - return StructuralError{"explicit string type given to non-string member"} + return nil, StructuralError{"explicit string type given to non-string member"} } switch tag { @@ -574,7 +579,7 @@ func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) for _, r := range v.String() { if r >= utf8.RuneSelf || !isPrintable(byte(r)) { if !utf8.ValidString(v.String()) { - return errors.New("asn1: string not valid UTF-8") + return nil, errors.New("asn1: string not valid UTF-8") } tag = TagUTF8String break @@ -591,46 +596,46 @@ func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) if params.set { if tag != TagSequence { - return StructuralError{"non sequence tagged as set"} + return nil, StructuralError{"non sequence tagged as set"} } tag = TagSet } - tags, body := out.fork() + t := new(taggedEncoder) - err = marshalBody(body, v, params) + t.body, err = makeBody(v, params) if err != nil { - return + return nil, err } - bodyLen := body.Len() + bodyLen := t.body.Len() - var explicitTag *forkableWriter if params.explicit { - explicitTag, tags = tags.fork() - } + t.tag = bytesEncoder(appendTagAndLength(t.scratch[:0], tagAndLength{class, tag, bodyLen, isCompound})) - if !params.explicit && params.tag != nil { - // implicit tag. - tag = *params.tag - class = ClassContextSpecific - } + tt := new(taggedEncoder) - err = marshalTagAndLength(tags, tagAndLength{class, tag, bodyLen, isCompound}) - if err != nil { - return - } + tt.body = t - if params.explicit { - err = marshalTagAndLength(explicitTag, tagAndLength{ + tt.tag = bytesEncoder(appendTagAndLength(tt.scratch[:0], tagAndLength{ class: ClassContextSpecific, tag: *params.tag, - length: bodyLen + tags.Len(), + length: bodyLen + t.tag.Len(), isCompound: true, - }) + })) + + return tt, nil + } + + if params.tag != nil { + // implicit tag. + tag = *params.tag + class = ClassContextSpecific } - return err + t.tag = bytesEncoder(appendTagAndLength(t.scratch[:0], tagAndLength{class, tag, bodyLen, isCompound})) + + return t, nil } // Marshal returns the ASN.1 encoding of val. @@ -643,13 +648,11 @@ func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) // printable: causes strings to be marshaled as ASN.1, PrintableString strings. // utf8: causes strings to be marshaled as ASN.1, UTF8 strings func Marshal(val interface{}) ([]byte, error) { - var out bytes.Buffer - v := reflect.ValueOf(val) - f := newForkableWriter() - err := marshalField(f, v, fieldParameters{}) + e, err := makeField(reflect.ValueOf(val), fieldParameters{}) if err != nil { return nil, err } - _, err = f.writeTo(&out) - return out.Bytes(), err + b := make([]byte, e.Len()) + e.Encode(b) + return b, nil } diff --git a/libgo/go/encoding/asn1/marshal_test.go b/libgo/go/encoding/asn1/marshal_test.go index cdca8aa3363..10db1aa575a 100644 --- a/libgo/go/encoding/asn1/marshal_test.go +++ b/libgo/go/encoding/asn1/marshal_test.go @@ -8,6 +8,7 @@ import ( "bytes" "encoding/hex" "math/big" + "strings" "testing" "time" ) @@ -167,9 +168,42 @@ func TestMarshal(t *testing.T) { } } +type marshalErrTest struct { + in interface{} + err string +} + +var marshalErrTests = []marshalErrTest{ + {bigIntStruct{nil}, "empty integer"}, +} + +func TestMarshalError(t *testing.T) { + for i, test := range marshalErrTests { + _, err := Marshal(test.in) + if err == nil { + t.Errorf("#%d should fail, but success", i) + continue + } + + if !strings.Contains(err.Error(), test.err) { + t.Errorf("#%d got: %v want %v", i, err, test.err) + } + } +} + func TestInvalidUTF8(t *testing.T) { _, err := Marshal(string([]byte{0xff, 0xff})) if err == nil { t.Errorf("invalid UTF8 string was accepted") } } + +func BenchmarkMarshal(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + for _, test := range marshalTests { + Marshal(test.in) + } + } +} diff --git a/libgo/go/encoding/base64/base64.go b/libgo/go/encoding/base64/base64.go index c2116d8a343..d2efad4518b 100644 --- a/libgo/go/encoding/base64/base64.go +++ b/libgo/go/encoding/base64/base64.go @@ -23,6 +23,7 @@ type Encoding struct { encode [64]byte decodeMap [256]byte padChar rune + strict bool } const ( @@ -62,6 +63,14 @@ func (enc Encoding) WithPadding(padding rune) *Encoding { return &enc } +// Strict creates a new encoding identical to enc except with +// strict decoding enabled. In this mode, the decoder requires that +// trailing padding bits are zero, as described in RFC 4648 section 3.5. +func (enc Encoding) Strict() *Encoding { + enc.strict = true + return &enc +} + // StdEncoding is the standard base64 encoding, as defined in // RFC 4648. var StdEncoding = NewEncoding(encodeStd) @@ -311,15 +320,24 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { // Convert 4x 6bit source bytes into 3 bytes val := uint(dbuf[0])<<18 | uint(dbuf[1])<<12 | uint(dbuf[2])<<6 | uint(dbuf[3]) + dbuf[2], dbuf[1], dbuf[0] = byte(val>>0), byte(val>>8), byte(val>>16) switch dlen { case 4: - dst[2] = byte(val >> 0) + dst[2] = dbuf[2] + dbuf[2] = 0 fallthrough case 3: - dst[1] = byte(val >> 8) + dst[1] = dbuf[1] + if enc.strict && dbuf[2] != 0 { + return n, end, CorruptInputError(si - 1) + } + dbuf[1] = 0 fallthrough case 2: - dst[0] = byte(val >> 16) + dst[0] = dbuf[0] + if enc.strict && (dbuf[1] != 0 || dbuf[2] != 0) { + return n, end, CorruptInputError(si - 2) + } } dst = dst[dinc:] n += dlen - 1 diff --git a/libgo/go/encoding/base64/base64_test.go b/libgo/go/encoding/base64/base64_test.go index 19ddb92f644..e2e1d59f3c0 100644 --- a/libgo/go/encoding/base64/base64_test.go +++ b/libgo/go/encoding/base64/base64_test.go @@ -85,6 +85,11 @@ var encodingTests = []encodingTest{ {RawStdEncoding, rawRef}, {RawURLEncoding, rawUrlRef}, {funnyEncoding, funnyRef}, + {StdEncoding.Strict(), stdRef}, + {URLEncoding.Strict(), urlRef}, + {RawStdEncoding.Strict(), rawRef}, + {RawURLEncoding.Strict(), rawUrlRef}, + {funnyEncoding.Strict(), funnyRef}, } var bigtest = testpair{ @@ -436,6 +441,22 @@ func TestDecoderIssue7733(t *testing.T) { } } +func TestDecoderIssue15656(t *testing.T) { + _, err := StdEncoding.Strict().DecodeString("WvLTlMrX9NpYDQlEIFlnDB==") + want := CorruptInputError(22) + if !reflect.DeepEqual(want, err) { + t.Errorf("Error = %v; want CorruptInputError(22)", err) + } + _, err = StdEncoding.Strict().DecodeString("WvLTlMrX9NpYDQlEIFlnDA==") + if err != nil { + t.Errorf("Error = %v; want nil", err) + } + _, err = StdEncoding.DecodeString("WvLTlMrX9NpYDQlEIFlnDB==") + if err != nil { + t.Errorf("Error = %v; want nil", err) + } +} + func BenchmarkEncodeToString(b *testing.B) { data := make([]byte, 8192) b.SetBytes(int64(len(data))) diff --git a/libgo/go/encoding/binary/binary.go b/libgo/go/encoding/binary/binary.go index 46c6add062a..38342545966 100644 --- a/libgo/go/encoding/binary/binary.go +++ b/libgo/go/encoding/binary/binary.go @@ -7,7 +7,7 @@ // // Numbers are translated by reading and writing fixed-size values. // A fixed-size value is either a fixed-size arithmetic -// type (int8, uint8, int16, float32, complex64, ...) +// type (bool, int8, uint8, int16, float32, complex64, ...) // or an array or struct containing only fixed-size values. // // The varint functions encode and decode single integer values using @@ -147,6 +147,8 @@ func (bigEndian) GoString() string { return "binary.BigEndian" } // of fixed-size values. // Bytes read from r are decoded using the specified byte order // and written to successive fields of the data. +// When decoding boolean values, a zero byte is decoded as false, and +// any other non-zero byte is decoded as true. // When reading into structs, the field data for fields with // blank (_) field names is skipped; i.e., blank field names // may be used for padding. @@ -169,6 +171,8 @@ func Read(r io.Reader, order ByteOrder, data interface{}) error { return err } switch data := data.(type) { + case *bool: + *data = b[0] != 0 case *int8: *data = int8(b[0]) case *uint8: @@ -185,8 +189,12 @@ func Read(r io.Reader, order ByteOrder, data interface{}) error { *data = int64(order.Uint64(bs)) case *uint64: *data = order.Uint64(bs) - case []int8: + case []bool: for i, x := range bs { // Easier to loop over the input for 8-bit values. + data[i] = x != 0 + } + case []int8: + for i, x := range bs { data[i] = int8(x) } case []uint8: @@ -243,6 +251,7 @@ func Read(r io.Reader, order ByteOrder, data interface{}) error { // Write writes the binary representation of data into w. // Data must be a fixed-size value or a slice of fixed-size // values, or a pointer to such data. +// Boolean values encode as one byte: 1 for true, and 0 for false. // Bytes written to w are encoded using the specified byte order // and read from successive fields of the data. // When writing structs, zero values are written for fields @@ -258,6 +267,26 @@ func Write(w io.Writer, order ByteOrder, data interface{}) error { bs = b[:n] } switch v := data.(type) { + case *bool: + if *v { + b[0] = 1 + } else { + b[0] = 0 + } + case bool: + if v { + b[0] = 1 + } else { + b[0] = 0 + } + case []bool: + for i, x := range v { + if x { + bs[i] = 1 + } else { + bs[i] = 0 + } + } case *int8: b[0] = byte(*v) case int8: @@ -378,7 +407,8 @@ func sizeof(t reflect.Type) int { } return sum - case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + case reflect.Bool, + reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128: return int(t.Size()) @@ -395,6 +425,21 @@ type coder struct { type decoder coder type encoder coder +func (d *decoder) bool() bool { + x := d.buf[0] + d.buf = d.buf[1:] + return x != 0 +} + +func (e *encoder) bool(x bool) { + if x { + e.buf[0] = 1 + } else { + e.buf[0] = 0 + } + e.buf = e.buf[1:] +} + func (d *decoder) uint8() uint8 { x := d.buf[0] d.buf = d.buf[1:] @@ -485,6 +530,9 @@ func (d *decoder) value(v reflect.Value) { d.value(v.Index(i)) } + case reflect.Bool: + v.SetBool(d.bool()) + case reflect.Int8: v.SetInt(int64(d.int8())) case reflect.Int16: @@ -547,6 +595,9 @@ func (e *encoder) value(v reflect.Value) { e.value(v.Index(i)) } + case reflect.Bool: + e.bool(v.Bool()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: switch v.Type().Kind() { case reflect.Int8: @@ -609,7 +660,7 @@ func (e *encoder) skip(v reflect.Value) { // It returns zero if the type cannot be implemented by the fast path in Read or Write. func intDataSize(data interface{}) int { switch data := data.(type) { - case int8, uint8, *int8, *uint8: + case bool, int8, uint8, *bool, *int8, *uint8: return 1 case []int8: return len(data) diff --git a/libgo/go/encoding/binary/binary_test.go b/libgo/go/encoding/binary/binary_test.go index c0728e943e8..fc7f2765efb 100644 --- a/libgo/go/encoding/binary/binary_test.go +++ b/libgo/go/encoding/binary/binary_test.go @@ -27,6 +27,8 @@ type Struct struct { Complex64 complex64 Complex128 complex128 Array [4]uint8 + Bool bool + BoolArray [4]bool } type T struct { @@ -58,6 +60,9 @@ var s = Struct{ ), [4]uint8{0x43, 0x44, 0x45, 0x46}, + + true, + [4]bool{true, false, true, false}, } var big = []byte{ @@ -76,6 +81,9 @@ var big = []byte{ 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, + + 1, + 1, 0, 1, 0, } var little = []byte{ @@ -94,6 +102,9 @@ var little = []byte{ 58, 57, 56, 55, 54, 53, 52, 51, 66, 65, 64, 63, 62, 61, 60, 59, 67, 68, 69, 70, + + 1, + 1, 0, 1, 0, } var src = []byte{1, 2, 3, 4, 5, 6, 7, 8} @@ -141,6 +152,25 @@ func TestWriteSlice(t *testing.T) { checkResult(t, "WriteSlice", BigEndian, err, buf.Bytes(), src) } +func TestReadBool(t *testing.T) { + var res bool + var err error + err = Read(bytes.NewReader([]byte{0}), BigEndian, &res) + checkResult(t, "ReadBool", BigEndian, err, res, false) + res = false + err = Read(bytes.NewReader([]byte{1}), BigEndian, &res) + checkResult(t, "ReadBool", BigEndian, err, res, true) + res = false + err = Read(bytes.NewReader([]byte{2}), BigEndian, &res) + checkResult(t, "ReadBool", BigEndian, err, res, true) +} + +func TestReadBoolSlice(t *testing.T) { + slice := make([]bool, 4) + err := Read(bytes.NewReader([]byte{0, 1, 2, 255}), BigEndian, slice) + checkResult(t, "ReadBoolSlice", BigEndian, err, slice, []bool{false, true, true, true}) +} + // Addresses of arrays are easier to manipulate with reflection than are slices. var intArrays = []interface{}{ &[100]int8{}, @@ -422,16 +452,15 @@ func BenchmarkReadInts(b *testing.B) { Read(r, BigEndian, &ls.Uint32) Read(r, BigEndian, &ls.Uint64) } - + b.StopTimer() want := s want.Float32 = 0 want.Float64 = 0 want.Complex64 = 0 want.Complex128 = 0 - for i := range want.Array { - want.Array[i] = 0 - } - b.StopTimer() + want.Array = [4]uint8{0, 0, 0, 0} + want.Bool = false + want.BoolArray = [4]bool{false, false, false, false} if b.N > 0 && !reflect.DeepEqual(ls, want) { b.Fatalf("struct doesn't match:\ngot %v;\nwant %v", ls, want) } diff --git a/libgo/go/encoding/csv/reader.go b/libgo/go/encoding/csv/reader.go index a5e03a9f8ef..c8c4ca77583 100644 --- a/libgo/go/encoding/csv/reader.go +++ b/libgo/go/encoding/csv/reader.go @@ -114,7 +114,14 @@ type Reader struct { line int column int r *bufio.Reader - field bytes.Buffer + // lineBuffer holds the unescaped fields read by readField, one after another. + // The fields can be accessed by using the indexes in fieldIndexes. + // Example: for the row `a,"b","c""d",e` lineBuffer will contain `abc"de` and + // fieldIndexes will contain the indexes 0, 1, 2, 5. + lineBuffer bytes.Buffer + // Indexes of fields inside lineBuffer + // The i'th field starts at offset fieldIndexes[i] in lineBuffer. + fieldIndexes []int } // NewReader returns a new Reader that reads from r. @@ -134,8 +141,12 @@ func (r *Reader) error(err error) error { } } -// Read reads one record from r. The record is a slice of strings with each -// string representing one field. +// Read reads one record (a slice of fields) from r. +// If the record has an unexpected number of fields, +// Read returns the record along with the error ErrFieldCount. +// Except for that case, Read always returns either a non-nil +// record or a non-nil error, but not both. +// If there is no data left to be read, Read returns nil, io.EOF. func (r *Reader) Read() (record []string, err error) { for { record, err = r.parseRecord() @@ -233,31 +244,54 @@ func (r *Reader) parseRecord() (fields []string, err error) { } r.r.UnreadRune() + r.lineBuffer.Reset() + r.fieldIndexes = r.fieldIndexes[:0] + // At this point we have at least one field. for { + idx := r.lineBuffer.Len() + haveField, delim, err := r.parseField() if haveField { - // If FieldsPerRecord is greater than 0 we can assume the final - // length of fields to be equal to FieldsPerRecord. - if r.FieldsPerRecord > 0 && fields == nil { - fields = make([]string, 0, r.FieldsPerRecord) - } - fields = append(fields, r.field.String()) + r.fieldIndexes = append(r.fieldIndexes, idx) } + if delim == '\n' || err == io.EOF { - return fields, err - } else if err != nil { + if len(r.fieldIndexes) == 0 { + return nil, err + } + break + } + + if err != nil { return nil, err } } + + fieldCount := len(r.fieldIndexes) + // Using this approach (creating a single string and taking slices of it) + // means that a single reference to any of the fields will retain the whole + // string. The risk of a nontrivial space leak caused by this is considered + // minimal and a tradeoff for better performance through the combined + // allocations. + line := r.lineBuffer.String() + fields = make([]string, fieldCount) + + for i, idx := range r.fieldIndexes { + if i == fieldCount-1 { + fields[i] = line[idx:] + } else { + fields[i] = line[idx:r.fieldIndexes[i+1]] + } + } + + return fields, nil } // parseField parses the next field in the record. The read field is -// located in r.field. Delim is the first character not part of the field +// appended to r.lineBuffer. Delim is the first character not part of the field // (r.Comma or '\n'). func (r *Reader) parseField() (haveField bool, delim rune, err error) { - r.field.Reset() - r1, err := r.readRune() for err == nil && r.TrimLeadingSpace && r1 != '\n' && unicode.IsSpace(r1) { r1, err = r.readRune() @@ -310,19 +344,19 @@ func (r *Reader) parseField() (haveField bool, delim rune, err error) { return false, 0, r.error(ErrQuote) } // accept the bare quote - r.field.WriteRune('"') + r.lineBuffer.WriteRune('"') } case '\n': r.line++ r.column = -1 } - r.field.WriteRune(r1) + r.lineBuffer.WriteRune(r1) } default: // unquoted field for { - r.field.WriteRune(r1) + r.lineBuffer.WriteRune(r1) r1, err = r.readRune() if err != nil || r1 == r.Comma { break diff --git a/libgo/go/encoding/csv/reader_test.go b/libgo/go/encoding/csv/reader_test.go index be1002d034a..7b3aca4c5f3 100644 --- a/libgo/go/encoding/csv/reader_test.go +++ b/libgo/go/encoding/csv/reader_test.go @@ -5,6 +5,7 @@ package csv import ( + "io" "reflect" "strings" "testing" @@ -292,8 +293,52 @@ func TestRead(t *testing.T) { } } -func BenchmarkRead(b *testing.B) { - data := `x,y,z,w +// nTimes is an io.Reader which yields the string s n times. +type nTimes struct { + s string + n int + off int +} + +func (r *nTimes) Read(p []byte) (n int, err error) { + for { + if r.n <= 0 || r.s == "" { + return n, io.EOF + } + n0 := copy(p, r.s[r.off:]) + p = p[n0:] + n += n0 + r.off += n0 + if r.off == len(r.s) { + r.off = 0 + r.n-- + } + if len(p) == 0 { + return + } + } +} + +// benchmarkRead measures reading the provided CSV rows data. +// initReader, if non-nil, modifies the Reader before it's used. +func benchmarkRead(b *testing.B, initReader func(*Reader), rows string) { + b.ReportAllocs() + r := NewReader(&nTimes{s: rows, n: b.N}) + if initReader != nil { + initReader(r) + } + for { + _, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + b.Fatal(err) + } + } +} + +const benchmarkCSVData = `x,y,z,w x,y,z, x,y,, x,,, @@ -305,11 +350,22 @@ x,,, "","","","" ` - for i := 0; i < b.N; i++ { - _, err := NewReader(strings.NewReader(data)).ReadAll() +func BenchmarkRead(b *testing.B) { + benchmarkRead(b, nil, benchmarkCSVData) +} - if err != nil { - b.Fatalf("could not read data: %s", err) - } - } +func BenchmarkReadWithFieldsPerRecord(b *testing.B) { + benchmarkRead(b, func(r *Reader) { r.FieldsPerRecord = 4 }, benchmarkCSVData) +} + +func BenchmarkReadWithoutFieldsPerRecord(b *testing.B) { + benchmarkRead(b, func(r *Reader) { r.FieldsPerRecord = -1 }, benchmarkCSVData) +} + +func BenchmarkReadLargeFields(b *testing.B) { + benchmarkRead(b, nil, strings.Repeat(`xxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv +xxxxxxxxxxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvv +,,zzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv +`, 3)) } diff --git a/libgo/go/encoding/gob/encoder.go b/libgo/go/encoding/gob/encoder.go index d6c8fdd9630..40ec81b6e69 100644 --- a/libgo/go/encoding/gob/encoder.go +++ b/libgo/go/encoding/gob/encoder.go @@ -215,6 +215,9 @@ func (enc *Encoder) sendTypeId(state *encoderState, ut *userTypeInfo) { // guaranteeing that all necessary type information has been transmitted first. // Passing a nil pointer to EncodeValue will panic, as they cannot be transmitted by gob. func (enc *Encoder) EncodeValue(value reflect.Value) error { + if value.Kind() == reflect.Invalid { + return errors.New("gob: cannot encode nil value") + } if value.Kind() == reflect.Ptr && value.IsNil() { panic("gob: cannot encode nil pointer of type " + value.Type().String()) } diff --git a/libgo/go/encoding/gob/encoder_test.go b/libgo/go/encoding/gob/encoder_test.go index 22090a18a6f..9256848b50e 100644 --- a/libgo/go/encoding/gob/encoder_test.go +++ b/libgo/go/encoding/gob/encoder_test.go @@ -830,6 +830,20 @@ func TestPtrToMapOfMap(t *testing.T) { } } +// Test that untyped nils generate an error, not a panic. +// See Issue 16204. +func TestCatchInvalidNilValue(t *testing.T) { + encodeErr, panicErr := encodeAndRecover(nil) + if panicErr != nil { + t.Fatalf("panicErr=%v, should not panic encoding untyped nil", panicErr) + } + if encodeErr == nil { + t.Errorf("got err=nil, want non-nil error when encoding untyped nil value") + } else if !strings.Contains(encodeErr.Error(), "nil value") { + t.Errorf("expected 'nil value' error; got err=%v", encodeErr) + } +} + // A top-level nil pointer generates a panic with a helpful string-valued message. func TestTopLevelNilPointer(t *testing.T) { var ip *int diff --git a/libgo/go/encoding/hex/example_test.go b/libgo/go/encoding/hex/example_test.go new file mode 100644 index 00000000000..fb1554eba7e --- /dev/null +++ b/libgo/go/encoding/hex/example_test.go @@ -0,0 +1,100 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package hex_test + +import ( + "encoding/hex" + "fmt" + "log" + "os" +) + +func ExampleEncode() { + src := []byte("Hello Gopher!") + + dst := make([]byte, hex.EncodedLen(len(src))) + hex.Encode(dst, src) + + fmt.Printf("%s\n", dst) + + // Output: + // 48656c6c6f20476f7068657221 +} + +func ExampleDecode() { + src := []byte("48656c6c6f20476f7068657221") + + dst := make([]byte, hex.DecodedLen(len(src))) + n, err := hex.Decode(dst, src) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s\n", dst[:n]) + + // Output: + // Hello Gopher! +} + +func ExampleDecodeString() { + const s = "48656c6c6f20476f7068657221" + decoded, err := hex.DecodeString(s) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s\n", decoded) + + // Output: + // Hello Gopher! +} + +func ExampleDump() { + content := []byte("Go is an open source programming language.") + + fmt.Printf("%s", hex.Dump(content)) + + // Output: + // 00000000 47 6f 20 69 73 20 61 6e 20 6f 70 65 6e 20 73 6f |Go is an open so| + // 00000010 75 72 63 65 20 70 72 6f 67 72 61 6d 6d 69 6e 67 |urce programming| + // 00000020 20 6c 61 6e 67 75 61 67 65 2e | language.| +} + +func ExampleDumper() { + lines := []string{ + "Go is an open source programming language.", + "\n", + "We encourage all Go users to subscribe to golang-announce.", + } + + stdoutDumper := hex.Dumper(os.Stdout) + + defer stdoutDumper.Close() + + for _, line := range lines { + stdoutDumper.Write([]byte(line)) + } + + // Output: + // 00000000 47 6f 20 69 73 20 61 6e 20 6f 70 65 6e 20 73 6f |Go is an open so| + // 00000010 75 72 63 65 20 70 72 6f 67 72 61 6d 6d 69 6e 67 |urce programming| + // 00000020 20 6c 61 6e 67 75 61 67 65 2e 0a 57 65 20 65 6e | language..We en| + // 00000030 63 6f 75 72 61 67 65 20 61 6c 6c 20 47 6f 20 75 |courage all Go u| + // 00000040 73 65 72 73 20 74 6f 20 73 75 62 73 63 72 69 62 |sers to subscrib| + // 00000050 65 20 74 6f 20 67 6f 6c 61 6e 67 2d 61 6e 6e 6f |e to golang-anno| + // 00000060 75 6e 63 65 2e |unce.| +} + +func ExampleEncodeToString() { + src := []byte("Hello") + encodedStr := hex.EncodeToString(src) + + fmt.Printf("%s\n", encodedStr) + + // Output: + // 48656c6c6f +} diff --git a/libgo/go/encoding/hex/hex.go b/libgo/go/encoding/hex/hex.go index 73a25034be1..b43c1c4b456 100644 --- a/libgo/go/encoding/hex/hex.go +++ b/libgo/go/encoding/hex/hex.go @@ -12,9 +12,13 @@ import ( "io" ) -const hextable = "0123456789abcdef" +var hextable = [16]byte{ + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'a', 'b', 'c', 'd', 'e', 'f', +} // EncodedLen returns the length of an encoding of n source bytes. +// Specifically, it returns n * 2. func EncodedLen(n int) int { return n * 2 } // Encode encodes src into EncodedLen(len(src)) @@ -40,12 +44,15 @@ func (e InvalidByteError) Error() string { return fmt.Sprintf("encoding/hex: invalid byte: %#U", rune(e)) } +// DecodedLen returns the length of a decoding of x source bytes. +// Specifically, it returns x / 2. func DecodedLen(x int) int { return x / 2 } -// Decode decodes src into DecodedLen(len(src)) bytes, returning the actual -// number of bytes written to dst. +// Decode decodes src into DecodedLen(len(src)) bytes, +// returning the actual number of bytes written to dst. // -// If Decode encounters invalid input, it returns an error describing the failure. +// Decode expects that src contain only hexadecimal +// characters and that src should have an even length. func Decode(dst, src []byte) (int, error) { if len(src)%2 == 1 { return 0, ErrLength diff --git a/libgo/go/encoding/hex/hex_test.go b/libgo/go/encoding/hex/hex_test.go index b969636cd5e..64dabbd10a2 100644 --- a/libgo/go/encoding/hex/hex_test.go +++ b/libgo/go/encoding/hex/hex_test.go @@ -6,6 +6,7 @@ package hex import ( "bytes" + "fmt" "testing" ) @@ -151,3 +152,18 @@ var expectedHexDump = []byte(`00000000 1e 1f 20 21 22 23 24 25 26 27 28 29 2a 00000010 2e 2f 30 31 32 33 34 35 36 37 38 39 3a 3b 3c 3d |./0123456789:;<=| 00000020 3e 3f 40 41 42 43 44 45 |>?@ABCDE| `) + +var sink []byte + +func BenchmarkEncode(b *testing.B) { + for _, size := range []int{256, 1024, 4096, 16384} { + src := bytes.Repeat([]byte{2, 3, 5, 7, 9, 11, 13, 17}, size/8) + sink = make([]byte, 2*size) + + b.Run(fmt.Sprintf("%v", size), func(b *testing.B) { + for i := 0; i < b.N; i++ { + Encode(sink, src) + } + }) + } +} diff --git a/libgo/go/encoding/json/decode.go b/libgo/go/encoding/json/decode.go index 2eda875bfde..77fc4607dba 100644 --- a/libgo/go/encoding/json/decode.go +++ b/libgo/go/encoding/json/decode.go @@ -34,6 +34,13 @@ import ( // the value pointed at by the pointer. If the pointer is nil, Unmarshal // allocates a new value for it to point to. // +// To unmarshal JSON into a value implementing the Unmarshaler interface, +// Unmarshal calls that value's UnmarshalJSON method, including +// when the input is a JSON null. +// Otherwise, if the value implements encoding.TextUnmarshaler +// and the input is a JSON quoted string, Unmarshal calls that value's +// UnmarshalText method with the unquoted form of the string. +// // To unmarshal JSON into a struct, Unmarshal matches incoming object // keys to the keys used by Marshal (either the struct field name or its tag), // preferring an exact match but also accepting a case-insensitive match. @@ -63,8 +70,8 @@ import ( // // To unmarshal a JSON object into a map, Unmarshal first establishes a map to // use. If the map is nil, Unmarshal allocates a new map. Otherwise Unmarshal -// reuses the existing map, keeping existing entries. Unmarshal then stores key- -// value pairs from the JSON object into the map. The map's key type must +// reuses the existing map, keeping existing entries. Unmarshal then stores +// key-value pairs from the JSON object into the map. The map's key type must // either be a string, an integer, or implement encoding.TextUnmarshaler. // // If a JSON value is not appropriate for a given target type, @@ -102,6 +109,9 @@ func Unmarshal(data []byte, v interface{}) error { // The input can be assumed to be a valid encoding of // a JSON value. UnmarshalJSON must copy the JSON data // if it wishes to retain the data after returning. +// +// By convention, to approximate the behavior of Unmarshal itself, +// Unmarshalers implement UnmarshalJSON([]byte("null")) as a no-op. type Unmarshaler interface { UnmarshalJSON([]byte) error } @@ -112,9 +122,14 @@ type UnmarshalTypeError struct { Value string // description of JSON value - "bool", "array", "number -5" Type reflect.Type // type of Go value it could not be assigned to Offset int64 // error occurred after reading Offset bytes + Struct string // name of the struct type containing the field + Field string // name of the field holding the Go value } func (e *UnmarshalTypeError) Error() string { + if e.Struct != "" || e.Field != "" { + return "json: cannot unmarshal " + e.Value + " into Go struct field " + e.Struct + "." + e.Field + " of type " + e.Type.String() + } return "json: cannot unmarshal " + e.Value + " into Go value of type " + e.Type.String() } @@ -248,10 +263,14 @@ func isValidNumber(s string) bool { // decodeState represents the state while decoding a JSON value. type decodeState struct { - data []byte - off int // read offset in data - scan scanner - nextscan scanner // for calls to nextValue + data []byte + off int // read offset in data + scan scanner + nextscan scanner // for calls to nextValue + errorContext struct { // provides context for type errors + Struct string + Field string + } savedError error useNumber bool } @@ -265,22 +284,37 @@ func (d *decodeState) init(data []byte) *decodeState { d.data = data d.off = 0 d.savedError = nil + d.errorContext.Struct = "" + d.errorContext.Field = "" return d } // error aborts the decoding by panicking with err. func (d *decodeState) error(err error) { - panic(err) + panic(d.addErrorContext(err)) } // saveError saves the first err it is called with, // for reporting at the end of the unmarshal. func (d *decodeState) saveError(err error) { if d.savedError == nil { - d.savedError = err + d.savedError = d.addErrorContext(err) } } +// addErrorContext returns a new error enhanced with information from d.errorContext +func (d *decodeState) addErrorContext(err error) error { + if d.errorContext.Struct != "" || d.errorContext.Field != "" { + switch err := err.(type) { + case *UnmarshalTypeError: + err.Struct = d.errorContext.Struct + err.Field = d.errorContext.Field + return err + } + } + return err +} + // next cuts off and returns the next full JSON value in d.data[d.off:]. // The next value is known to be an object or array, not a literal. func (d *decodeState) next() []byte { @@ -434,8 +468,10 @@ func (d *decodeState) indirect(v reflect.Value, decodingNull bool) (Unmarshaler, if u, ok := v.Interface().(Unmarshaler); ok { return u, nil, reflect.Value{} } - if u, ok := v.Interface().(encoding.TextUnmarshaler); ok { - return nil, u, reflect.Value{} + if !decodingNull { + if u, ok := v.Interface().(encoding.TextUnmarshaler); ok { + return nil, u, reflect.Value{} + } } } v = v.Elem() @@ -457,7 +493,7 @@ func (d *decodeState) array(v reflect.Value) { return } if ut != nil { - d.saveError(&UnmarshalTypeError{"array", v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "array", Type: v.Type(), Offset: int64(d.off)}) d.off-- d.next() return @@ -476,7 +512,7 @@ func (d *decodeState) array(v reflect.Value) { // Otherwise it's invalid. fallthrough default: - d.saveError(&UnmarshalTypeError{"array", v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "array", Type: v.Type(), Offset: int64(d.off)}) d.off-- d.next() return @@ -566,7 +602,7 @@ func (d *decodeState) object(v reflect.Value) { return } if ut != nil { - d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "object", Type: v.Type(), Offset: int64(d.off)}) d.off-- d.next() // skip over { } in input return @@ -594,7 +630,7 @@ func (d *decodeState) object(v reflect.Value) { reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: default: if !reflect.PtrTo(t.Key()).Implements(textUnmarshalerType) { - d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "object", Type: v.Type(), Offset: int64(d.off)}) d.off-- d.next() // skip over { } in input return @@ -604,9 +640,9 @@ func (d *decodeState) object(v reflect.Value) { v.Set(reflect.MakeMap(t)) } case reflect.Struct: - + // ok default: - d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "object", Type: v.Type(), Offset: int64(d.off)}) d.off-- d.next() // skip over { } in input return @@ -671,6 +707,8 @@ func (d *decodeState) object(v reflect.Value) { } subv = subv.Field(i) } + d.errorContext.Field = f.name + d.errorContext.Struct = v.Type().Name() } } @@ -682,7 +720,6 @@ func (d *decodeState) object(v reflect.Value) { d.error(errPhase) } - // Read value. if destring { switch qv := d.valueQuoted().(type) { case nil: @@ -714,7 +751,7 @@ func (d *decodeState) object(v reflect.Value) { s := string(key) n, err := strconv.ParseInt(s, 10, 64) if err != nil || reflect.Zero(kt).OverflowInt(n) { - d.saveError(&UnmarshalTypeError{"number " + s, kt, int64(start + 1)}) + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: kt, Offset: int64(start + 1)}) return } kv = reflect.ValueOf(n).Convert(kt) @@ -722,7 +759,7 @@ func (d *decodeState) object(v reflect.Value) { s := string(key) n, err := strconv.ParseUint(s, 10, 64) if err != nil || reflect.Zero(kt).OverflowUint(n) { - d.saveError(&UnmarshalTypeError{"number " + s, kt, int64(start + 1)}) + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: kt, Offset: int64(start + 1)}) return } kv = reflect.ValueOf(n).Convert(kt) @@ -741,6 +778,9 @@ func (d *decodeState) object(v reflect.Value) { if op != scanObjectValue { d.error(errPhase) } + + d.errorContext.Struct = "" + d.errorContext.Field = "" } } @@ -767,7 +807,7 @@ func (d *decodeState) convertNumber(s string) (interface{}, error) { } f, err := strconv.ParseFloat(s, 64) if err != nil { - return nil, &UnmarshalTypeError{"number " + s, reflect.TypeOf(0.0), int64(d.off)} + return nil, &UnmarshalTypeError{Value: "number " + s, Type: reflect.TypeOf(0.0), Offset: int64(d.off)} } return f, nil } @@ -786,8 +826,8 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) return } - wantptr := item[0] == 'n' // null - u, ut, pv := d.indirect(v, wantptr) + isNull := item[0] == 'n' // null + u, ut, pv := d.indirect(v, isNull) if u != nil { err := u.UnmarshalJSON(item) if err != nil { @@ -800,7 +840,16 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if fromQuoted { d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) } else { - d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) + var val string + switch item[0] { + case 'n': + val = "null" + case 't', 'f': + val = "bool" + default: + val = "number" + } + d.saveError(&UnmarshalTypeError{Value: val, Type: v.Type(), Offset: int64(d.off)}) } return } @@ -823,19 +872,31 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool switch c := item[0]; c { case 'n': // null + // The main parser checks that only true and false can reach here, + // but if this was a quoted string input, it could be anything. + if fromQuoted && string(item) != "null" { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + break + } switch v.Kind() { case reflect.Interface, reflect.Ptr, reflect.Map, reflect.Slice: v.Set(reflect.Zero(v.Type())) // otherwise, ignore null for primitives/string } case 't', 'f': // true, false - value := c == 't' + value := item[0] == 't' + // The main parser checks that only true and false can reach here, + // but if this was a quoted string input, it could be anything. + if fromQuoted && string(item) != "true" && string(item) != "false" { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + break + } switch v.Kind() { default: if fromQuoted { d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) } else { - d.saveError(&UnmarshalTypeError{"bool", v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "bool", Type: v.Type(), Offset: int64(d.off)}) } case reflect.Bool: v.SetBool(value) @@ -843,7 +904,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if v.NumMethod() == 0 { v.Set(reflect.ValueOf(value)) } else { - d.saveError(&UnmarshalTypeError{"bool", v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "bool", Type: v.Type(), Offset: int64(d.off)}) } } @@ -858,10 +919,10 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool } switch v.Kind() { default: - d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.off)}) case reflect.Slice: if v.Type().Elem().Kind() != reflect.Uint8 { - d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.off)}) break } b := make([]byte, base64.StdEncoding.DecodedLen(len(s))) @@ -877,7 +938,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if v.NumMethod() == 0 { v.Set(reflect.ValueOf(string(s))) } else { - d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.off)}) } } @@ -902,7 +963,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if fromQuoted { d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) } else { - d.error(&UnmarshalTypeError{"number", v.Type(), int64(d.off)}) + d.error(&UnmarshalTypeError{Value: "number", Type: v.Type(), Offset: int64(d.off)}) } case reflect.Interface: n, err := d.convertNumber(s) @@ -911,7 +972,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool break } if v.NumMethod() != 0 { - d.saveError(&UnmarshalTypeError{"number", v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "number", Type: v.Type(), Offset: int64(d.off)}) break } v.Set(reflect.ValueOf(n)) @@ -919,7 +980,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: n, err := strconv.ParseInt(s, 10, 64) if err != nil || v.OverflowInt(n) { - d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: v.Type(), Offset: int64(d.off)}) break } v.SetInt(n) @@ -927,7 +988,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: n, err := strconv.ParseUint(s, 10, 64) if err != nil || v.OverflowUint(n) { - d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: v.Type(), Offset: int64(d.off)}) break } v.SetUint(n) @@ -935,7 +996,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool case reflect.Float32, reflect.Float64: n, err := strconv.ParseFloat(s, v.Type().Bits()) if err != nil || v.OverflowFloat(n) { - d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: v.Type(), Offset: int64(d.off)}) break } v.SetFloat(n) diff --git a/libgo/go/encoding/json/decode_test.go b/libgo/go/encoding/json/decode_test.go index 255ff5c66a7..bd38ddd3190 100644 --- a/libgo/go/encoding/json/decode_test.go +++ b/libgo/go/encoding/json/decode_test.go @@ -11,6 +11,7 @@ import ( "fmt" "image" "math" + "math/big" "net" "reflect" "strconv" @@ -33,6 +34,11 @@ type V struct { F1 interface{} F2 int32 F3 Number + F4 *VOuter +} + +type VOuter struct { + V V } // ifaceNumAsFloat64/ifaceNumAsNumber are used to test unmarshaling with and @@ -374,6 +380,10 @@ type unmarshalTest struct { golden bool } +type B struct { + B bool `json:",string"` +} + var unmarshalTests = []unmarshalTest{ // basic types {in: `true`, ptr: new(bool), out: true}, @@ -389,7 +399,7 @@ var unmarshalTests = []unmarshalTest{ {in: `"g-clef: \uD834\uDD1E"`, ptr: new(string), out: "g-clef: \U0001D11E"}, {in: `"invalid: \uD834x\uDD1E"`, ptr: new(string), out: "invalid: \uFFFDx\uFFFD"}, {in: "null", ptr: new(interface{}), out: nil}, - {in: `{"X": [1,2,3], "Y": 4}`, ptr: new(T), out: T{Y: 4}, err: &UnmarshalTypeError{"array", reflect.TypeOf(""), 7}}, + {in: `{"X": [1,2,3], "Y": 4}`, ptr: new(T), out: T{Y: 4}, err: &UnmarshalTypeError{"array", reflect.TypeOf(""), 7, "T", "X"}}, {in: `{"x": 1}`, ptr: new(tx), out: tx{}}, {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: float64(1), F2: int32(2), F3: Number("3")}}, {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: Number("1"), F2: int32(2), F3: Number("3")}, useNumber: true}, @@ -504,22 +514,22 @@ var unmarshalTests = []unmarshalTest{ { in: `{"abc":"abc"}`, ptr: new(map[int]string), - err: &UnmarshalTypeError{"number abc", reflect.TypeOf(0), 2}, + err: &UnmarshalTypeError{Value: "number abc", Type: reflect.TypeOf(0), Offset: 2}, }, { in: `{"256":"abc"}`, ptr: new(map[uint8]string), - err: &UnmarshalTypeError{"number 256", reflect.TypeOf(uint8(0)), 2}, + err: &UnmarshalTypeError{Value: "number 256", Type: reflect.TypeOf(uint8(0)), Offset: 2}, }, { in: `{"128":"abc"}`, ptr: new(map[int8]string), - err: &UnmarshalTypeError{"number 128", reflect.TypeOf(int8(0)), 2}, + err: &UnmarshalTypeError{Value: "number 128", Type: reflect.TypeOf(int8(0)), Offset: 2}, }, { in: `{"-1":"abc"}`, ptr: new(map[uint8]string), - err: &UnmarshalTypeError{"number -1", reflect.TypeOf(uint8(0)), 2}, + err: &UnmarshalTypeError{Value: "number -1", Type: reflect.TypeOf(uint8(0)), Offset: 2}, }, // Map keys can be encoding.TextUnmarshalers. @@ -653,12 +663,12 @@ var unmarshalTests = []unmarshalTest{ { in: `{"2009-11-10T23:00:00Z": "hello world"}`, ptr: &map[Point]string{}, - err: &UnmarshalTypeError{"object", reflect.TypeOf(map[Point]string{}), 1}, + err: &UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(map[Point]string{}), Offset: 1}, }, { in: `{"asdf": "hello world"}`, ptr: &map[unmarshaler]string{}, - err: &UnmarshalTypeError{"object", reflect.TypeOf(map[unmarshaler]string{}), 1}, + err: &UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(map[unmarshaler]string{}), Offset: 1}, }, // related to issue 13783. @@ -738,6 +748,51 @@ var unmarshalTests = []unmarshalTest{ out: []intWithPtrMarshalText{1, 2, 3}, golden: true, }, + + {in: `0.000001`, ptr: new(float64), out: 0.000001, golden: true}, + {in: `1e-7`, ptr: new(float64), out: 1e-7, golden: true}, + {in: `100000000000000000000`, ptr: new(float64), out: 100000000000000000000.0, golden: true}, + {in: `1e+21`, ptr: new(float64), out: 1e21, golden: true}, + {in: `-0.000001`, ptr: new(float64), out: -0.000001, golden: true}, + {in: `-1e-7`, ptr: new(float64), out: -1e-7, golden: true}, + {in: `-100000000000000000000`, ptr: new(float64), out: -100000000000000000000.0, golden: true}, + {in: `-1e+21`, ptr: new(float64), out: -1e21, golden: true}, + {in: `999999999999999900000`, ptr: new(float64), out: 999999999999999900000.0, golden: true}, + {in: `9007199254740992`, ptr: new(float64), out: 9007199254740992.0, golden: true}, + {in: `9007199254740993`, ptr: new(float64), out: 9007199254740992.0, golden: false}, + + { + in: `{"V": {"F2": "hello"}}`, + ptr: new(VOuter), + err: &UnmarshalTypeError{ + Value: "string", + Struct: "V", + Field: "F2", + Type: reflect.TypeOf(int32(0)), + Offset: 20, + }, + }, + { + in: `{"V": {"F4": {}, "F2": "hello"}}`, + ptr: new(VOuter), + err: &UnmarshalTypeError{ + Value: "string", + Struct: "V", + Field: "F2", + Type: reflect.TypeOf(int32(0)), + Offset: 30, + }, + }, + + // issue 15146. + // invalid inputs in wrongStringTests below. + {in: `{"B":"true"}`, ptr: new(B), out: B{true}, golden: true}, + {in: `{"B":"false"}`, ptr: new(B), out: B{false}, golden: true}, + {in: `{"B": "maybe"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "maybe" into bool`)}, + {in: `{"B": "tru"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "tru" into bool`)}, + {in: `{"B": "False"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "False" into bool`)}, + {in: `{"B": "null"}`, ptr: new(B), out: B{false}}, + {in: `{"B": "nul"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "nul" into bool`)}, } func TestMarshal(t *testing.T) { @@ -1470,40 +1525,148 @@ func TestInterfaceSet(t *testing.T) { } } +type NullTest struct { + Bool bool + Int int + Int8 int8 + Int16 int16 + Int32 int32 + Int64 int64 + Uint uint + Uint8 uint8 + Uint16 uint16 + Uint32 uint32 + Uint64 uint64 + Float32 float32 + Float64 float64 + String string + PBool *bool + Map map[string]string + Slice []string + Interface interface{} + + PRaw *RawMessage + PTime *time.Time + PBigInt *big.Int + PText *MustNotUnmarshalText + PBuffer *bytes.Buffer // has methods, just not relevant ones + PStruct *struct{} + + Raw RawMessage + Time time.Time + BigInt big.Int + Text MustNotUnmarshalText + Buffer bytes.Buffer + Struct struct{} +} + +type NullTestStrings struct { + Bool bool `json:",string"` + Int int `json:",string"` + Int8 int8 `json:",string"` + Int16 int16 `json:",string"` + Int32 int32 `json:",string"` + Int64 int64 `json:",string"` + Uint uint `json:",string"` + Uint8 uint8 `json:",string"` + Uint16 uint16 `json:",string"` + Uint32 uint32 `json:",string"` + Uint64 uint64 `json:",string"` + Float32 float32 `json:",string"` + Float64 float64 `json:",string"` + String string `json:",string"` + PBool *bool `json:",string"` + Map map[string]string `json:",string"` + Slice []string `json:",string"` + Interface interface{} `json:",string"` + + PRaw *RawMessage `json:",string"` + PTime *time.Time `json:",string"` + PBigInt *big.Int `json:",string"` + PText *MustNotUnmarshalText `json:",string"` + PBuffer *bytes.Buffer `json:",string"` + PStruct *struct{} `json:",string"` + + Raw RawMessage `json:",string"` + Time time.Time `json:",string"` + BigInt big.Int `json:",string"` + Text MustNotUnmarshalText `json:",string"` + Buffer bytes.Buffer `json:",string"` + Struct struct{} `json:",string"` +} + // JSON null values should be ignored for primitives and string values instead of resulting in an error. // Issue 2540 func TestUnmarshalNulls(t *testing.T) { + // Unmarshal docs: + // The JSON null value unmarshals into an interface, map, pointer, or slice + // by setting that Go value to nil. Because null is often used in JSON to mean + // ``not present,'' unmarshaling a JSON null into any other Go type has no effect + // on the value and produces no error. + jsonData := []byte(`{ - "Bool" : null, - "Int" : null, - "Int8" : null, - "Int16" : null, - "Int32" : null, - "Int64" : null, - "Uint" : null, - "Uint8" : null, - "Uint16" : null, - "Uint32" : null, - "Uint64" : null, - "Float32" : null, - "Float64" : null, - "String" : null}`) - - nulls := All{ - Bool: true, - Int: 2, - Int8: 3, - Int16: 4, - Int32: 5, - Int64: 6, - Uint: 7, - Uint8: 8, - Uint16: 9, - Uint32: 10, - Uint64: 11, - Float32: 12.1, - Float64: 13.1, - String: "14"} + "Bool" : null, + "Int" : null, + "Int8" : null, + "Int16" : null, + "Int32" : null, + "Int64" : null, + "Uint" : null, + "Uint8" : null, + "Uint16" : null, + "Uint32" : null, + "Uint64" : null, + "Float32" : null, + "Float64" : null, + "String" : null, + "PBool": null, + "Map": null, + "Slice": null, + "Interface": null, + "PRaw": null, + "PTime": null, + "PBigInt": null, + "PText": null, + "PBuffer": null, + "PStruct": null, + "Raw": null, + "Time": null, + "BigInt": null, + "Text": null, + "Buffer": null, + "Struct": null + }`) + nulls := NullTest{ + Bool: true, + Int: 2, + Int8: 3, + Int16: 4, + Int32: 5, + Int64: 6, + Uint: 7, + Uint8: 8, + Uint16: 9, + Uint32: 10, + Uint64: 11, + Float32: 12.1, + Float64: 13.1, + String: "14", + PBool: new(bool), + Map: map[string]string{}, + Slice: []string{}, + Interface: new(MustNotUnmarshalJSON), + PRaw: new(RawMessage), + PTime: new(time.Time), + PBigInt: new(big.Int), + PText: new(MustNotUnmarshalText), + PStruct: new(struct{}), + PBuffer: new(bytes.Buffer), + Raw: RawMessage("123"), + Time: time.Unix(123456789, 0), + BigInt: *big.NewInt(123), + } + + before := nulls.Time.String() err := Unmarshal(jsonData, &nulls) if err != nil { @@ -1512,9 +1675,61 @@ func TestUnmarshalNulls(t *testing.T) { if !nulls.Bool || nulls.Int != 2 || nulls.Int8 != 3 || nulls.Int16 != 4 || nulls.Int32 != 5 || nulls.Int64 != 6 || nulls.Uint != 7 || nulls.Uint8 != 8 || nulls.Uint16 != 9 || nulls.Uint32 != 10 || nulls.Uint64 != 11 || nulls.Float32 != 12.1 || nulls.Float64 != 13.1 || nulls.String != "14" { - t.Errorf("Unmarshal of null values affected primitives") } + + if nulls.PBool != nil { + t.Errorf("Unmarshal of null did not clear nulls.PBool") + } + if nulls.Map != nil { + t.Errorf("Unmarshal of null did not clear nulls.Map") + } + if nulls.Slice != nil { + t.Errorf("Unmarshal of null did not clear nulls.Slice") + } + if nulls.Interface != nil { + t.Errorf("Unmarshal of null did not clear nulls.Interface") + } + if nulls.PRaw != nil { + t.Errorf("Unmarshal of null did not clear nulls.PRaw") + } + if nulls.PTime != nil { + t.Errorf("Unmarshal of null did not clear nulls.PTime") + } + if nulls.PBigInt != nil { + t.Errorf("Unmarshal of null did not clear nulls.PBigInt") + } + if nulls.PText != nil { + t.Errorf("Unmarshal of null did not clear nulls.PText") + } + if nulls.PBuffer != nil { + t.Errorf("Unmarshal of null did not clear nulls.PBuffer") + } + if nulls.PStruct != nil { + t.Errorf("Unmarshal of null did not clear nulls.PStruct") + } + + if string(nulls.Raw) != "null" { + t.Errorf("Unmarshal of RawMessage null did not record null: %v", string(nulls.Raw)) + } + if nulls.Time.String() != before { + t.Errorf("Unmarshal of time.Time null set time to %v", nulls.Time.String()) + } + if nulls.BigInt.String() != "123" { + t.Errorf("Unmarshal of big.Int null set int to %v", nulls.BigInt.String()) + } +} + +type MustNotUnmarshalJSON struct{} + +func (x MustNotUnmarshalJSON) UnmarshalJSON(data []byte) error { + return errors.New("MustNotUnmarshalJSON was used") +} + +type MustNotUnmarshalText struct{} + +func (x MustNotUnmarshalText) UnmarshalText(text []byte) error { + return errors.New("MustNotUnmarshalText was used") } func TestStringKind(t *testing.T) { @@ -1540,8 +1755,8 @@ func TestStringKind(t *testing.T) { } } -// Custom types with []byte as underlying type could not be marshalled -// and then unmarshalled. +// Custom types with []byte as underlying type could not be marshaled +// and then unmarshaled. // Issue 8962. func TestByteKind(t *testing.T) { type byteKind []byte @@ -1753,7 +1968,7 @@ var invalidUnmarshalTextTests = []struct { {nil, "json: Unmarshal(nil)"}, {struct{}{}, "json: Unmarshal(non-pointer struct {})"}, {(*int)(nil), "json: Unmarshal(nil *int)"}, - {new(net.IP), "json: cannot unmarshal string into Go value of type *net.IP"}, + {new(net.IP), "json: cannot unmarshal number into Go value of type *net.IP"}, } func TestInvalidUnmarshalText(t *testing.T) { diff --git a/libgo/go/encoding/json/encode.go b/libgo/go/encoding/json/encode.go index 6bb6de8f079..8f21ddaed94 100644 --- a/libgo/go/encoding/json/encode.go +++ b/libgo/go/encoding/json/encode.go @@ -34,7 +34,7 @@ import ( // and is not a nil pointer, Marshal calls its MarshalJSON method // to produce JSON. If no MarshalJSON method is present but the // value implements encoding.TextMarshaler instead, Marshal calls -// its MarshalText method. +// its MarshalText method and encodes the result as a JSON string. // The nil pointer exception is not strictly necessary // but mimics a similar, necessary exception in the behavior of // UnmarshalJSON. @@ -50,25 +50,33 @@ import ( // The angle brackets "<" and ">" are escaped to "\u003c" and "\u003e" // to keep some browsers from misinterpreting JSON output as HTML. // Ampersand "&" is also escaped to "\u0026" for the same reason. -// This escaping can be disabled using an Encoder with DisableHTMLEscaping. +// This escaping can be disabled using an Encoder that had SetEscapeHTML(false) +// called on it. // // Array and slice values encode as JSON arrays, except that // []byte encodes as a base64-encoded string, and a nil slice // encodes as the null JSON value. // -// Struct values encode as JSON objects. Each exported struct field -// becomes a member of the object unless -// - the field's tag is "-", or -// - the field is empty and its tag specifies the "omitempty" option. -// The empty values are false, 0, any -// nil pointer or interface value, and any array, slice, map, or string of -// length zero. The object's default key string is the struct field name -// but can be specified in the struct field's tag value. The "json" key in -// the struct field's tag value is the key name, followed by an optional comma -// and options. Examples: +// Struct values encode as JSON objects. +// Each exported struct field becomes a member of the object, using the +// field name as the object key, unless the field is omitted for one of the +// reasons given below. // -// // Field is ignored by this package. -// Field int `json:"-"` +// The encoding of each struct field can be customized by the format string +// stored under the "json" key in the struct field's tag. +// The format string gives the name of the field, possibly followed by a +// comma-separated list of options. The name may be empty in order to +// specify options without overriding the default field name. +// +// The "omitempty" option specifies that the field should be omitted +// from the encoding if the field has an empty value, defined as +// false, 0, a nil pointer, a nil interface value, and any empty array, +// slice, map, or string. +// +// As a special case, if the field tag is "-", the field is always omitted. +// Note that a field with name "-" can still be generated using the tag "-,". +// +// Examples of struct field tags and their meanings: // // // Field appears in JSON as key "myName". // Field int `json:"myName"` @@ -83,6 +91,12 @@ import ( // // Note the leading comma. // Field int `json:",omitempty"` // +// // Field is ignored by this package. +// Field int `json:"-"` +// +// // Field appears in JSON as key "-". +// Field int `json:"-,"` +// // The "string" option signals that a field is stored as JSON inside a // JSON-encoded string. It applies only to fields of string, floating point, // integer, or boolean types. This extra level of encoding is sometimes used @@ -110,7 +124,9 @@ import ( // // 1) Of those fields, if any are JSON-tagged, only tagged fields are considered, // even if there are multiple untagged fields that would otherwise conflict. +// // 2) If there is exactly one field (tagged or not according to the first rule), that is selected. +// // 3) Otherwise there are multiple fields, and all are ignored; no error occurs. // // Handling of anonymous struct fields is new in Go 1.1. @@ -427,7 +443,11 @@ func marshalerEncoder(e *encodeState, v reflect.Value, opts encOpts) { e.WriteString("null") return } - m := v.Interface().(Marshaler) + m, ok := v.Interface().(Marshaler) + if !ok { + e.WriteString("null") + return + } b, err := m.MarshalJSON() if err == nil { // copy JSON into buffer, checking validity. @@ -525,7 +545,31 @@ func (bits floatEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { if math.IsInf(f, 0) || math.IsNaN(f) { e.error(&UnsupportedValueError{v, strconv.FormatFloat(f, 'g', -1, int(bits))}) } - b := strconv.AppendFloat(e.scratch[:0], f, 'g', -1, int(bits)) + + // Convert as if by ES6 number to string conversion. + // This matches most other JSON generators. + // See golang.org/issue/6384 and golang.org/issue/14135. + // Like fmt %g, but the exponent cutoffs are different + // and exponents themselves are not padded to two digits. + b := e.scratch[:0] + abs := math.Abs(f) + fmt := byte('f') + // Note: Must use float32 comparisons for underlying float32 value to get precise cutoffs right. + if abs != 0 { + if bits == 64 && (abs < 1e-6 || abs >= 1e21) || bits == 32 && (float32(abs) < 1e-6 || float32(abs) >= 1e21) { + fmt = 'e' + } + } + b = strconv.AppendFloat(b, f, fmt, -1, int(bits)) + if fmt == 'e' { + // clean up e-09 to e-9 + n := len(b) + if n >= 4 && b[n-4] == 'e' && b[n-3] == '-' && b[n-2] == '0' { + b[n-2] = b[n-1] + b = b[:n-1] + } + } + if opts.quoted { e.WriteByte('"') } @@ -635,7 +679,7 @@ func (me *mapEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { e.error(&MarshalerError{v.Type(), err}) } } - sort.Sort(byString(sv)) + sort.Slice(sv, func(i, j int) bool { return sv[i].s < sv[j].s }) for i, kv := range sv { if i > 0 { @@ -834,15 +878,6 @@ func (w *reflectWithString) resolve() error { panic("unexpected map key type") } -// byString is a slice of reflectWithString where the reflect.Value is either -// a string or an encoding.TextMarshaler. -// It implements the methods to sort by string. -type byString []reflectWithString - -func (sv byString) Len() int { return len(sv) } -func (sv byString) Swap(i, j int) { sv[i], sv[j] = sv[j], sv[i] } -func (sv byString) Less(i, j int) bool { return sv[i].s < sv[j].s } - // NOTE: keep in sync with stringBytes below. func (e *encodeState) string(s string, escapeHTML bool) int { len0 := e.Len() @@ -850,8 +885,7 @@ func (e *encodeState) string(s string, escapeHTML bool) int { start := 0 for i := 0; i < len(s); { if b := s[i]; b < utf8.RuneSelf { - if 0x20 <= b && b != '\\' && b != '"' && - (!escapeHTML || b != '<' && b != '>' && b != '&') { + if htmlSafeSet[b] || (!escapeHTML && safeSet[b]) { i++ continue } @@ -928,8 +962,7 @@ func (e *encodeState) stringBytes(s []byte, escapeHTML bool) int { start := 0 for i := 0; i < len(s); { if b := s[i]; b < utf8.RuneSelf { - if 0x20 <= b && b != '\\' && b != '"' && - (!escapeHTML || b != '<' && b != '>' && b != '&') { + if htmlSafeSet[b] || (!escapeHTML && safeSet[b]) { i++ continue } @@ -1018,28 +1051,6 @@ func fillField(f field) field { return f } -// byName sorts field by name, breaking ties with depth, -// then breaking ties with "name came from json tag", then -// breaking ties with index sequence. -type byName []field - -func (x byName) Len() int { return len(x) } - -func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] } - -func (x byName) Less(i, j int) bool { - if x[i].name != x[j].name { - return x[i].name < x[j].name - } - if len(x[i].index) != len(x[j].index) { - return len(x[i].index) < len(x[j].index) - } - if x[i].tag != x[j].tag { - return x[i].tag - } - return byIndex(x).Less(i, j) -} - // byIndex sorts field by index sequence. type byIndex []field @@ -1157,7 +1168,22 @@ func typeFields(t reflect.Type) []field { } } - sort.Sort(byName(fields)) + sort.Slice(fields, func(i, j int) bool { + x := fields + // sort field by name, breaking ties with depth, then + // breaking ties with "name came from json tag", then + // breaking ties with index sequence. + if x[i].name != x[j].name { + return x[i].name < x[j].name + } + if len(x[i].index) != len(x[j].index) { + return len(x[i].index) < len(x[j].index) + } + if x[i].tag != x[j].tag { + return x[i].tag + } + return byIndex(x).Less(i, j) + }) // Delete all fields that are hidden by the Go rules for embedded fields, // except that fields with JSON tags are promoted. diff --git a/libgo/go/encoding/json/encode_test.go b/libgo/go/encoding/json/encode_test.go index b484022a70e..6d574cfc47b 100644 --- a/libgo/go/encoding/json/encode_test.go +++ b/libgo/go/encoding/json/encode_test.go @@ -7,8 +7,11 @@ package json import ( "bytes" "fmt" + "log" "math" "reflect" + "regexp" + "strconv" "testing" "unicode" ) @@ -290,6 +293,44 @@ type BugX struct { BugB } +// Issue 16042. Even if a nil interface value is passed in +// as long as it implements MarshalJSON, it should be marshaled. +type nilMarshaler string + +func (nm *nilMarshaler) MarshalJSON() ([]byte, error) { + if nm == nil { + return Marshal("0zenil0") + } + return Marshal("zenil:" + string(*nm)) +} + +// Issue 16042. +func TestNilMarshal(t *testing.T) { + testCases := []struct { + v interface{} + want string + }{ + {v: nil, want: `null`}, + {v: new(float64), want: `0`}, + {v: []interface{}(nil), want: `null`}, + {v: []string(nil), want: `null`}, + {v: map[string]string(nil), want: `null`}, + {v: []byte(nil), want: `null`}, + {v: struct{ M string }{"gopher"}, want: `{"M":"gopher"}`}, + {v: struct{ M Marshaler }{}, want: `{"M":null}`}, + {v: struct{ M Marshaler }{(*nilMarshaler)(nil)}, want: `{"M":"0zenil0"}`}, + {v: struct{ M interface{} }{(*nilMarshaler)(nil)}, want: `{"M":null}`}, + } + + for _, tt := range testCases { + out, err := Marshal(tt.v) + if err != nil || string(out) != tt.want { + t.Errorf("Marshal(%#v) = %#q, %#v, want %#q, nil", tt.v, out, err, tt.want) + continue + } + } +} + // Issue 5245. func TestEmbeddedBug(t *testing.T) { v := BugB{ @@ -375,6 +416,7 @@ func TestDuplicatedFieldDisappears(t *testing.T) { } func TestStringBytes(t *testing.T) { + t.Parallel() // Test that encodeState.stringBytes and encodeState.string use the same encoding. var r []rune for i := '\u0000'; i <= unicode.MaxRune; i++ { @@ -418,30 +460,6 @@ func TestStringBytes(t *testing.T) { } } -func TestIssue6458(t *testing.T) { - type Foo struct { - M RawMessage - } - x := Foo{RawMessage(`"foo"`)} - - b, err := Marshal(&x) - if err != nil { - t.Fatal(err) - } - if want := `{"M":"foo"}`; string(b) != want { - t.Errorf("Marshal(&x) = %#q; want %#q", b, want) - } - - b, err = Marshal(x) - if err != nil { - t.Fatal(err) - } - - if want := `{"M":"ImZvbyI="}`; string(b) != want { - t.Errorf("Marshal(x) = %#q; want %#q", b, want) - } -} - func TestIssue10281(t *testing.T) { type Foo struct { N Number @@ -483,7 +501,7 @@ func TestEncodePointerString(t *testing.T) { t.Fatalf("Unmarshal: %v", err) } if back.N == nil { - t.Fatalf("Unmarshalled nil N field") + t.Fatalf("Unmarshaled nil N field") } if *back.N != 42 { t.Fatalf("*N = %d; want 42", *back.N) @@ -611,3 +629,208 @@ func TestTextMarshalerMapKeysAreSorted(t *testing.T) { t.Errorf("Marshal map with text.Marshaler keys: got %#q, want %#q", b, want) } } + +var re = regexp.MustCompile + +// syntactic checks on form of marshaled floating point numbers. +var badFloatREs = []*regexp.Regexp{ + re(`p`), // no binary exponential notation + re(`^\+`), // no leading + sign + re(`^-?0[^.]`), // no unnecessary leading zeros + re(`^-?\.`), // leading zero required before decimal point + re(`\.(e|$)`), // no trailing decimal + re(`\.[0-9]+0(e|$)`), // no trailing zero in fraction + re(`^-?(0|[0-9]{2,})\..*e`), // exponential notation must have normalized mantissa + re(`e[0-9]`), // positive exponent must be signed + re(`e[+-]0`), // exponent must not have leading zeros + re(`e-[1-6]$`), // not tiny enough for exponential notation + re(`e+(.|1.|20)$`), // not big enough for exponential notation + re(`^-?0\.0000000`), // too tiny, should use exponential notation + re(`^-?[0-9]{22}`), // too big, should use exponential notation + re(`[1-9][0-9]{16}[1-9]`), // too many significant digits in integer + re(`[1-9][0-9.]{17}[1-9]`), // too many significant digits in decimal + // below here for float32 only + re(`[1-9][0-9]{8}[1-9]`), // too many significant digits in integer + re(`[1-9][0-9.]{9}[1-9]`), // too many significant digits in decimal +} + +func TestMarshalFloat(t *testing.T) { + t.Parallel() + nfail := 0 + test := func(f float64, bits int) { + vf := interface{}(f) + if bits == 32 { + f = float64(float32(f)) // round + vf = float32(f) + } + bout, err := Marshal(vf) + if err != nil { + t.Errorf("Marshal(%T(%g)): %v", vf, vf, err) + nfail++ + return + } + out := string(bout) + + // result must convert back to the same float + g, err := strconv.ParseFloat(out, bits) + if err != nil { + t.Errorf("Marshal(%T(%g)) = %q, cannot parse back: %v", vf, vf, out, err) + nfail++ + return + } + if f != g || fmt.Sprint(f) != fmt.Sprint(g) { // fmt.Sprint handles ±0 + t.Errorf("Marshal(%T(%g)) = %q (is %g, not %g)", vf, vf, out, float32(g), vf) + nfail++ + return + } + + bad := badFloatREs + if bits == 64 { + bad = bad[:len(bad)-2] + } + for _, re := range bad { + if re.MatchString(out) { + t.Errorf("Marshal(%T(%g)) = %q, must not match /%s/", vf, vf, out, re) + nfail++ + return + } + } + } + + var ( + bigger = math.Inf(+1) + smaller = math.Inf(-1) + ) + + var digits = "1.2345678901234567890123" + for i := len(digits); i >= 2; i-- { + for exp := -30; exp <= 30; exp++ { + for _, sign := range "+-" { + for bits := 32; bits <= 64; bits += 32 { + s := fmt.Sprintf("%c%se%d", sign, digits[:i], exp) + f, err := strconv.ParseFloat(s, bits) + if err != nil { + log.Fatal(err) + } + next := math.Nextafter + if bits == 32 { + next = func(g, h float64) float64 { + return float64(math.Nextafter32(float32(g), float32(h))) + } + } + test(f, bits) + test(next(f, bigger), bits) + test(next(f, smaller), bits) + if nfail > 50 { + t.Fatalf("stopping test early") + } + } + } + } + } + test(0, 64) + test(math.Copysign(0, -1), 64) + test(0, 32) + test(math.Copysign(0, -1), 32) +} + +func TestMarshalRawMessageValue(t *testing.T) { + type ( + T1 struct { + M RawMessage `json:",omitempty"` + } + T2 struct { + M *RawMessage `json:",omitempty"` + } + ) + + var ( + rawNil = RawMessage(nil) + rawEmpty = RawMessage([]byte{}) + rawText = RawMessage([]byte(`"foo"`)) + ) + + tests := []struct { + in interface{} + want string + ok bool + }{ + // Test with nil RawMessage. + {rawNil, "null", true}, + {&rawNil, "null", true}, + {[]interface{}{rawNil}, "[null]", true}, + {&[]interface{}{rawNil}, "[null]", true}, + {[]interface{}{&rawNil}, "[null]", true}, + {&[]interface{}{&rawNil}, "[null]", true}, + {struct{ M RawMessage }{rawNil}, `{"M":null}`, true}, + {&struct{ M RawMessage }{rawNil}, `{"M":null}`, true}, + {struct{ M *RawMessage }{&rawNil}, `{"M":null}`, true}, + {&struct{ M *RawMessage }{&rawNil}, `{"M":null}`, true}, + {map[string]interface{}{"M": rawNil}, `{"M":null}`, true}, + {&map[string]interface{}{"M": rawNil}, `{"M":null}`, true}, + {map[string]interface{}{"M": &rawNil}, `{"M":null}`, true}, + {&map[string]interface{}{"M": &rawNil}, `{"M":null}`, true}, + {T1{rawNil}, "{}", true}, + {T2{&rawNil}, `{"M":null}`, true}, + {&T1{rawNil}, "{}", true}, + {&T2{&rawNil}, `{"M":null}`, true}, + + // Test with empty, but non-nil, RawMessage. + {rawEmpty, "", false}, + {&rawEmpty, "", false}, + {[]interface{}{rawEmpty}, "", false}, + {&[]interface{}{rawEmpty}, "", false}, + {[]interface{}{&rawEmpty}, "", false}, + {&[]interface{}{&rawEmpty}, "", false}, + {struct{ X RawMessage }{rawEmpty}, "", false}, + {&struct{ X RawMessage }{rawEmpty}, "", false}, + {struct{ X *RawMessage }{&rawEmpty}, "", false}, + {&struct{ X *RawMessage }{&rawEmpty}, "", false}, + {map[string]interface{}{"nil": rawEmpty}, "", false}, + {&map[string]interface{}{"nil": rawEmpty}, "", false}, + {map[string]interface{}{"nil": &rawEmpty}, "", false}, + {&map[string]interface{}{"nil": &rawEmpty}, "", false}, + {T1{rawEmpty}, "{}", true}, + {T2{&rawEmpty}, "", false}, + {&T1{rawEmpty}, "{}", true}, + {&T2{&rawEmpty}, "", false}, + + // Test with RawMessage with some text. + // + // The tests below marked with Issue6458 used to generate "ImZvbyI=" instead "foo". + // This behavior was intentionally changed in Go 1.8. + // See https://github.com/golang/go/issues/14493#issuecomment-255857318 + {rawText, `"foo"`, true}, // Issue6458 + {&rawText, `"foo"`, true}, + {[]interface{}{rawText}, `["foo"]`, true}, // Issue6458 + {&[]interface{}{rawText}, `["foo"]`, true}, // Issue6458 + {[]interface{}{&rawText}, `["foo"]`, true}, + {&[]interface{}{&rawText}, `["foo"]`, true}, + {struct{ M RawMessage }{rawText}, `{"M":"foo"}`, true}, // Issue6458 + {&struct{ M RawMessage }{rawText}, `{"M":"foo"}`, true}, + {struct{ M *RawMessage }{&rawText}, `{"M":"foo"}`, true}, + {&struct{ M *RawMessage }{&rawText}, `{"M":"foo"}`, true}, + {map[string]interface{}{"M": rawText}, `{"M":"foo"}`, true}, // Issue6458 + {&map[string]interface{}{"M": rawText}, `{"M":"foo"}`, true}, // Issue6458 + {map[string]interface{}{"M": &rawText}, `{"M":"foo"}`, true}, + {&map[string]interface{}{"M": &rawText}, `{"M":"foo"}`, true}, + {T1{rawText}, `{"M":"foo"}`, true}, // Issue6458 + {T2{&rawText}, `{"M":"foo"}`, true}, + {&T1{rawText}, `{"M":"foo"}`, true}, + {&T2{&rawText}, `{"M":"foo"}`, true}, + } + + for i, tt := range tests { + b, err := Marshal(tt.in) + if ok := (err == nil); ok != tt.ok { + if err != nil { + t.Errorf("test %d, unexpected failure: %v", i, err) + } else { + t.Errorf("test %d, unexpected success", i) + } + } + if got := string(b); got != tt.want { + t.Errorf("test %d, Marshal(%#v) = %q, want %q", i, tt.in, got, tt.want) + } + } +} diff --git a/libgo/go/encoding/json/example_marshaling_test.go b/libgo/go/encoding/json/example_marshaling_test.go new file mode 100644 index 00000000000..1c4f783a693 --- /dev/null +++ b/libgo/go/encoding/json/example_marshaling_test.go @@ -0,0 +1,75 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package json_test + +import ( + "encoding/json" + "fmt" + "log" + "strings" +) + +type Animal int + +const ( + Unknown Animal = iota + Gopher + Zebra +) + +func (a *Animal) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + switch strings.ToLower(s) { + default: + *a = Unknown + case "gopher": + *a = Gopher + case "zebra": + *a = Zebra + } + + return nil +} + +func (a Animal) MarshalJSON() ([]byte, error) { + var s string + switch a { + default: + s = "unknown" + case Gopher: + s = "gopher" + case Zebra: + s = "zebra" + } + + return json.Marshal(s) +} + +func Example_customMarshalJSON() { + blob := `["gopher","armadillo","zebra","unknown","gopher","bee","gopher","zebra"]` + var zoo []Animal + if err := json.Unmarshal([]byte(blob), &zoo); err != nil { + log.Fatal(err) + } + + census := make(map[Animal]int) + for _, animal := range zoo { + census[animal] += 1 + } + + fmt.Printf("Zoo Census:\n* Gophers: %d\n* Zebras: %d\n* Unknown: %d\n", + census[Gopher], census[Zebra], census[Unknown]) + + // Output: + // Zoo Census: + // * Gophers: 3 + // * Zebras: 2 + // * Unknown: 3 +} diff --git a/libgo/go/encoding/json/scanner_test.go b/libgo/go/encoding/json/scanner_test.go index 70a28974f78..c5c1be31f1e 100644 --- a/libgo/go/encoding/json/scanner_test.go +++ b/libgo/go/encoding/json/scanner_test.go @@ -119,6 +119,7 @@ func TestCompactBig(t *testing.T) { } func TestIndentBig(t *testing.T) { + t.Parallel() initBig() var buf bytes.Buffer if err := Indent(&buf, jsonBig, "", "\t"); err != nil { diff --git a/libgo/go/encoding/json/stream.go b/libgo/go/encoding/json/stream.go index 87f0e57c6cd..95e30ce36d5 100644 --- a/libgo/go/encoding/json/stream.go +++ b/libgo/go/encoding/json/stream.go @@ -246,9 +246,12 @@ func (enc *Encoder) SetEscapeHTML(on bool) { // be used to delay JSON decoding or precompute a JSON encoding. type RawMessage []byte -// MarshalJSON returns *m as the JSON encoding of m. -func (m *RawMessage) MarshalJSON() ([]byte, error) { - return *m, nil +// MarshalJSON returns m as the JSON encoding of m. +func (m RawMessage) MarshalJSON() ([]byte, error) { + if m == nil { + return []byte("null"), nil + } + return m, nil } // UnmarshalJSON sets *m to a copy of data. diff --git a/libgo/go/encoding/json/tables.go b/libgo/go/encoding/json/tables.go new file mode 100644 index 00000000000..10acdc18c6b --- /dev/null +++ b/libgo/go/encoding/json/tables.go @@ -0,0 +1,218 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import "unicode/utf8" + +// safeSet holds the value true if the ASCII character with the given array +// position can be represented inside a JSON string without any further +// escaping. +// +// All values are true except for the ASCII control characters (0-31), the +// double quote ("), and the backslash character ("\"). +var safeSet = [utf8.RuneSelf]bool{ + ' ': true, + '!': true, + '"': false, + '#': true, + '$': true, + '%': true, + '&': true, + '\'': true, + '(': true, + ')': true, + '*': true, + '+': true, + ',': true, + '-': true, + '.': true, + '/': true, + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + ':': true, + ';': true, + '<': true, + '=': true, + '>': true, + '?': true, + '@': true, + 'A': true, + 'B': true, + 'C': true, + 'D': true, + 'E': true, + 'F': true, + 'G': true, + 'H': true, + 'I': true, + 'J': true, + 'K': true, + 'L': true, + 'M': true, + 'N': true, + 'O': true, + 'P': true, + 'Q': true, + 'R': true, + 'S': true, + 'T': true, + 'U': true, + 'V': true, + 'W': true, + 'X': true, + 'Y': true, + 'Z': true, + '[': true, + '\\': false, + ']': true, + '^': true, + '_': true, + '`': true, + 'a': true, + 'b': true, + 'c': true, + 'd': true, + 'e': true, + 'f': true, + 'g': true, + 'h': true, + 'i': true, + 'j': true, + 'k': true, + 'l': true, + 'm': true, + 'n': true, + 'o': true, + 'p': true, + 'q': true, + 'r': true, + 's': true, + 't': true, + 'u': true, + 'v': true, + 'w': true, + 'x': true, + 'y': true, + 'z': true, + '{': true, + '|': true, + '}': true, + '~': true, + '\u007f': true, +} + +// htmlSafeSet holds the value true if the ASCII character with the given +// array position can be safely represented inside a JSON string, embedded +// inside of HTML <script> tags, without any additional escaping. +// +// All values are true except for the ASCII control characters (0-31), the +// double quote ("), the backslash character ("\"), HTML opening and closing +// tags ("<" and ">"), and the ampersand ("&"). +var htmlSafeSet = [utf8.RuneSelf]bool{ + ' ': true, + '!': true, + '"': false, + '#': true, + '$': true, + '%': true, + '&': false, + '\'': true, + '(': true, + ')': true, + '*': true, + '+': true, + ',': true, + '-': true, + '.': true, + '/': true, + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + ':': true, + ';': true, + '<': false, + '=': true, + '>': false, + '?': true, + '@': true, + 'A': true, + 'B': true, + 'C': true, + 'D': true, + 'E': true, + 'F': true, + 'G': true, + 'H': true, + 'I': true, + 'J': true, + 'K': true, + 'L': true, + 'M': true, + 'N': true, + 'O': true, + 'P': true, + 'Q': true, + 'R': true, + 'S': true, + 'T': true, + 'U': true, + 'V': true, + 'W': true, + 'X': true, + 'Y': true, + 'Z': true, + '[': true, + '\\': false, + ']': true, + '^': true, + '_': true, + '`': true, + 'a': true, + 'b': true, + 'c': true, + 'd': true, + 'e': true, + 'f': true, + 'g': true, + 'h': true, + 'i': true, + 'j': true, + 'k': true, + 'l': true, + 'm': true, + 'n': true, + 'o': true, + 'p': true, + 'q': true, + 'r': true, + 's': true, + 't': true, + 'u': true, + 'v': true, + 'w': true, + 'x': true, + 'y': true, + 'z': true, + '{': true, + '|': true, + '}': true, + '~': true, + '\u007f': true, +} diff --git a/libgo/go/encoding/json/tagkey_test.go b/libgo/go/encoding/json/tagkey_test.go index c1739ea97f7..f77c49c7647 100644 --- a/libgo/go/encoding/json/tagkey_test.go +++ b/libgo/go/encoding/json/tagkey_test.go @@ -44,6 +44,10 @@ type punctuationTag struct { V string `json:"!#$%&()*+-./:<=>?@[]^_{|}~"` // https://golang.org/issue/3546 } +type dashTag struct { + V string `json:"-,"` +} + type emptyTag struct { W string } @@ -80,6 +84,7 @@ var structTagObjectKeyTests = []struct { {basicLatin6xTag{"6x"}, "6x", "abcdefghijklmno"}, {basicLatin7xTag{"7x"}, "7x", "pqrstuvwxyz"}, {miscPlaneTag{"いろはにほへと"}, "いろはにほへと", "色は匂へど"}, + {dashTag{"foo"}, "foo", "-"}, {emptyTag{"Pour Moi"}, "Pour Moi", "W"}, {misnamedTag{"Animal Kingdom"}, "Animal Kingdom", "X"}, {badFormatTag{"Orfevre"}, "Orfevre", "Y"}, diff --git a/libgo/go/encoding/pem/example_test.go b/libgo/go/encoding/pem/example_test.go new file mode 100644 index 00000000000..ffd962bd68f --- /dev/null +++ b/libgo/go/encoding/pem/example_test.go @@ -0,0 +1,46 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package pem_test + +import ( + "crypto/x509" + "encoding/pem" + "fmt" + "log" +) + +func ExampleDecode() { + var pubPEMData = []byte(` +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAlRuRnThUjU8/prwYxbty +WPT9pURI3lbsKMiB6Fn/VHOKE13p4D8xgOCADpdRagdT6n4etr9atzDKUSvpMtR3 +CP5noNc97WiNCggBjVWhs7szEe8ugyqF23XwpHQ6uV1LKH50m92MbOWfCtjU9p/x +qhNpQQ1AZhqNy5Gevap5k8XzRmjSldNAFZMY7Yv3Gi+nyCwGwpVtBUwhuLzgNFK/ +yDtw2WcWmUU7NuC8Q6MWvPebxVtCfVp/iQU6q60yyt6aGOBkhAX0LpKAEhKidixY +nP9PNVBvxgu3XZ4P36gZV6+ummKdBVnc3NqwBLu5+CcdRdusmHPHd5pHf4/38Z3/ +6qU2a/fPvWzceVTEgZ47QjFMTCTmCwNt29cvi7zZeQzjtwQgn4ipN9NibRH/Ax/q +TbIzHfrJ1xa2RteWSdFjwtxi9C20HUkjXSeI4YlzQMH0fPX6KCE7aVePTOnB69I/ +a9/q96DiXZajwlpq3wFctrs1oXqBp5DVrCIj8hU2wNgB7LtQ1mCtsYz//heai0K9 +PhE4X6hiE0YmeAZjR0uHl8M/5aW9xCoJ72+12kKpWAa0SFRWLy6FejNYCYpkupVJ +yecLk/4L1W0l6jQQZnWErXZYe0PNFcmwGXy1Rep83kfBRNKRy5tvocalLlwXLdUk +AIU+2GKjyT3iMuzZxxFxPFMCAwEAAQ== +-----END PUBLIC KEY----- +and some more`) + + block, rest := pem.Decode(pubPEMData) + if block == nil || block.Type != "PUBLIC KEY" { + log.Fatal("failed to decode PEM block containing public key") + } + + pub, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("Got a %T, with remaining data: %q", pub, rest) + // Output: Got a *rsa.PublicKey, with remaining data: "and some more" +} diff --git a/libgo/go/encoding/pem/pem.go b/libgo/go/encoding/pem/pem.go index ff2bed161ad..fbf49997d5e 100644 --- a/libgo/go/encoding/pem/pem.go +++ b/libgo/go/encoding/pem/pem.go @@ -119,19 +119,36 @@ func Decode(data []byte) (p *Block, rest []byte) { rest = next } - var endIndex int + var endIndex, endTrailerIndex int + // If there were no headers, the END line might occur // immediately, without a leading newline. if len(p.Headers) == 0 && bytes.HasPrefix(rest, pemEnd[1:]) { endIndex = 0 + endTrailerIndex = len(pemEnd) - 1 } else { endIndex = bytes.Index(rest, pemEnd) + endTrailerIndex = endIndex + len(pemEnd) } if endIndex < 0 { return decodeError(data, rest) } + // After the "-----" of the ending line should be the same type and a + // final five dashes. + endTrailer := rest[endTrailerIndex:] + endTrailerLen := len(typeLine) + len(pemEndOfLine) + if len(endTrailer) < endTrailerLen { + return decodeError(data, rest) + } + + endTrailer = endTrailer[:endTrailerLen] + if !bytes.HasPrefix(endTrailer, typeLine) || + !bytes.HasSuffix(endTrailer, pemEndOfLine) { + return decodeError(data, rest) + } + base64Data := removeWhitespace(rest[:endIndex]) p.Bytes = make([]byte, base64.StdEncoding.DecodedLen(len(base64Data))) n, err := base64.StdEncoding.Decode(p.Bytes, base64Data) diff --git a/libgo/go/encoding/pem/pem_test.go b/libgo/go/encoding/pem/pem_test.go index 958dbc1a3a6..6321dec3820 100644 --- a/libgo/go/encoding/pem/pem_test.go +++ b/libgo/go/encoding/pem/pem_test.go @@ -78,6 +78,48 @@ func TestDecode(t *testing.T) { } } +const pemTooFewEndingDashes = ` +-----BEGIN FOO----- +dGVzdA== +-----END FOO----` + +const pemWrongEndingType = ` +-----BEGIN FOO----- +dGVzdA== +-----END BAR-----` + +const pemMissingEndingSpace = ` +-----BEGIN FOO----- +dGVzdA== +-----ENDBAR-----` + +var badPEMTests = []struct { + name string + input string +}{ + { + "too few trailing dashes", + pemTooFewEndingDashes, + }, + { + "incorrect ending type", + pemWrongEndingType, + }, + { + "missing ending space", + pemMissingEndingSpace, + }, +} + +func TestBadDecode(t *testing.T) { + for _, test := range badPEMTests { + result, _ := Decode([]byte(test.input)) + if result != nil { + t.Errorf("unexpected success while parsing %q", test.name) + } + } +} + func TestEncode(t *testing.T) { r := EncodeToMemory(privateKey2) if string(r) != pemPrivateKey2 { diff --git a/libgo/go/encoding/xml/marshal.go b/libgo/go/encoding/xml/marshal.go index abb078ce270..1176f5d717d 100644 --- a/libgo/go/encoding/xml/marshal.go +++ b/libgo/go/encoding/xml/marshal.go @@ -24,10 +24,10 @@ const ( // Marshal returns the XML encoding of v. // -// Marshal handles an array or slice by marshalling each of the elements. -// Marshal handles a pointer by marshalling the value it points at or, if the +// Marshal handles an array or slice by marshaling each of the elements. +// Marshal handles a pointer by marshaling the value it points at or, if the // pointer is nil, by writing nothing. Marshal handles an interface value by -// marshalling the value it contains or, if the interface value is nil, by +// marshaling the value it contains or, if the interface value is nil, by // writing nothing. Marshal handles all other data by writing one or more XML // elements containing the data. // @@ -36,9 +36,9 @@ const ( // - the value of the XMLName field of type Name // - the tag of the struct field used to obtain the data // - the name of the struct field used to obtain the data -// - the name of the marshalled type +// - the name of the marshaled type // -// The XML element for a struct contains marshalled elements for each of the +// The XML element for a struct contains marshaled elements for each of the // exported fields of the struct, with these exceptions: // - the XMLName field, described above, is omitted. // - a field with tag "-" is omitted. @@ -51,9 +51,9 @@ const ( // - a field with tag ",cdata" is written as character data // wrapped in one or more <![CDATA[ ... ]]> tags, not as an XML element. // - a field with tag ",innerxml" is written verbatim, not subject -// to the usual marshalling procedure. +// to the usual marshaling procedure. // - a field with tag ",comment" is written as an XML comment, not -// subject to the usual marshalling procedure. It must not contain +// subject to the usual marshaling procedure. It must not contain // the "--" string within it. // - a field with a tag including the "omitempty" option is omitted // if the field value is empty. The empty values are false, 0, any @@ -494,7 +494,6 @@ func (p *printer) marshalValue(val reflect.Value, finfo *fieldInfo, startTemplat continue } fv := finfo.value(val) - name := Name{Space: finfo.xmlns, Local: finfo.name} if finfo.flags&fOmitEmpty != 0 && isEmptyValue(fv) { continue @@ -504,69 +503,10 @@ func (p *printer) marshalValue(val reflect.Value, finfo *fieldInfo, startTemplat continue } - if fv.CanInterface() && fv.Type().Implements(marshalerAttrType) { - attr, err := fv.Interface().(MarshalerAttr).MarshalXMLAttr(name) - if err != nil { - return err - } - if attr.Name.Local != "" { - start.Attr = append(start.Attr, attr) - } - continue - } - - if fv.CanAddr() { - pv := fv.Addr() - if pv.CanInterface() && pv.Type().Implements(marshalerAttrType) { - attr, err := pv.Interface().(MarshalerAttr).MarshalXMLAttr(name) - if err != nil { - return err - } - if attr.Name.Local != "" { - start.Attr = append(start.Attr, attr) - } - continue - } - } - - if fv.CanInterface() && fv.Type().Implements(textMarshalerType) { - text, err := fv.Interface().(encoding.TextMarshaler).MarshalText() - if err != nil { - return err - } - start.Attr = append(start.Attr, Attr{name, string(text)}) - continue - } - - if fv.CanAddr() { - pv := fv.Addr() - if pv.CanInterface() && pv.Type().Implements(textMarshalerType) { - text, err := pv.Interface().(encoding.TextMarshaler).MarshalText() - if err != nil { - return err - } - start.Attr = append(start.Attr, Attr{name, string(text)}) - continue - } - } - - // Dereference or skip nil pointer, interface values. - switch fv.Kind() { - case reflect.Ptr, reflect.Interface: - if fv.IsNil() { - continue - } - fv = fv.Elem() - } - - s, b, err := p.marshalSimple(fv.Type(), fv) - if err != nil { + name := Name{Space: finfo.xmlns, Local: finfo.name} + if err := p.marshalAttr(&start, name, fv); err != nil { return err } - if b != nil { - s = string(b) - } - start.Attr = append(start.Attr, Attr{name, s}) } if err := p.writeStart(&start); err != nil { @@ -596,6 +536,90 @@ func (p *printer) marshalValue(val reflect.Value, finfo *fieldInfo, startTemplat return p.cachedWriteError() } +// marshalAttr marshals an attribute with the given name and value, adding to start.Attr. +func (p *printer) marshalAttr(start *StartElement, name Name, val reflect.Value) error { + if val.CanInterface() && val.Type().Implements(marshalerAttrType) { + attr, err := val.Interface().(MarshalerAttr).MarshalXMLAttr(name) + if err != nil { + return err + } + if attr.Name.Local != "" { + start.Attr = append(start.Attr, attr) + } + return nil + } + + if val.CanAddr() { + pv := val.Addr() + if pv.CanInterface() && pv.Type().Implements(marshalerAttrType) { + attr, err := pv.Interface().(MarshalerAttr).MarshalXMLAttr(name) + if err != nil { + return err + } + if attr.Name.Local != "" { + start.Attr = append(start.Attr, attr) + } + return nil + } + } + + if val.CanInterface() && val.Type().Implements(textMarshalerType) { + text, err := val.Interface().(encoding.TextMarshaler).MarshalText() + if err != nil { + return err + } + start.Attr = append(start.Attr, Attr{name, string(text)}) + return nil + } + + if val.CanAddr() { + pv := val.Addr() + if pv.CanInterface() && pv.Type().Implements(textMarshalerType) { + text, err := pv.Interface().(encoding.TextMarshaler).MarshalText() + if err != nil { + return err + } + start.Attr = append(start.Attr, Attr{name, string(text)}) + return nil + } + } + + // Dereference or skip nil pointer, interface values. + switch val.Kind() { + case reflect.Ptr, reflect.Interface: + if val.IsNil() { + return nil + } + val = val.Elem() + } + + // Walk slices. + if val.Kind() == reflect.Slice && val.Type().Elem().Kind() != reflect.Uint8 { + n := val.Len() + for i := 0; i < n; i++ { + if err := p.marshalAttr(start, name, val.Index(i)); err != nil { + return err + } + } + return nil + } + + if val.Type() == attrType { + start.Attr = append(start.Attr, val.Interface().(Attr)) + return nil + } + + s, b, err := p.marshalSimple(val.Type(), val) + if err != nil { + return err + } + if b != nil { + s = string(b) + } + start.Attr = append(start.Attr, Attr{name, s}) + return nil +} + // defaultStart returns the default start element to use, // given the reflect type, field info, and start template. func defaultStart(typ reflect.Type, finfo *fieldInfo, startTemplate *StartElement) StartElement { @@ -760,14 +784,6 @@ func (p *printer) marshalStruct(tinfo *typeInfo, val reflect.Value) error { } vf := finfo.value(val) - // Dereference or skip nil pointer, interface values. - switch vf.Kind() { - case reflect.Ptr, reflect.Interface: - if !vf.IsNil() { - vf = vf.Elem() - } - } - switch finfo.flags & fMode { case fCDATA, fCharData: emit := EscapeText @@ -800,6 +816,16 @@ func (p *printer) marshalStruct(tinfo *typeInfo, val reflect.Value) error { continue } } + // Drill into interfaces and pointers. + // This can turn into an infinite loop given a cyclic chain, + // but it matches the Go 1 behavior. + for vf.Kind() == reflect.Interface || vf.Kind() == reflect.Ptr { + if vf.IsNil() { + return nil + } + vf = vf.Elem() + } + var scratch [64]byte switch vf.Kind() { case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: diff --git a/libgo/go/encoding/xml/marshal_test.go b/libgo/go/encoding/xml/marshal_test.go index fe8b16fe431..d79b99a1e0b 100644 --- a/libgo/go/encoding/xml/marshal_test.go +++ b/libgo/go/encoding/xml/marshal_test.go @@ -199,6 +199,17 @@ type AttrTest struct { Bytes []byte `xml:",attr"` } +type AttrsTest struct { + Attrs []Attr `xml:",any,attr"` + Int int `xml:",attr"` + Named int `xml:"int,attr"` + Float float64 `xml:",attr"` + Uint8 uint8 `xml:",attr"` + Bool bool `xml:",attr"` + Str string `xml:",attr"` + Bytes []byte `xml:",attr"` +} + type OmitAttrTest struct { Int int `xml:",attr,omitempty"` Named int `xml:"int,attr,omitempty"` @@ -207,6 +218,7 @@ type OmitAttrTest struct { Bool bool `xml:",attr,omitempty"` Str string `xml:",attr,omitempty"` Bytes []byte `xml:",attr,omitempty"` + PStr *string `xml:",attr,omitempty"` } type OmitFieldTest struct { @@ -217,6 +229,7 @@ type OmitFieldTest struct { Bool bool `xml:",omitempty"` Str string `xml:",omitempty"` Bytes []byte `xml:",omitempty"` + PStr *string `xml:",omitempty"` Ptr *PresenceTest `xml:",omitempty"` } @@ -317,6 +330,10 @@ func (m *MyMarshalerAttrTest) MarshalXMLAttr(name Name) (Attr, error) { return Attr{name, "hello world"}, nil } +func (m *MyMarshalerAttrTest) UnmarshalXMLAttr(attr Attr) error { + return nil +} + type MarshalerStruct struct { Foo MyMarshalerAttrTest `xml:",attr"` } @@ -373,12 +390,13 @@ var ( nameAttr = "Sarah" ageAttr = uint(12) contentsAttr = "lorem ipsum" + empty = "" ) // Unless explicitly stated as such (or *Plain), all of the // tests below are two-way tests. When introducing new tests, // please try to make them two-way as well to ensure that -// marshalling and unmarshalling are as symmetrical as feasible. +// marshaling and unmarshaling are as symmetrical as feasible. var marshalTests = []struct { Value interface{} ExpectXML string @@ -823,6 +841,53 @@ var marshalTests = []struct { ` Bool="false" Str="" Bytes=""></AttrTest>`, }, { + Value: &AttrsTest{ + Attrs: []Attr{ + {Name: Name{Local: "Answer"}, Value: "42"}, + {Name: Name{Local: "Int"}, Value: "8"}, + {Name: Name{Local: "int"}, Value: "9"}, + {Name: Name{Local: "Float"}, Value: "23.5"}, + {Name: Name{Local: "Uint8"}, Value: "255"}, + {Name: Name{Local: "Bool"}, Value: "true"}, + {Name: Name{Local: "Str"}, Value: "str"}, + {Name: Name{Local: "Bytes"}, Value: "byt"}, + }, + }, + ExpectXML: `<AttrsTest Answer="42" Int="8" int="9" Float="23.5" Uint8="255" Bool="true" Str="str" Bytes="byt" Int="0" int="0" Float="0" Uint8="0" Bool="false" Str="" Bytes=""></AttrsTest>`, + MarshalOnly: true, + }, + { + Value: &AttrsTest{ + Attrs: []Attr{ + {Name: Name{Local: "Answer"}, Value: "42"}, + }, + Int: 8, + Named: 9, + Float: 23.5, + Uint8: 255, + Bool: true, + Str: "str", + Bytes: []byte("byt"), + }, + ExpectXML: `<AttrsTest Answer="42" Int="8" int="9" Float="23.5" Uint8="255" Bool="true" Str="str" Bytes="byt"></AttrsTest>`, + }, + { + Value: &AttrsTest{ + Attrs: []Attr{ + {Name: Name{Local: "Int"}, Value: "0"}, + {Name: Name{Local: "int"}, Value: "0"}, + {Name: Name{Local: "Float"}, Value: "0"}, + {Name: Name{Local: "Uint8"}, Value: "0"}, + {Name: Name{Local: "Bool"}, Value: "false"}, + {Name: Name{Local: "Str"}}, + {Name: Name{Local: "Bytes"}}, + }, + Bytes: []byte{}, + }, + ExpectXML: `<AttrsTest Int="0" int="0" Float="0" Uint8="0" Bool="false" Str="" Bytes="" Int="0" int="0" Float="0" Uint8="0" Bool="false" Str="" Bytes=""></AttrsTest>`, + MarshalOnly: true, + }, + { Value: &OmitAttrTest{ Int: 8, Named: 9, @@ -831,9 +896,10 @@ var marshalTests = []struct { Bool: true, Str: "str", Bytes: []byte("byt"), + PStr: &empty, }, ExpectXML: `<OmitAttrTest Int="8" int="9" Float="23.5" Uint8="255"` + - ` Bool="true" Str="str" Bytes="byt"></OmitAttrTest>`, + ` Bool="true" Str="str" Bytes="byt" PStr=""></OmitAttrTest>`, }, { Value: &OmitAttrTest{}, @@ -864,6 +930,7 @@ var marshalTests = []struct { Bool: true, Str: "str", Bytes: []byte("byt"), + PStr: &empty, Ptr: &PresenceTest{}, }, ExpectXML: `<OmitFieldTest>` + @@ -874,6 +941,7 @@ var marshalTests = []struct { `<Bool>true</Bool>` + `<Str>str</Str>` + `<Bytes>byt</Bytes>` + + `<PStr></PStr>` + `<Ptr></Ptr>` + `</OmitFieldTest>`, }, @@ -1092,7 +1160,7 @@ type AttrParent struct { } type BadAttr struct { - Name []string `xml:"name,attr"` + Name map[string]string `xml:"name,attr"` } var marshalErrorTests = []struct { @@ -1128,8 +1196,8 @@ var marshalErrorTests = []struct { Err: `xml: X>Y chain not valid with attr flag`, }, { - Value: BadAttr{[]string{"X", "Y"}}, - Err: `xml: unsupported type: []string`, + Value: BadAttr{map[string]string{"X": "Y"}}, + Err: `xml: unsupported type: map[string]string`, }, } @@ -1732,7 +1800,7 @@ func TestDecodeEncode(t *testing.T) { in.WriteString(`<?xml version="1.0" encoding="UTF-8"?> <?Target Instruction?> <root> -</root> +</root> `) dec := NewDecoder(&in) enc := NewEncoder(&out) @@ -1823,3 +1891,14 @@ func TestSimpleUseOfEncodeToken(t *testing.T) { t.Errorf("enc.EncodeToken: expected %q; got %q", want, buf.String()) } } + +// Issue 16158. Decoder.unmarshalAttr ignores the return value of copyValue. +func TestIssue16158(t *testing.T) { + const data = `<foo b="HELLOWORLD"></foo>` + err := Unmarshal([]byte(data), &struct { + B byte `xml:"b,attr,omitempty"` + }{}) + if err == nil { + t.Errorf("Unmarshal: expected error, got nil") + } +} diff --git a/libgo/go/encoding/xml/read.go b/libgo/go/encoding/xml/read.go index 937432e3a7e..5a89d5f504a 100644 --- a/libgo/go/encoding/xml/read.go +++ b/libgo/go/encoding/xml/read.go @@ -52,6 +52,11 @@ import ( // the explicit name in a struct field tag of the form "name,attr", // Unmarshal records the attribute value in that field. // +// * If the XML element has an attribute not handled by the previous +// rule and the struct has a field with an associated tag containing +// ",any,attr", Unmarshal records the attribute value in the first +// such field. +// // * If the XML element contains character data, that data is // accumulated in the first struct field that has tag ",chardata". // The struct field may have type []byte or string. @@ -85,7 +90,7 @@ import ( // * An anonymous struct field is handled as if the fields of its // value were part of the outer struct. // -// * A struct field with tag "-" is never unmarshalled into. +// * A struct field with tag "-" is never unmarshaled into. // // Unmarshal maps an XML element to a string or []byte by saving the // concatenation of that element's character data in the string or @@ -94,8 +99,12 @@ import ( // Unmarshal maps an attribute value to a string or []byte by saving // the value in the string or slice. // -// Unmarshal maps an XML element to a slice by extending the length of -// the slice and mapping the element to the newly created value. +// Unmarshal maps an attribute value to an Attr by saving the attribute, +// including its name, in the Attr. +// +// Unmarshal maps an XML element or attribute value to a slice by +// extending the length of the slice and mapping the element or attribute +// to the newly created value. // // Unmarshal maps an XML element or attribute value to a bool by // setting it to the boolean value represented by the string. @@ -133,7 +142,7 @@ func (d *Decoder) DecodeElement(v interface{}, start *StartElement) error { return d.unmarshal(val.Elem(), start) } -// An UnmarshalError represents an error in the unmarshalling process. +// An UnmarshalError represents an error in the unmarshaling process. type UnmarshalError string func (e UnmarshalError) Error() string { return string(e) } @@ -232,7 +241,6 @@ func (p *Decoder) unmarshalAttr(val reflect.Value, attr Attr) error { } val = val.Elem() } - if val.CanInterface() && val.Type().Implements(unmarshalerAttrType) { // This is an unmarshaler with a non-pointer receiver, // so it's likely to be incorrect, but we do what we're told. @@ -258,11 +266,30 @@ func (p *Decoder) unmarshalAttr(val reflect.Value, attr Attr) error { } } - copyValue(val, []byte(attr.Value)) - return nil + if val.Type().Kind() == reflect.Slice && val.Type().Elem().Kind() != reflect.Uint8 { + // Slice of element values. + // Grow slice. + n := val.Len() + val.Set(reflect.Append(val, reflect.Zero(val.Type().Elem()))) + + // Recur to read element into slice. + if err := p.unmarshalAttr(val.Index(n), attr); err != nil { + val.SetLen(n) + return err + } + return nil + } + + if val.Type() == attrType { + val.Set(reflect.ValueOf(attr)) + return nil + } + + return copyValue(val, []byte(attr.Value)) } var ( + attrType = reflect.TypeOf(Attr{}) unmarshalerType = reflect.TypeOf((*Unmarshaler)(nil)).Elem() unmarshalerAttrType = reflect.TypeOf((*UnmarshalerAttr)(nil)).Elem() textUnmarshalerType = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem() @@ -359,16 +386,7 @@ func (p *Decoder) unmarshal(val reflect.Value, start *StartElement) error { // Slice of element values. // Grow slice. n := v.Len() - if n >= v.Cap() { - ncap := 2 * n - if ncap < 4 { - ncap = 4 - } - new := reflect.MakeSlice(typ, n, ncap) - reflect.Copy(new, v) - v.Set(new) - } - v.SetLen(n + 1) + v.Set(reflect.Append(val, reflect.Zero(v.Type().Elem()))) // Recur to read element into slice. if err := p.unmarshal(v.Index(n), start); err != nil { @@ -415,22 +433,40 @@ func (p *Decoder) unmarshal(val reflect.Value, start *StartElement) error { } // Assign attributes. - // Also, determine whether we need to save character data or comments. - for i := range tinfo.fields { - finfo := &tinfo.fields[i] - switch finfo.flags & fMode { - case fAttr: - strv := finfo.value(sv) - // Look for attribute. - for _, a := range start.Attr { + for _, a := range start.Attr { + handled := false + any := -1 + for i := range tinfo.fields { + finfo := &tinfo.fields[i] + switch finfo.flags & fMode { + case fAttr: + strv := finfo.value(sv) if a.Name.Local == finfo.name && (finfo.xmlns == "" || finfo.xmlns == a.Name.Space) { if err := p.unmarshalAttr(strv, a); err != nil { return err } - break + handled = true + } + + case fAny | fAttr: + if any == -1 { + any = i } } + } + if !handled && any >= 0 { + finfo := &tinfo.fields[any] + strv := finfo.value(sv) + if err := p.unmarshalAttr(strv, a); err != nil { + return err + } + } + } + // Determine whether we need to save character data or comments. + for i := range tinfo.fields { + finfo := &tinfo.fields[i] + switch finfo.flags & fMode { case fCDATA, fCharData: if !saveData.IsValid() { saveData = finfo.value(sv) @@ -546,7 +582,9 @@ Loop: case reflect.String: t.SetString(string(saveXMLData)) case reflect.Slice: - t.Set(reflect.ValueOf(saveXMLData)) + if t.Type().Elem().Kind() == reflect.Uint8 { + t.Set(reflect.ValueOf(saveXMLData)) + } } return nil diff --git a/libgo/go/encoding/xml/read_test.go b/libgo/go/encoding/xml/read_test.go index 7a98092803a..273c303d16d 100644 --- a/libgo/go/encoding/xml/read_test.go +++ b/libgo/go/encoding/xml/read_test.go @@ -705,7 +705,7 @@ func TestUnmarshalIntoInterface(t *testing.T) { } pea, ok := pod.Pea.(*Pea) if !ok { - t.Fatalf("unmarshalled into wrong type: have %T want *Pea", pod.Pea) + t.Fatalf("unmarshaled into wrong type: have %T want *Pea", pod.Pea) } have, want := pea.Cotelydon, "Green stuff" if have != want { @@ -733,3 +733,22 @@ func TestMalformedComment(t *testing.T) { } } } + +type IXField struct { + Five int `xml:"five"` + NotInnerXML []string `xml:",innerxml"` +} + +// Issue 15600. ",innerxml" on a field that can't hold it. +func TestInvalidInnerXMLType(t *testing.T) { + v := new(IXField) + if err := Unmarshal([]byte(`<tag><five>5</five><innertag/></tag>`), v); err != nil { + t.Errorf("Unmarshal failed: got %v", err) + } + if v.Five != 5 { + t.Errorf("Five = %v, want 5", v.Five) + } + if v.NotInnerXML != nil { + t.Errorf("NotInnerXML = %v, want nil", v.NotInnerXML) + } +} diff --git a/libgo/go/encoding/xml/typeinfo.go b/libgo/go/encoding/xml/typeinfo.go index 70da962ffaa..6623c78308e 100644 --- a/libgo/go/encoding/xml/typeinfo.go +++ b/libgo/go/encoding/xml/typeinfo.go @@ -48,7 +48,7 @@ var tinfoLock sync.RWMutex var nameType = reflect.TypeOf(Name{}) // getTypeInfo returns the typeInfo structure with details necessary -// for marshalling and unmarshalling typ. +// for marshaling and unmarshaling typ. func getTypeInfo(typ reflect.Type) (*typeInfo, error) { tinfoLock.RLock() tinfo, ok := tinfoMap[typ] @@ -151,7 +151,7 @@ func structFieldInfo(typ reflect.Type, f *reflect.StructField) (*fieldInfo, erro switch mode := finfo.flags & fMode; mode { case 0: finfo.flags |= fElement - case fAttr, fCDATA, fCharData, fInnerXml, fComment, fAny: + case fAttr, fCDATA, fCharData, fInnerXml, fComment, fAny, fAny | fAttr: if f.Name == "XMLName" || tag != "" && mode != fAttr { valid = false } @@ -214,7 +214,7 @@ func structFieldInfo(typ reflect.Type, f *reflect.StructField) (*fieldInfo, erro } // If the field type has an XMLName field, the names must match - // so that the behavior of both marshalling and unmarshalling + // so that the behavior of both marshaling and unmarshaling // is straightforward and unambiguous. if finfo.flags&fElement != 0 { ftyp := f.Type @@ -334,7 +334,7 @@ Loop: return nil } -// A TagPathError represents an error in the unmarshalling process +// A TagPathError represents an error in the unmarshaling process // caused by the use of field tags with conflicting paths. type TagPathError struct { Struct reflect.Type |