diff options
Diffstat (limited to 'libgo/go/encoding/json')
-rw-r--r-- | libgo/go/encoding/json/bench_test.go | 28 | ||||
-rw-r--r-- | libgo/go/encoding/json/decode.go | 194 | ||||
-rw-r--r-- | libgo/go/encoding/json/decode_test.go | 57 | ||||
-rw-r--r-- | libgo/go/encoding/json/encode.go | 164 | ||||
-rw-r--r-- | libgo/go/encoding/json/encode_test.go | 28 | ||||
-rw-r--r-- | libgo/go/encoding/json/example_text_marshaling_test.go | 69 | ||||
-rw-r--r-- | libgo/go/encoding/json/scanner.go | 2 | ||||
-rw-r--r-- | libgo/go/encoding/json/stream.go | 24 | ||||
-rw-r--r-- | libgo/go/encoding/json/stream_test.go | 34 |
9 files changed, 388 insertions, 212 deletions
diff --git a/libgo/go/encoding/json/bench_test.go b/libgo/go/encoding/json/bench_test.go index bd322db2e6f..72cb349062c 100644 --- a/libgo/go/encoding/json/bench_test.go +++ b/libgo/go/encoding/json/bench_test.go @@ -114,6 +114,34 @@ func BenchmarkCodeMarshal(b *testing.B) { b.SetBytes(int64(len(codeJSON))) } +func benchMarshalBytes(n int) func(*testing.B) { + sample := []byte("hello world") + // Use a struct pointer, to avoid an allocation when passing it as an + // interface parameter to Marshal. + v := &struct { + Bytes []byte + }{ + bytes.Repeat(sample, (n/len(sample))+1)[:n], + } + return func(b *testing.B) { + for i := 0; i < b.N; i++ { + if _, err := Marshal(v); err != nil { + b.Fatal("Marshal:", err) + } + } + } +} + +func BenchmarkMarshalBytes(b *testing.B) { + // 32 fits within encodeState.scratch. + b.Run("32", benchMarshalBytes(32)) + // 256 doesn't fit in encodeState.scratch, but is small enough to + // allocate and avoid the slower base64.NewEncoder. + b.Run("256", benchMarshalBytes(256)) + // 4096 is large enough that we want to avoid allocating for it. + b.Run("4096", benchMarshalBytes(4096)) +} + func BenchmarkCodeDecoder(b *testing.B) { if codeJSON == nil { b.StopTimer() diff --git a/libgo/go/encoding/json/decode.go b/libgo/go/encoding/json/decode.go index 7d235087e60..731553dca6b 100644 --- a/libgo/go/encoding/json/decode.go +++ b/libgo/go/encoding/json/decode.go @@ -11,7 +11,6 @@ import ( "bytes" "encoding" "encoding/base64" - "errors" "fmt" "reflect" "strconv" @@ -179,7 +178,7 @@ func (d *decodeState) unmarshal(v interface{}) error { // test must be applied at the top level of the value. err := d.value(rv) if err != nil { - return err + return d.addErrorContext(err) } return d.savedError } @@ -267,7 +266,7 @@ type decodeState struct { opcode int // last read result scan scanner errorContext struct { // provides context for type errors - Struct string + Struct reflect.Type Field string } savedError error @@ -280,16 +279,16 @@ func (d *decodeState) readIndex() int { return d.off - 1 } -// errPhase is used for errors that should not happen unless -// there is a bug in the JSON decoder or something is editing -// the data slice while the decoder executes. -var errPhase = errors.New("JSON decoder out of sync - data changing underfoot?") +// phasePanicMsg is used as a panic message when we end up with something that +// shouldn't happen. It can indicate a bug in the JSON decoder, or that +// something is editing the data slice while the decoder executes. +const phasePanicMsg = "JSON decoder out of sync - data changing underfoot?" func (d *decodeState) init(data []byte) *decodeState { d.data = data d.off = 0 d.savedError = nil - d.errorContext.Struct = "" + d.errorContext.Struct = nil d.errorContext.Field = "" return d } @@ -304,10 +303,10 @@ func (d *decodeState) saveError(err error) { // addErrorContext returns a new error enhanced with information from d.errorContext func (d *decodeState) addErrorContext(err error) error { - if d.errorContext.Struct != "" || d.errorContext.Field != "" { + if d.errorContext.Struct != nil || d.errorContext.Field != "" { switch err := err.(type) { case *UnmarshalTypeError: - err.Struct = d.errorContext.Struct + err.Struct = d.errorContext.Struct.Name() err.Field = d.errorContext.Field return err } @@ -332,13 +331,12 @@ func (d *decodeState) skip() { // scanNext processes the byte at d.data[d.off]. func (d *decodeState) scanNext() { - s, data, i := &d.scan, d.data, d.off - if i < len(data) { - d.opcode = s.step(s, data[i]) - d.off = i + 1 + if d.off < len(d.data) { + d.opcode = d.scan.step(&d.scan, d.data[d.off]) + d.off++ } else { - d.opcode = s.eof() - d.off = len(data) + 1 // mark processed EOF with len+1 + d.opcode = d.scan.eof() + d.off = len(d.data) + 1 // mark processed EOF with len+1 } } @@ -346,7 +344,7 @@ func (d *decodeState) scanNext() { // receives a scan code not equal to op. func (d *decodeState) scanWhile(op int) { s, data, i := &d.scan, d.data, d.off - for i < len(d.data) { + for i < len(data) { newOp := s.step(s, data[i]) i++ if newOp != op { @@ -356,7 +354,7 @@ func (d *decodeState) scanWhile(op int) { } } - d.off = len(d.data) + 1 // mark processed EOF with len+1 + d.off = len(data) + 1 // mark processed EOF with len+1 d.opcode = d.scan.eof() } @@ -366,7 +364,7 @@ func (d *decodeState) scanWhile(op int) { func (d *decodeState) value(v reflect.Value) error { switch d.opcode { default: - return errPhase + panic(phasePanicMsg) case scanBeginArray: if v.IsValid() { @@ -408,30 +406,23 @@ type unquotedValue struct{} // quoted string literal or literal null into an interface value. // If it finds anything other than a quoted string literal or null, // valueQuoted returns unquotedValue{}. -func (d *decodeState) valueQuoted() (interface{}, error) { +func (d *decodeState) valueQuoted() interface{} { switch d.opcode { default: - return nil, errPhase + panic(phasePanicMsg) - case scanBeginArray: - d.skip() - d.scanNext() - - case scanBeginObject: + case scanBeginArray, scanBeginObject: d.skip() d.scanNext() case scanBeginLiteral: - v, err := d.literalInterface() - if err != nil { - return nil, err - } + v := d.literalInterface() switch v.(type) { case nil, string: - return v, nil + return v } } - return unquotedValue{}, nil + return unquotedValue{} } // indirect walks down v allocating pointers as needed, @@ -482,7 +473,7 @@ func indirect(v reflect.Value, decodingNull bool) (Unmarshaler, encoding.TextUnm if v.IsNil() { v.Set(reflect.New(v.Type().Elem())) } - if v.Type().NumMethod() > 0 { + if v.Type().NumMethod() > 0 && v.CanInterface() { if u, ok := v.Interface().(Unmarshaler); ok { return u, nil, reflect.Value{} } @@ -525,10 +516,7 @@ func (d *decodeState) array(v reflect.Value) error { case reflect.Interface: if v.NumMethod() == 0 { // Decoding into nil interface? Switch to non-reflect code. - ai, err := d.arrayInterface() - if err != nil { - return err - } + ai := d.arrayInterface() v.Set(reflect.ValueOf(ai)) return nil } @@ -538,8 +526,7 @@ func (d *decodeState) array(v reflect.Value) error { d.saveError(&UnmarshalTypeError{Value: "array", Type: v.Type(), Offset: int64(d.off)}) d.skip() return nil - case reflect.Array: - case reflect.Slice: + case reflect.Array, reflect.Slice: break } @@ -589,7 +576,7 @@ func (d *decodeState) array(v reflect.Value) error { break } if d.opcode != scanArrayValue { - return errPhase + panic(phasePanicMsg) } } @@ -611,7 +598,7 @@ func (d *decodeState) array(v reflect.Value) error { } var nullLiteral = []byte("null") -var textUnmarshalerType = reflect.TypeOf(new(encoding.TextUnmarshaler)).Elem() +var textUnmarshalerType = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem() // object consumes an object from d.data[d.off-1:], decoding into v. // The first byte ('{') of the object has been read already. @@ -629,17 +616,17 @@ func (d *decodeState) object(v reflect.Value) error { return nil } v = pv + t := v.Type() // Decoding into nil interface? Switch to non-reflect code. if v.Kind() == reflect.Interface && v.NumMethod() == 0 { - oi, err := d.objectInterface() - if err != nil { - return err - } + oi := d.objectInterface() v.Set(reflect.ValueOf(oi)) return nil } + var fields []field + // Check type of target: // struct or // map[T1]T2 where T1 is string, an integer type, @@ -648,14 +635,13 @@ func (d *decodeState) object(v reflect.Value) error { case reflect.Map: // Map key must either have string kind, have an integer kind, // or be an encoding.TextUnmarshaler. - t := v.Type() switch t.Key().Kind() { case reflect.String, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: default: if !reflect.PtrTo(t.Key()).Implements(textUnmarshalerType) { - d.saveError(&UnmarshalTypeError{Value: "object", Type: v.Type(), Offset: int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "object", Type: t, Offset: int64(d.off)}) d.skip() return nil } @@ -664,9 +650,10 @@ func (d *decodeState) object(v reflect.Value) error { v.Set(reflect.MakeMap(t)) } case reflect.Struct: + fields = cachedTypeFields(t) // ok default: - d.saveError(&UnmarshalTypeError{Value: "object", Type: v.Type(), Offset: int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "object", Type: t, Offset: int64(d.off)}) d.skip() return nil } @@ -682,7 +669,7 @@ func (d *decodeState) object(v reflect.Value) error { break } if d.opcode != scanBeginLiteral { - return errPhase + panic(phasePanicMsg) } // Read key. @@ -691,7 +678,7 @@ func (d *decodeState) object(v reflect.Value) error { item := d.data[start:d.readIndex()] key, ok := unquoteBytes(item) if !ok { - return errPhase + panic(phasePanicMsg) } // Figure out field corresponding to key. @@ -699,7 +686,7 @@ func (d *decodeState) object(v reflect.Value) error { destring := false // whether the value is wrapped in a string to be decoded first if v.Kind() == reflect.Map { - elemType := v.Type().Elem() + elemType := t.Elem() if !mapElem.IsValid() { mapElem = reflect.New(elemType).Elem() } else { @@ -708,7 +695,6 @@ func (d *decodeState) object(v reflect.Value) error { subv = mapElem } else { var f *field - fields := cachedTypeFields(v.Type()) for i := range fields { ff := &fields[i] if bytes.Equal(ff.nameBytes, key) { @@ -745,7 +731,7 @@ func (d *decodeState) object(v reflect.Value) error { subv = subv.Field(i) } d.errorContext.Field = f.name - d.errorContext.Struct = v.Type().Name() + d.errorContext.Struct = t } else if d.disallowUnknownFields { d.saveError(fmt.Errorf("json: unknown field %q", key)) } @@ -756,16 +742,12 @@ func (d *decodeState) object(v reflect.Value) error { d.scanWhile(scanSkipSpace) } if d.opcode != scanObjectKey { - return errPhase + panic(phasePanicMsg) } d.scanWhile(scanSkipSpace) if destring { - q, err := d.valueQuoted() - if err != nil { - return err - } - switch qv := q.(type) { + switch qv := d.valueQuoted().(type) { case nil: if err := d.literalStore(nullLiteral, subv, false); err != nil { return err @@ -786,13 +768,13 @@ func (d *decodeState) object(v reflect.Value) error { // Write value back to map; // if using struct, subv points into struct already. if v.Kind() == reflect.Map { - kt := v.Type().Key() + kt := t.Key() var kv reflect.Value switch { case kt.Kind() == reflect.String: kv = reflect.ValueOf(key).Convert(kt) case reflect.PtrTo(kt).Implements(textUnmarshalerType): - kv = reflect.New(v.Type().Key()) + kv = reflect.New(kt) if err := d.literalStore(item, kv, true); err != nil { return err } @@ -804,7 +786,7 @@ func (d *decodeState) object(v reflect.Value) error { n, err := strconv.ParseInt(s, 10, 64) if err != nil || reflect.Zero(kt).OverflowInt(n) { d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: kt, Offset: int64(start + 1)}) - return nil + break } kv = reflect.ValueOf(n).Convert(kt) case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: @@ -812,14 +794,16 @@ func (d *decodeState) object(v reflect.Value) error { n, err := strconv.ParseUint(s, 10, 64) if err != nil || reflect.Zero(kt).OverflowUint(n) { d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: kt, Offset: int64(start + 1)}) - return nil + break } kv = reflect.ValueOf(n).Convert(kt) default: panic("json: Unexpected key type") // should never occur } } - v.SetMapIndex(kv, subv) + if kv.IsValid() { + v.SetMapIndex(kv, subv) + } } // Next token must be , or }. @@ -830,7 +814,7 @@ func (d *decodeState) object(v reflect.Value) error { break } if d.opcode != scanObjectValue { - return errPhase + panic(phasePanicMsg) } d.errorContext = originalErrorContext @@ -874,18 +858,16 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if item[0] != '"' { if fromQuoted { d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) - } else { - var val string - switch item[0] { - case 'n': - val = "null" - case 't', 'f': - val = "bool" - default: - val = "number" - } - d.saveError(&UnmarshalTypeError{Value: val, Type: v.Type(), Offset: int64(d.readIndex())}) + return nil + } + val := "number" + switch item[0] { + case 'n': + val = "null" + case 't', 'f': + val = "bool" } + d.saveError(&UnmarshalTypeError{Value: val, Type: v.Type(), Offset: int64(d.readIndex())}) return nil } s, ok := unquoteBytes(item) @@ -893,7 +875,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if fromQuoted { return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) } - return errPhase + panic(phasePanicMsg) } return ut.UnmarshalText(s) } @@ -944,7 +926,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if fromQuoted { return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) } - return errPhase + panic(phasePanicMsg) } switch v.Kind() { default: @@ -976,7 +958,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if fromQuoted { return fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type()) } - return errPhase + panic(phasePanicMsg) } s := string(item) switch v.Kind() { @@ -1037,24 +1019,24 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool // but they avoid the weight of reflection in this common case. // valueInterface is like value but returns interface{} -func (d *decodeState) valueInterface() (val interface{}, err error) { +func (d *decodeState) valueInterface() (val interface{}) { switch d.opcode { default: - err = errPhase + panic(phasePanicMsg) case scanBeginArray: - val, err = d.arrayInterface() + val = d.arrayInterface() d.scanNext() case scanBeginObject: - val, err = d.objectInterface() + val = d.objectInterface() d.scanNext() case scanBeginLiteral: - val, err = d.literalInterface() + val = d.literalInterface() } return } // arrayInterface is like array but returns []interface{}. -func (d *decodeState) arrayInterface() ([]interface{}, error) { +func (d *decodeState) arrayInterface() []interface{} { var v = make([]interface{}, 0) for { // Look ahead for ] - can only happen on first iteration. @@ -1063,11 +1045,7 @@ func (d *decodeState) arrayInterface() ([]interface{}, error) { break } - vi, err := d.valueInterface() - if err != nil { - return nil, err - } - v = append(v, vi) + v = append(v, d.valueInterface()) // Next token must be , or ]. if d.opcode == scanSkipSpace { @@ -1077,14 +1055,14 @@ func (d *decodeState) arrayInterface() ([]interface{}, error) { break } if d.opcode != scanArrayValue { - return nil, errPhase + panic(phasePanicMsg) } } - return v, nil + return v } // objectInterface is like object but returns map[string]interface{}. -func (d *decodeState) objectInterface() (map[string]interface{}, error) { +func (d *decodeState) objectInterface() map[string]interface{} { m := make(map[string]interface{}) for { // Read opening " of string key or closing }. @@ -1094,7 +1072,7 @@ func (d *decodeState) objectInterface() (map[string]interface{}, error) { break } if d.opcode != scanBeginLiteral { - return nil, errPhase + panic(phasePanicMsg) } // Read string key. @@ -1103,7 +1081,7 @@ func (d *decodeState) objectInterface() (map[string]interface{}, error) { item := d.data[start:d.readIndex()] key, ok := unquote(item) if !ok { - return nil, errPhase + panic(phasePanicMsg) } // Read : before value. @@ -1111,16 +1089,12 @@ func (d *decodeState) objectInterface() (map[string]interface{}, error) { d.scanWhile(scanSkipSpace) } if d.opcode != scanObjectKey { - return nil, errPhase + panic(phasePanicMsg) } d.scanWhile(scanSkipSpace) // Read value. - vi, err := d.valueInterface() - if err != nil { - return nil, err - } - m[key] = vi + m[key] = d.valueInterface() // Next token must be , or }. if d.opcode == scanSkipSpace { @@ -1130,16 +1104,16 @@ func (d *decodeState) objectInterface() (map[string]interface{}, error) { break } if d.opcode != scanObjectValue { - return nil, errPhase + panic(phasePanicMsg) } } - return m, nil + return m } // literalInterface consumes and returns a literal from d.data[d.off-1:] and // it reads the following byte ahead. The first byte of the literal has been // read already (that's how the caller knows it's a literal). -func (d *decodeState) literalInterface() (interface{}, error) { +func (d *decodeState) literalInterface() interface{} { // All bytes inside literal return scanContinue op code. start := d.readIndex() d.scanWhile(scanContinue) @@ -1148,27 +1122,27 @@ func (d *decodeState) literalInterface() (interface{}, error) { switch c := item[0]; c { case 'n': // null - return nil, nil + return nil case 't', 'f': // true, false - return c == 't', nil + return c == 't' case '"': // string s, ok := unquote(item) if !ok { - return nil, errPhase + panic(phasePanicMsg) } - return s, nil + return s default: // number if c != '-' && (c < '0' || c > '9') { - return nil, errPhase + panic(phasePanicMsg) } n, err := d.convertNumber(string(item)) if err != nil { d.saveError(err) } - return n, nil + return n } } diff --git a/libgo/go/encoding/json/decode_test.go b/libgo/go/encoding/json/decode_test.go index 5746ddf9861..54432600a53 100644 --- a/libgo/go/encoding/json/decode_test.go +++ b/libgo/go/encoding/json/decode_test.go @@ -41,6 +41,16 @@ type VOuter struct { V V } +type W struct { + S SS +} + +type SS string + +func (*SS) UnmarshalJSON(data []byte) error { + return &UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(SS(""))} +} + // ifaceNumAsFloat64/ifaceNumAsNumber are used to test unmarshaling with and // without UseNumber var ifaceNumAsFloat64 = map[string]interface{}{ @@ -142,7 +152,7 @@ var ( umstructXY = ustructText{unmarshalerText{"x", "y"}} ummapType = map[unmarshalerText]bool{} - ummapXY = map[unmarshalerText]bool{unmarshalerText{"x", "y"}: true} + ummapXY = map[unmarshalerText]bool{{"x", "y"}: true} ) // Test data structures for anonymous fields. @@ -256,6 +266,10 @@ type XYZ struct { Z interface{} } +type unexportedWithMethods struct{} + +func (unexportedWithMethods) F() {} + func sliceAddr(x []int) *[]int { return &x } func mapAddr(x map[string]int) *map[string]int { return &x } @@ -408,6 +422,7 @@ var unmarshalTests = []unmarshalTest{ {in: `{"X": 23}`, ptr: new(T), out: T{}, err: &UnmarshalTypeError{"number", reflect.TypeOf(""), 8, "T", "X"}}, {in: `{"x": 1}`, ptr: new(tx), out: tx{}}, {in: `{"x": 1}`, ptr: new(tx), out: tx{}}, {in: `{"x": 1}`, ptr: new(tx), err: fmt.Errorf("json: unknown field \"x\""), disallowUnknownFields: true}, + {in: `{"S": 23}`, ptr: new(W), out: W{}, err: &UnmarshalTypeError{"number", reflect.TypeOf(SS("")), 0, "W", "S"}}, {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: float64(1), F2: int32(2), F3: Number("3")}}, {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: Number("1"), F2: int32(2), F3: Number("3")}, useNumber: true}, {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsFloat64}, @@ -434,6 +449,7 @@ var unmarshalTests = []unmarshalTest{ {in: `{"X": "foo", "Y"}`, err: &SyntaxError{"invalid character '}' after object key", 17}}, {in: `[1, 2, 3+]`, err: &SyntaxError{"invalid character '+' after array element", 9}}, {in: `{"X":12x}`, err: &SyntaxError{"invalid character 'x' after object key:value pair", 8}, useNumber: true}, + {in: `[2, 3`, err: &SyntaxError{msg: "unexpected end of JSON input", Offset: 5}}, // raw value errors {in: "\x01 42", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, @@ -449,6 +465,7 @@ var unmarshalTests = []unmarshalTest{ {in: `[1, 2, 3]`, ptr: new([3]int), out: [3]int{1, 2, 3}}, {in: `[1, 2, 3]`, ptr: new([1]int), out: [1]int{1}}, {in: `[1, 2, 3]`, ptr: new([5]int), out: [5]int{1, 2, 3, 0, 0}}, + {in: `[1, 2, 3]`, ptr: new(MustNotUnmarshalJSON), err: errors.New("MustNotUnmarshalJSON was used")}, // empty array to interface test {in: `[]`, ptr: new([]interface{}), out: []interface{}{}}, @@ -541,6 +558,16 @@ var unmarshalTests = []unmarshalTest{ ptr: new(map[uint8]string), err: &UnmarshalTypeError{Value: "number -1", Type: reflect.TypeOf(uint8(0)), Offset: 2}, }, + { + in: `{"F":{"a":2,"3":4}}`, + ptr: new(map[string]map[int]int), + err: &UnmarshalTypeError{Value: "number a", Type: reflect.TypeOf(int(0)), Offset: 7}, + }, + { + in: `{"F":{"a":2,"3":4}}`, + ptr: new(map[string]map[uint]int), + err: &UnmarshalTypeError{Value: "number a", Type: reflect.TypeOf(uint(0)), Offset: 7}, + }, // Map keys can be encoding.TextUnmarshalers. {in: `{"x:y":true}`, ptr: &ummapType, out: ummapXY}, @@ -815,6 +842,7 @@ var unmarshalTests = []unmarshalTest{ {in: `{"B": "False"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "False" into bool`)}, {in: `{"B": "null"}`, ptr: new(B), out: B{false}}, {in: `{"B": "nul"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "nul" into bool`)}, + {in: `{"B": [2, 3]}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal unquoted value into bool`)}, // additional tests for disallowUnknownFields { @@ -883,6 +911,18 @@ var unmarshalTests = []unmarshalTest{ ptr: new(mapStringToStringData), err: &UnmarshalTypeError{Value: "number", Type: reflect.TypeOf(""), Offset: 21, Struct: "mapStringToStringData", Field: "data"}, }, + + // trying to decode JSON arrays or objects via TextUnmarshaler + { + in: `[1, 2, 3]`, + ptr: new(MustNotUnmarshalText), + err: &UnmarshalTypeError{Value: "array", Type: reflect.TypeOf(&MustNotUnmarshalText{}), Offset: 1}, + }, + { + in: `{"foo": "bar"}`, + ptr: new(MustNotUnmarshalText), + err: &UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(&MustNotUnmarshalText{}), Offset: 1}, + }, } func TestMarshal(t *testing.T) { @@ -1944,10 +1984,12 @@ type unexportedFields struct { Name string m map[string]interface{} `json:"-"` m2 map[string]interface{} `json:"abcd"` + + s []int `json:"-"` } func TestUnmarshalUnexported(t *testing.T) { - input := `{"Name": "Bob", "m": {"x": 123}, "m2": {"y": 456}, "abcd": {"z": 789}}` + input := `{"Name": "Bob", "m": {"x": 123}, "m2": {"y": 456}, "abcd": {"z": 789}, "s": [2, 3]}` want := &unexportedFields{Name: "Bob"} out := &unexportedFields{} @@ -2113,6 +2155,9 @@ func TestInvalidStringOption(t *testing.T) { // // (Issue 24152) If the embedded struct is given an explicit name, // ensure that the normal unmarshal logic does not panic in reflect. +// +// (Issue 28145) If the embedded struct is given an explicit name and has +// exported methods, don't cause a panic trying to get its value. func TestUnmarshalEmbeddedUnexported(t *testing.T) { type ( embed1 struct{ Q int } @@ -2152,6 +2197,9 @@ func TestUnmarshalEmbeddedUnexported(t *testing.T) { embed2 `json:"embed2"` Q int } + S9 struct { + unexportedWithMethods `json:"embed"` + } ) tests := []struct { @@ -2213,6 +2261,11 @@ func TestUnmarshalEmbeddedUnexported(t *testing.T) { in: `{"embed1": {"Q": 1}, "embed2": {"Q": 2}, "Q": 3}`, ptr: new(S8), out: &S8{embed1{1}, embed2{2}, 3}, + }, { + // Issue 228145, similar to the cases above. + in: `{"embed": {}}`, + ptr: new(S9), + out: &S9{}, }} for i, tt := range tests { diff --git a/libgo/go/encoding/json/encode.go b/libgo/go/encoding/json/encode.go index 28ca5fe9e00..f10124e67d1 100644 --- a/libgo/go/encoding/json/encode.go +++ b/libgo/go/encoding/json/encode.go @@ -381,8 +381,8 @@ func typeEncoder(t reflect.Type) encoderFunc { } var ( - marshalerType = reflect.TypeOf(new(Marshaler)).Elem() - textMarshalerType = reflect.TypeOf(new(encoding.TextMarshaler)).Elem() + marshalerType = reflect.TypeOf((*Marshaler)(nil)).Elem() + textMarshalerType = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem() ) // newTypeEncoder constructs an encoderFunc for a type. @@ -624,40 +624,49 @@ func unsupportedTypeEncoder(e *encodeState, v reflect.Value, _ encOpts) { } type structEncoder struct { - fields []field - fieldEncs []encoderFunc + fields []field } -func (se *structEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { - e.WriteByte('{') - first := true - for i, f := range se.fields { - fv := fieldByIndex(v, f.index) - if !fv.IsValid() || f.omitEmpty && isEmptyValue(fv) { +func (se structEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { + next := byte('{') +FieldLoop: + for i := range se.fields { + f := &se.fields[i] + + // Find the nested struct field by following f.index. + fv := v + for _, i := range f.index { + if fv.Kind() == reflect.Ptr { + if fv.IsNil() { + continue FieldLoop + } + fv = fv.Elem() + } + fv = fv.Field(i) + } + + if f.omitEmpty && isEmptyValue(fv) { continue } - if first { - first = false + e.WriteByte(next) + next = ',' + if opts.escapeHTML { + e.WriteString(f.nameEscHTML) } else { - e.WriteByte(',') + e.WriteString(f.nameNonEsc) } - e.string(f.name, opts.escapeHTML) - e.WriteByte(':') opts.quoted = f.quoted - se.fieldEncs[i](e, fv, opts) + f.encoder(e, fv, opts) + } + if next == '{' { + e.WriteString("{}") + } else { + e.WriteByte('}') } - e.WriteByte('}') } func newStructEncoder(t reflect.Type) encoderFunc { - fields := cachedTypeFields(t) - se := &structEncoder{ - fields: fields, - fieldEncs: make([]encoderFunc, len(fields)), - } - for i, f := range fields { - se.fieldEncs[i] = typeEncoder(typeByIndex(t, f.index)) - } + se := structEncoder{fields: cachedTypeFields(t)} return se.encode } @@ -665,7 +674,7 @@ type mapEncoder struct { elemEnc encoderFunc } -func (me *mapEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { +func (me mapEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { if v.IsNil() { e.WriteString("null") return @@ -704,7 +713,7 @@ func newMapEncoder(t reflect.Type) encoderFunc { return unsupportedTypeEncoder } } - me := &mapEncoder{typeEncoder(t.Elem())} + me := mapEncoder{typeEncoder(t.Elem())} return me.encode } @@ -715,14 +724,22 @@ func encodeByteSlice(e *encodeState, v reflect.Value, _ encOpts) { } s := v.Bytes() e.WriteByte('"') - if len(s) < 1024 { - // for small buffers, using Encode directly is much faster. - dst := make([]byte, base64.StdEncoding.EncodedLen(len(s))) + encodedLen := base64.StdEncoding.EncodedLen(len(s)) + if encodedLen <= len(e.scratch) { + // If the encoded bytes fit in e.scratch, avoid an extra + // allocation and use the cheaper Encoding.Encode. + dst := e.scratch[:encodedLen] + base64.StdEncoding.Encode(dst, s) + e.Write(dst) + } else if encodedLen <= 1024 { + // The encoded bytes are short enough to allocate for, and + // Encoding.Encode is still cheaper. + dst := make([]byte, encodedLen) base64.StdEncoding.Encode(dst, s) e.Write(dst) } else { - // for large buffers, avoid unnecessary extra temporary - // buffer space. + // The encoded bytes are too long to cheaply allocate, and + // Encoding.Encode is no longer noticeably cheaper. enc := base64.NewEncoder(base64.StdEncoding, e) enc.Write(s) enc.Close() @@ -735,7 +752,7 @@ type sliceEncoder struct { arrayEnc encoderFunc } -func (se *sliceEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { +func (se sliceEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { if v.IsNil() { e.WriteString("null") return @@ -751,7 +768,7 @@ func newSliceEncoder(t reflect.Type) encoderFunc { return encodeByteSlice } } - enc := &sliceEncoder{newArrayEncoder(t)} + enc := sliceEncoder{newArrayEncoder(t)} return enc.encode } @@ -759,7 +776,7 @@ type arrayEncoder struct { elemEnc encoderFunc } -func (ae *arrayEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { +func (ae arrayEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { e.WriteByte('[') n := v.Len() for i := 0; i < n; i++ { @@ -772,7 +789,7 @@ func (ae *arrayEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { } func newArrayEncoder(t reflect.Type) encoderFunc { - enc := &arrayEncoder{typeEncoder(t.Elem())} + enc := arrayEncoder{typeEncoder(t.Elem())} return enc.encode } @@ -780,7 +797,7 @@ type ptrEncoder struct { elemEnc encoderFunc } -func (pe *ptrEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { +func (pe ptrEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { if v.IsNil() { e.WriteString("null") return @@ -789,7 +806,7 @@ func (pe *ptrEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { } func newPtrEncoder(t reflect.Type) encoderFunc { - enc := &ptrEncoder{typeEncoder(t.Elem())} + enc := ptrEncoder{typeEncoder(t.Elem())} return enc.encode } @@ -797,7 +814,7 @@ type condAddrEncoder struct { canAddrEnc, elseEnc encoderFunc } -func (ce *condAddrEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { +func (ce condAddrEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { if v.CanAddr() { ce.canAddrEnc(e, v, opts) } else { @@ -808,7 +825,7 @@ func (ce *condAddrEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) // newCondAddrEncoder returns an encoder that checks whether its value // CanAddr and delegates to canAddrEnc if so, else to elseEnc. func newCondAddrEncoder(canAddrEnc, elseEnc encoderFunc) encoderFunc { - enc := &condAddrEncoder{canAddrEnc: canAddrEnc, elseEnc: elseEnc} + enc := condAddrEncoder{canAddrEnc: canAddrEnc, elseEnc: elseEnc} return enc.encode } @@ -822,28 +839,13 @@ func isValidTag(s string) bool { // Backslash and quote chars are reserved, but // otherwise any punctuation chars are allowed // in a tag name. - default: - if !unicode.IsLetter(c) && !unicode.IsDigit(c) { - return false - } + case !unicode.IsLetter(c) && !unicode.IsDigit(c): + return false } } return true } -func fieldByIndex(v reflect.Value, index []int) reflect.Value { - for _, i := range index { - if v.Kind() == reflect.Ptr { - if v.IsNil() { - return reflect.Value{} - } - v = v.Elem() - } - v = v.Field(i) - } - return v -} - func typeByIndex(t reflect.Type, index []int) reflect.Type { for _, i := range index { if t.Kind() == reflect.Ptr { @@ -893,18 +895,15 @@ func (e *encodeState) string(s string, escapeHTML bool) { if start < i { e.WriteString(s[start:i]) } + e.WriteByte('\\') switch b { case '\\', '"': - e.WriteByte('\\') e.WriteByte(b) case '\n': - e.WriteByte('\\') e.WriteByte('n') case '\r': - e.WriteByte('\\') e.WriteByte('r') case '\t': - e.WriteByte('\\') e.WriteByte('t') default: // This encodes bytes < 0x20 except for \t, \n and \r. @@ -912,7 +911,7 @@ func (e *encodeState) string(s string, escapeHTML bool) { // because they can lead to security holes when // user-controlled strings are rendered into JSON // and served to some browsers. - e.WriteString(`\u00`) + e.WriteString(`u00`) e.WriteByte(hex[b>>4]) e.WriteByte(hex[b&0xF]) } @@ -968,18 +967,15 @@ func (e *encodeState) stringBytes(s []byte, escapeHTML bool) { if start < i { e.Write(s[start:i]) } + e.WriteByte('\\') switch b { case '\\', '"': - e.WriteByte('\\') e.WriteByte(b) case '\n': - e.WriteByte('\\') e.WriteByte('n') case '\r': - e.WriteByte('\\') e.WriteByte('r') case '\t': - e.WriteByte('\\') e.WriteByte('t') default: // This encodes bytes < 0x20 except for \t, \n and \r. @@ -987,7 +983,7 @@ func (e *encodeState) stringBytes(s []byte, escapeHTML bool) { // because they can lead to security holes when // user-controlled strings are rendered into JSON // and served to some browsers. - e.WriteString(`\u00`) + e.WriteString(`u00`) e.WriteByte(hex[b>>4]) e.WriteByte(hex[b&0xF]) } @@ -1036,17 +1032,16 @@ type field struct { nameBytes []byte // []byte(name) equalFold func(s, t []byte) bool // bytes.EqualFold or equivalent + nameNonEsc string // `"` + name + `":` + nameEscHTML string // `"` + HTMLEscape(name) + `":` + tag bool index []int typ reflect.Type omitEmpty bool quoted bool -} -func fillField(f field) field { - f.nameBytes = []byte(f.name) - f.equalFold = foldFunc(f.nameBytes) - return f + encoder encoderFunc } // byIndex sorts field by index sequence. @@ -1086,6 +1081,9 @@ func typeFields(t reflect.Type) []field { // Fields found. var fields []field + // Buffer to run HTMLEscape on field names. + var nameEscBuf bytes.Buffer + for len(next) > 0 { current, next = next, current[:0] count, nextCount = nextCount, map[reflect.Type]int{} @@ -1152,14 +1150,26 @@ func typeFields(t reflect.Type) []field { if name == "" { name = sf.Name } - fields = append(fields, fillField(field{ + field := field{ name: name, tag: tagged, index: index, typ: ft, omitEmpty: opts.Contains("omitempty"), quoted: quoted, - })) + } + field.nameBytes = []byte(field.name) + field.equalFold = foldFunc(field.nameBytes) + + // Build nameEscHTML and nameNonEsc ahead of time. + nameEscBuf.Reset() + nameEscBuf.WriteString(`"`) + HTMLEscape(&nameEscBuf, field.nameBytes) + nameEscBuf.WriteString(`":`) + field.nameEscHTML = nameEscBuf.String() + field.nameNonEsc = `"` + field.name + `":` + + fields = append(fields, field) if count[f.typ] > 1 { // If there were multiple instances, add a second, // so that the annihilation code will see a duplicate. @@ -1173,7 +1183,7 @@ func typeFields(t reflect.Type) []field { // Record new anonymous struct to explore in next round. nextCount[ft]++ if nextCount[ft] == 1 { - next = append(next, fillField(field{name: ft.Name(), index: index, typ: ft})) + next = append(next, field{name: ft.Name(), index: index, typ: ft}) } } } @@ -1227,6 +1237,10 @@ func typeFields(t reflect.Type) []field { fields = out sort.Sort(byIndex(fields)) + for i := range fields { + f := &fields[i] + f.encoder = typeEncoder(typeByIndex(t, f.index)) + } return fields } diff --git a/libgo/go/encoding/json/encode_test.go b/libgo/go/encoding/json/encode_test.go index b90483cf35f..cd5eadf3c1c 100644 --- a/libgo/go/encoding/json/encode_test.go +++ b/libgo/go/encoding/json/encode_test.go @@ -405,6 +405,19 @@ func TestAnonymousFields(t *testing.T) { return S{s1{1, 2, s2{3, 4}}, 6} }, want: `{"MyInt1":1,"MyInt2":3}`, + }, { + // If an anonymous struct pointer field is nil, we should ignore + // the embedded fields behind it. Not properly doing so may + // result in the wrong output or reflect panics. + label: "EmbeddedFieldBehindNilPointer", + makeInput: func() interface{} { + type ( + S2 struct{ Field string } + S struct{ *S2 } + ) + return S{} + }, + want: `{}`, }} for _, tt := range tests { @@ -995,3 +1008,18 @@ func TestMarshalPanic(t *testing.T) { Marshal(&marshalPanic{}) t.Error("Marshal should have panicked") } + +func TestMarshalUncommonFieldNames(t *testing.T) { + v := struct { + A0, À, Aβ int + }{} + b, err := Marshal(v) + if err != nil { + t.Fatal("Marshal:", err) + } + want := `{"A0":0,"À":0,"Aβ":0}` + got := string(b) + if got != want { + t.Fatalf("Marshal: got %s want %s", got, want) + } +} diff --git a/libgo/go/encoding/json/example_text_marshaling_test.go b/libgo/go/encoding/json/example_text_marshaling_test.go new file mode 100644 index 00000000000..a8a3f830f68 --- /dev/null +++ b/libgo/go/encoding/json/example_text_marshaling_test.go @@ -0,0 +1,69 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package json_test + +import ( + "encoding/json" + "fmt" + "log" + "strings" +) + +type Size int + +const ( + Unrecognized Size = iota + Small + Large +) + +func (s *Size) UnmarshalText(text []byte) error { + switch strings.ToLower(string(text)) { + default: + *s = Unrecognized + case "small": + *s = Small + case "large": + *s = Large + } + return nil +} + +func (s Size) MarshalText() ([]byte, error) { + var name string + switch s { + default: + name = "unrecognized" + case Small: + name = "small" + case Large: + name = "large" + } + return []byte(name), nil +} + +func Example_textMarshalJSON() { + blob := `["small","regular","large","unrecognized","small","normal","small","large"]` + var inventory []Size + if err := json.Unmarshal([]byte(blob), &inventory); err != nil { + log.Fatal(err) + } + + counts := make(map[Size]int) + for _, size := range inventory { + counts[size] += 1 + } + + fmt.Printf("Inventory Counts:\n* Small: %d\n* Large: %d\n* Unrecognized: %d\n", + counts[Small], counts[Large], counts[Unrecognized]) + + // Output: + // Inventory Counts: + // * Small: 3 + // * Large: 2 + // * Unrecognized: 3 +} diff --git a/libgo/go/encoding/json/scanner.go b/libgo/go/encoding/json/scanner.go index 9e6d482e168..88572245fc7 100644 --- a/libgo/go/encoding/json/scanner.go +++ b/libgo/go/encoding/json/scanner.go @@ -289,7 +289,7 @@ func stateEndValue(s *scanner, c byte) int { // such as after reading `{}` or `[1,2,3]`. // Only space characters should be seen now. func stateEndTop(s *scanner, c byte) int { - if c != ' ' && c != '\t' && c != '\r' && c != '\n' { + if !isSpace(c) { // Complain about non-space byte on next call. s.error(c, "after top-level value") } diff --git a/libgo/go/encoding/json/stream.go b/libgo/go/encoding/json/stream.go index 75a4270df7d..7d5137fbc71 100644 --- a/libgo/go/encoding/json/stream.go +++ b/libgo/go/encoding/json/stream.go @@ -96,19 +96,19 @@ Input: // Look in the buffer for a new value. for i, c := range dec.buf[scanp:] { dec.scan.bytes++ - v := dec.scan.step(&dec.scan, c) - if v == scanEnd { + switch dec.scan.step(&dec.scan, c) { + case scanEnd: scanp += i break Input - } - // scanEnd is delayed one byte. - // We might block trying to get that byte from src, - // so instead invent a space byte. - if (v == scanEndObject || v == scanEndArray) && dec.scan.step(&dec.scan, ' ') == scanEnd { - scanp += i + 1 - break Input - } - if v == scanError { + case scanEndObject, scanEndArray: + // scanEnd is delayed one byte. + // We might block trying to get that byte from src, + // so instead invent a space byte. + if stateEndValue(&dec.scan, ' ') == scanEnd { + scanp += i + 1 + break Input + } + case scanError: dec.err = dec.scan.err return 0, dec.scan.err } @@ -471,7 +471,7 @@ func (dec *Decoder) tokenError(c byte) (Token, error) { case tokenObjectComma: context = " after object key:value pair" } - return nil, &SyntaxError{"invalid character " + quoteChar(c) + " " + context, dec.offset()} + return nil, &SyntaxError{"invalid character " + quoteChar(c) + context, dec.offset()} } // More reports whether there is another element in the diff --git a/libgo/go/encoding/json/stream_test.go b/libgo/go/encoding/json/stream_test.go index 83c01d170c0..aaf32e0a24c 100644 --- a/libgo/go/encoding/json/stream_test.go +++ b/libgo/go/encoding/json/stream_test.go @@ -93,6 +93,10 @@ func TestEncoderIndent(t *testing.T) { func TestEncoderSetEscapeHTML(t *testing.T) { var c C var ct CText + var tagStruct struct { + Valid int `json:"<>&#! "` + Invalid int `json:"\\"` + } for _, tt := range []struct { name string v interface{} @@ -102,6 +106,11 @@ func TestEncoderSetEscapeHTML(t *testing.T) { {"c", c, `"\u003c\u0026\u003e"`, `"<&>"`}, {"ct", ct, `"\"\u003c\u0026\u003e\""`, `"\"<&>\""`}, {`"<&>"`, "<&>", `"\u003c\u0026\u003e"`, `"<&>"`}, + { + "tagStruct", tagStruct, + `{"\u003c\u003e\u0026#! ":0,"Invalid":0}`, + `{"<>&#! ":0,"Invalid":0}`, + }, } { var buf bytes.Buffer enc := NewEncoder(&buf) @@ -192,10 +201,9 @@ func nlines(s string, n int) string { } func TestRawMessage(t *testing.T) { - // TODO(rsc): Should not need the * in *RawMessage var data struct { X float64 - Id *RawMessage + Id RawMessage Y float32 } const raw = `["\u0056",null]` @@ -204,8 +212,8 @@ func TestRawMessage(t *testing.T) { if err != nil { t.Fatalf("Unmarshal: %v", err) } - if string([]byte(*data.Id)) != raw { - t.Fatalf("Raw mismatch: have %#q want %#q", []byte(*data.Id), raw) + if string([]byte(data.Id)) != raw { + t.Fatalf("Raw mismatch: have %#q want %#q", []byte(data.Id), raw) } b, err := Marshal(&data) if err != nil { @@ -217,20 +225,22 @@ func TestRawMessage(t *testing.T) { } func TestNullRawMessage(t *testing.T) { - // TODO(rsc): Should not need the * in *RawMessage var data struct { - X float64 - Id *RawMessage - Y float32 + X float64 + Id RawMessage + IdPtr *RawMessage + Y float32 } - data.Id = new(RawMessage) - const msg = `{"X":0.1,"Id":null,"Y":0.2}` + const msg = `{"X":0.1,"Id":null,"IdPtr":null,"Y":0.2}` err := Unmarshal([]byte(msg), &data) if err != nil { t.Fatalf("Unmarshal: %v", err) } - if data.Id != nil { - t.Fatalf("Raw mismatch: have non-nil, want nil") + if want, got := "null", string(data.Id); want != got { + t.Fatalf("Raw mismatch: have %q, want %q", got, want) + } + if data.IdPtr != nil { + t.Fatalf("Raw pointer mismatch: have non-nil, want nil") } b, err := Marshal(&data) if err != nil { |