diff options
Diffstat (limited to 'libgo/go/encoding/gob')
-rw-r--r-- | libgo/go/encoding/gob/codec_test.go | 74 | ||||
-rw-r--r-- | libgo/go/encoding/gob/decode.go | 49 | ||||
-rw-r--r-- | libgo/go/encoding/gob/decoder.go | 4 | ||||
-rw-r--r-- | libgo/go/encoding/gob/doc.go | 90 | ||||
-rw-r--r-- | libgo/go/encoding/gob/encode.go | 30 | ||||
-rw-r--r-- | libgo/go/encoding/gob/encoder_test.go | 65 | ||||
-rw-r--r-- | libgo/go/encoding/gob/error.go | 1 | ||||
-rw-r--r-- | libgo/go/encoding/gob/gobencdec_test.go | 2 | ||||
-rw-r--r-- | libgo/go/encoding/gob/timing_test.go | 188 | ||||
-rw-r--r-- | libgo/go/encoding/gob/type.go | 56 | ||||
-rw-r--r-- | libgo/go/encoding/gob/type_test.go | 6 |
11 files changed, 301 insertions, 264 deletions
diff --git a/libgo/go/encoding/gob/codec_test.go b/libgo/go/encoding/gob/codec_test.go index d4002cbccab..eb9f306bcf5 100644 --- a/libgo/go/encoding/gob/codec_test.go +++ b/libgo/go/encoding/gob/codec_test.go @@ -47,7 +47,6 @@ func testError(t *testing.T) { if e := recover(); e != nil { t.Error(e.(gobError).err) // Will re-panic if not one of our errors, such as a runtime error. } - return } func newDecBuffer(data []byte) *decBuffer { @@ -321,7 +320,7 @@ func TestScalarEncInstructions(t *testing.T) { } } -func execDec(typ string, instr *decInstr, state *decoderState, t *testing.T, value reflect.Value) { +func execDec(instr *decInstr, state *decoderState, t *testing.T, value reflect.Value) { defer testError(t) v := int(state.decodeUint()) if v+state.fieldnum != 6 { @@ -348,7 +347,7 @@ func TestScalarDecInstructions(t *testing.T) { var data bool instr := &decInstr{decBool, 6, nil, ovfl} state := newDecodeStateFromData(boolResult) - execDec("bool", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != true { t.Errorf("bool a = %v not true", data) } @@ -358,7 +357,7 @@ func TestScalarDecInstructions(t *testing.T) { var data int instr := &decInstr{decOpTable[reflect.Int], 6, nil, ovfl} state := newDecodeStateFromData(signedResult) - execDec("int", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("int a = %v not 17", data) } @@ -369,7 +368,7 @@ func TestScalarDecInstructions(t *testing.T) { var data uint instr := &decInstr{decOpTable[reflect.Uint], 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uint", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("uint a = %v not 17", data) } @@ -380,7 +379,7 @@ func TestScalarDecInstructions(t *testing.T) { var data int8 instr := &decInstr{decInt8, 6, nil, ovfl} state := newDecodeStateFromData(signedResult) - execDec("int8", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("int8 a = %v not 17", data) } @@ -391,7 +390,7 @@ func TestScalarDecInstructions(t *testing.T) { var data uint8 instr := &decInstr{decUint8, 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uint8", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("uint8 a = %v not 17", data) } @@ -402,7 +401,7 @@ func TestScalarDecInstructions(t *testing.T) { var data int16 instr := &decInstr{decInt16, 6, nil, ovfl} state := newDecodeStateFromData(signedResult) - execDec("int16", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("int16 a = %v not 17", data) } @@ -413,7 +412,7 @@ func TestScalarDecInstructions(t *testing.T) { var data uint16 instr := &decInstr{decUint16, 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uint16", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("uint16 a = %v not 17", data) } @@ -424,7 +423,7 @@ func TestScalarDecInstructions(t *testing.T) { var data int32 instr := &decInstr{decInt32, 6, nil, ovfl} state := newDecodeStateFromData(signedResult) - execDec("int32", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("int32 a = %v not 17", data) } @@ -435,7 +434,7 @@ func TestScalarDecInstructions(t *testing.T) { var data uint32 instr := &decInstr{decUint32, 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uint32", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("uint32 a = %v not 17", data) } @@ -446,7 +445,7 @@ func TestScalarDecInstructions(t *testing.T) { var data uintptr instr := &decInstr{decOpTable[reflect.Uintptr], 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uintptr", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("uintptr a = %v not 17", data) } @@ -457,7 +456,7 @@ func TestScalarDecInstructions(t *testing.T) { var data int64 instr := &decInstr{decInt64, 6, nil, ovfl} state := newDecodeStateFromData(signedResult) - execDec("int64", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("int64 a = %v not 17", data) } @@ -468,7 +467,7 @@ func TestScalarDecInstructions(t *testing.T) { var data uint64 instr := &decInstr{decUint64, 6, nil, ovfl} state := newDecodeStateFromData(unsignedResult) - execDec("uint64", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("uint64 a = %v not 17", data) } @@ -479,7 +478,7 @@ func TestScalarDecInstructions(t *testing.T) { var data float32 instr := &decInstr{decFloat32, 6, nil, ovfl} state := newDecodeStateFromData(floatResult) - execDec("float32", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("float32 a = %v not 17", data) } @@ -490,7 +489,7 @@ func TestScalarDecInstructions(t *testing.T) { var data float64 instr := &decInstr{decFloat64, 6, nil, ovfl} state := newDecodeStateFromData(floatResult) - execDec("float64", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17 { t.Errorf("float64 a = %v not 17", data) } @@ -501,7 +500,7 @@ func TestScalarDecInstructions(t *testing.T) { var data complex64 instr := &decInstr{decOpTable[reflect.Complex64], 6, nil, ovfl} state := newDecodeStateFromData(complexResult) - execDec("complex", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17+19i { t.Errorf("complex a = %v not 17+19i", data) } @@ -512,7 +511,7 @@ func TestScalarDecInstructions(t *testing.T) { var data complex128 instr := &decInstr{decOpTable[reflect.Complex128], 6, nil, ovfl} state := newDecodeStateFromData(complexResult) - execDec("complex", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != 17+19i { t.Errorf("complex a = %v not 17+19i", data) } @@ -523,7 +522,7 @@ func TestScalarDecInstructions(t *testing.T) { var data []byte instr := &decInstr{decUint8Slice, 6, nil, ovfl} state := newDecodeStateFromData(bytesResult) - execDec("bytes", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if string(data) != "hello" { t.Errorf(`bytes a = %q not "hello"`, string(data)) } @@ -534,7 +533,7 @@ func TestScalarDecInstructions(t *testing.T) { var data string instr := &decInstr{decString, 6, nil, ovfl} state := newDecodeStateFromData(bytesResult) - execDec("bytes", instr, state, t, reflect.ValueOf(&data)) + execDec(instr, state, t, reflect.ValueOf(&data)) if data != "hello" { t.Errorf(`bytes a = %q not "hello"`, data) } @@ -545,11 +544,18 @@ func TestEndToEnd(t *testing.T) { type T2 struct { T string } - s1 := "string1" - s2 := "string2" + type T3 struct { + X float64 + Z *int + } type T1 struct { A, B, C int M map[string]*float64 + M2 map[int]T3 + Mstring map[string]string + Mintptr map[int]*int + Mcomp map[complex128]complex128 + Marr map[[2]string][2]*float64 EmptyMap map[string]int // to check that we receive a non-nil map. N *[3]float64 Strs *[2]string @@ -561,11 +567,35 @@ func TestEndToEnd(t *testing.T) { } pi := 3.14159 e := 2.71828 + two := 2.0 + meaning := 42 + fingers := 5 + s1 := "string1" + s2 := "string2" + var comp1 complex128 = complex(1.0, 1.0) + var comp2 complex128 = complex(1.0, 1.0) + var arr1 [2]string + arr1[0] = s1 + arr1[1] = s2 + var arr2 [2]string + arr2[0] = s2 + arr2[1] = s1 + var floatArr1 [2]*float64 + floatArr1[0] = &pi + floatArr1[1] = &e + var floatArr2 [2]*float64 + floatArr2[0] = &e + floatArr2[1] = &two t1 := &T1{ A: 17, B: 18, C: -5, M: map[string]*float64{"pi": &pi, "e": &e}, + M2: map[int]T3{4: T3{X: pi, Z: &meaning}, 10: T3{X: e, Z: &fingers}}, + Mstring: map[string]string{"pi": "3.14", "e": "2.71"}, + Mintptr: map[int]*int{meaning: &fingers, fingers: &meaning}, + Mcomp: map[complex128]complex128{comp1: comp2, comp2: comp1}, + Marr: map[[2]string][2]*float64{arr1: floatArr1, arr2: floatArr2}, EmptyMap: make(map[string]int), N: &[3]float64{1.5, 2.5, 3.5}, Strs: &[2]string{s1, s2}, diff --git a/libgo/go/encoding/gob/decode.go b/libgo/go/encoding/gob/decode.go index 9645dc57906..8dece42e908 100644 --- a/libgo/go/encoding/gob/decode.go +++ b/libgo/go/encoding/gob/decode.go @@ -11,6 +11,7 @@ import ( "errors" "io" "math" + "math/bits" "reflect" ) @@ -313,12 +314,7 @@ func decUint64(i *decInstr, state *decoderState, value reflect.Value) { // (for example) transmit more compactly. This routine does the // unswizzling. func float64FromBits(u uint64) float64 { - var v uint64 - for i := 0; i < 8; i++ { - v <<= 8 - v |= u & 0xFF - u >>= 8 - } + v := bits.ReverseBytes64(u) return math.Float64frombits(v) } @@ -430,7 +426,7 @@ type decEngine struct { // decodeSingle decodes a top-level value that is not a struct and stores it in value. // Such values are preceded by a zero, making them have the memory layout of a // struct field (although with an illegal field number). -func (dec *Decoder) decodeSingle(engine *decEngine, ut *userTypeInfo, value reflect.Value) { +func (dec *Decoder) decodeSingle(engine *decEngine, value reflect.Value) { state := dec.newDecoderState(&dec.buf) defer dec.freeDecoderState(state) state.fieldnum = singletonField @@ -446,7 +442,7 @@ func (dec *Decoder) decodeSingle(engine *decEngine, ut *userTypeInfo, value refl // differ from ut.indir, which was computed when the engine was built. // This state cannot arise for decodeSingle, which is called directly // from the user's value, not from the innards of an engine. -func (dec *Decoder) decodeStruct(engine *decEngine, ut *userTypeInfo, value reflect.Value) { +func (dec *Decoder) decodeStruct(engine *decEngine, value reflect.Value) { state := dec.newDecoderState(&dec.buf) defer dec.freeDecoderState(state) state.fieldnum = -1 @@ -538,7 +534,7 @@ func (dec *Decoder) decodeArrayHelper(state *decoderState, value reflect.Value, // decodeArray decodes an array and stores it in value. // The length is an unsigned integer preceding the elements. Even though the length is redundant // (it's part of the type), it's a useful check and is included in the encoding. -func (dec *Decoder) decodeArray(atyp reflect.Type, state *decoderState, value reflect.Value, elemOp decOp, length int, ovfl error, helper decHelper) { +func (dec *Decoder) decodeArray(state *decoderState, value reflect.Value, elemOp decOp, length int, ovfl error, helper decHelper) { if n := state.decodeUint(); n != uint64(length) { errorf("length mismatch in decodeArray") } @@ -546,12 +542,12 @@ func (dec *Decoder) decodeArray(atyp reflect.Type, state *decoderState, value re } // decodeIntoValue is a helper for map decoding. -func decodeIntoValue(state *decoderState, op decOp, isPtr bool, value reflect.Value, ovfl error) reflect.Value { - instr := &decInstr{op, 0, nil, ovfl} +func decodeIntoValue(state *decoderState, op decOp, isPtr bool, value reflect.Value, instr *decInstr) reflect.Value { v := value if isPtr { v = decAlloc(value) } + op(instr, state, v) return value } @@ -561,17 +557,24 @@ func decodeIntoValue(state *decoderState, op decOp, isPtr bool, value reflect.Va // Because the internals of maps are not visible to us, we must // use reflection rather than pointer magic. func (dec *Decoder) decodeMap(mtyp reflect.Type, state *decoderState, value reflect.Value, keyOp, elemOp decOp, ovfl error) { + n := int(state.decodeUint()) if value.IsNil() { - // Allocate map. - value.Set(reflect.MakeMap(mtyp)) + value.Set(reflect.MakeMapWithSize(mtyp, n)) } - n := int(state.decodeUint()) keyIsPtr := mtyp.Key().Kind() == reflect.Ptr elemIsPtr := mtyp.Elem().Kind() == reflect.Ptr + keyInstr := &decInstr{keyOp, 0, nil, ovfl} + elemInstr := &decInstr{elemOp, 0, nil, ovfl} + keyP := reflect.New(mtyp.Key()) + keyZ := reflect.Zero(mtyp.Key()) + elemP := reflect.New(mtyp.Elem()) + elemZ := reflect.Zero(mtyp.Elem()) for i := 0; i < n; i++ { - key := decodeIntoValue(state, keyOp, keyIsPtr, allocValue(mtyp.Key()), ovfl) - elem := decodeIntoValue(state, elemOp, elemIsPtr, allocValue(mtyp.Elem()), ovfl) + key := decodeIntoValue(state, keyOp, keyIsPtr, keyP.Elem(), keyInstr) + elem := decodeIntoValue(state, elemOp, elemIsPtr, elemP.Elem(), elemInstr) value.SetMapIndex(key, elem) + keyP.Elem().Set(keyZ) + elemP.Elem().Set(elemZ) } } @@ -657,12 +660,12 @@ func (dec *Decoder) decodeInterface(ityp reflect.Type, state *decoderState, valu errorf("name too long (%d bytes): %.20q...", len(name), name) } // The concrete type must be registered. - registerLock.RLock() - typ, ok := nameToConcreteType[string(name)] - registerLock.RUnlock() + typi, ok := nameToConcreteType.Load(string(name)) if !ok { errorf("name not registered for interface: %q", name) } + typ := typi.(reflect.Type) + // Read the type id of the concrete value. concreteId := dec.decodeTypeSequence(true) if concreteId < 0 { @@ -813,7 +816,7 @@ func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string, inProg ovfl := overflow(name) helper := decArrayHelper[t.Elem().Kind()] op = func(i *decInstr, state *decoderState, value reflect.Value) { - state.dec.decodeArray(t, state, value, *elemOp, t.Len(), ovfl, helper) + state.dec.decodeArray(state, value, *elemOp, t.Len(), ovfl, helper) } case reflect.Map: @@ -854,7 +857,7 @@ func (dec *Decoder) decOpFor(wireId typeId, rt reflect.Type, name string, inProg } op = func(i *decInstr, state *decoderState, value reflect.Value) { // indirect through enginePtr to delay evaluation for recursive structs. - dec.decodeStruct(*enginePtr, ut, value) + dec.decodeStruct(*enginePtr, value) } case reflect.Interface: op = func(i *decInstr, state *decoderState, value reflect.Value) { @@ -1197,9 +1200,9 @@ func (dec *Decoder) decodeValue(wireId typeId, value reflect.Value) { name := base.Name() errorf("type mismatch: no fields matched compiling decoder for %s", name) } - dec.decodeStruct(engine, ut, value) + dec.decodeStruct(engine, value) } else { - dec.decodeSingle(engine, ut, value) + dec.decodeSingle(engine, value) } } diff --git a/libgo/go/encoding/gob/decoder.go b/libgo/go/encoding/gob/decoder.go index c1829417733..8e0b1dd3750 100644 --- a/libgo/go/encoding/gob/decoder.go +++ b/libgo/go/encoding/gob/decoder.go @@ -19,6 +19,10 @@ const tooBig = 1 << 30 // A Decoder manages the receipt of type and data information read from the // remote side of a connection. +// +// The Decoder does only basic sanity checking on decoded input sizes, +// and its limits are not configurable. Take caution when decoding gob data +// from untrusted sources. type Decoder struct { mutex sync.Mutex // each item must be received atomically r io.Reader // source of the data diff --git a/libgo/go/encoding/gob/doc.go b/libgo/go/encoding/gob/doc.go index 1536574fed2..db734ecc1ec 100644 --- a/libgo/go/encoding/gob/doc.go +++ b/libgo/go/encoding/gob/doc.go @@ -4,7 +4,7 @@ /* Package gob manages streams of gobs - binary values exchanged between an -Encoder (transmitter) and a Decoder (receiver). A typical use is transporting +Encoder (transmitter) and a Decoder (receiver). A typical use is transporting arguments and results of remote procedure calls (RPCs) such as those provided by package "net/rpc". @@ -14,28 +14,28 @@ amortizing the cost of compilation. Basics -A stream of gobs is self-describing. Each data item in the stream is preceded by +A stream of gobs is self-describing. Each data item in the stream is preceded by a specification of its type, expressed in terms of a small set of predefined -types. Pointers are not transmitted, but the things they point to are +types. Pointers are not transmitted, but the things they point to are transmitted; that is, the values are flattened. Nil pointers are not permitted, as they have no value. Recursive types work fine, but -recursive values (data with cycles) are problematic. This may change. +recursive values (data with cycles) are problematic. This may change. To use gobs, create an Encoder and present it with a series of data items as -values or addresses that can be dereferenced to values. The Encoder makes sure -all type information is sent before it is needed. At the receive side, a +values or addresses that can be dereferenced to values. The Encoder makes sure +all type information is sent before it is needed. At the receive side, a Decoder retrieves values from the encoded stream and unpacks them into local variables. Types and Values -The source and destination values/types need not correspond exactly. For structs, +The source and destination values/types need not correspond exactly. For structs, fields (identified by name) that are in the source but absent from the receiving -variable will be ignored. Fields that are in the receiving variable but missing -from the transmitted type or value will be ignored in the destination. If a field +variable will be ignored. Fields that are in the receiving variable but missing +from the transmitted type or value will be ignored in the destination. If a field with the same name is present in both, their types must be compatible. Both the receiver and transmitter will do all necessary indirection and dereferencing to -convert between gobs and actual Go values. For instance, a gob type that is +convert between gobs and actual Go values. For instance, a gob type that is schematically, struct { A, B int } @@ -63,8 +63,8 @@ Attempting to receive into these types will draw a decode error: struct { C, D int } // no field names in common Integers are transmitted two ways: arbitrary precision signed integers or -arbitrary precision unsigned integers. There is no int8, int16 etc. -discrimination in the gob format; there are only signed and unsigned integers. As +arbitrary precision unsigned integers. There is no int8, int16 etc. +discrimination in the gob format; there are only signed and unsigned integers. As described below, the transmitter sends the value in a variable-length encoding; the receiver accepts the value and stores it in the destination variable. Floating-point numbers are always sent using IEEE-754 64-bit precision (see @@ -72,7 +72,7 @@ below). Signed integers may be received into any signed integer variable: int, int16, etc.; unsigned integers may be received into any unsigned integer variable; and floating -point values may be received into any floating point variable. However, +point values may be received into any floating point variable. However, the destination variable must be able to represent the value or the decode operation will fail. @@ -106,17 +106,17 @@ Encoding Details This section documents the encoding, details that are not important for most users. Details are presented bottom-up. -An unsigned integer is sent one of two ways. If it is less than 128, it is sent -as a byte with that value. Otherwise it is sent as a minimal-length big-endian +An unsigned integer is sent one of two ways. If it is less than 128, it is sent +as a byte with that value. Otherwise it is sent as a minimal-length big-endian (high byte first) byte stream holding the value, preceded by one byte holding the -byte count, negated. Thus 0 is transmitted as (00), 7 is transmitted as (07) and +byte count, negated. Thus 0 is transmitted as (00), 7 is transmitted as (07) and 256 is transmitted as (FE 01 00). A boolean is encoded within an unsigned integer: 0 for false, 1 for true. -A signed integer, i, is encoded within an unsigned integer, u. Within u, bits 1 +A signed integer, i, is encoded within an unsigned integer, u. Within u, bits 1 upward contain the value; bit 0 says whether they should be complemented upon -receipt. The encode algorithm looks like this: +receipt. The encode algorithm looks like this: var u uint if i < 0 { @@ -127,14 +127,14 @@ receipt. The encode algorithm looks like this: encodeUnsigned(u) The low bit is therefore analogous to a sign bit, but making it the complement bit -instead guarantees that the largest negative integer is not a special case. For +instead guarantees that the largest negative integer is not a special case. For example, -129=^128=(^256>>1) encodes as (FE 01 01). Floating-point numbers are always sent as a representation of a float64 value. -That value is converted to a uint64 using math.Float64bits. The uint64 is then -byte-reversed and sent as a regular unsigned integer. The byte-reversal means the -exponent and high-precision part of the mantissa go first. Since the low bits are -often zero, this can save encoding bytes. For instance, 17.0 is encoded in only +That value is converted to a uint64 using math.Float64bits. The uint64 is then +byte-reversed and sent as a regular unsigned integer. The byte-reversal means the +exponent and high-precision part of the mantissa go first. Since the low bits are +often zero, this can save encoding bytes. For instance, 17.0 is encoded in only three bytes (FE 31 40). Strings and slices of bytes are sent as an unsigned count followed by that many @@ -151,33 +151,39 @@ is nil and not at the top level. In slices and arrays, as well as maps, all elements, even zero-valued elements, are transmitted, even if all the elements are zero. -Structs are sent as a sequence of (field number, field value) pairs. The field -value is sent using the standard gob encoding for its type, recursively. If a +Structs are sent as a sequence of (field number, field value) pairs. The field +value is sent using the standard gob encoding for its type, recursively. If a field has the zero value for its type (except for arrays; see above), it is omitted -from the transmission. The field number is defined by the type of the encoded +from the transmission. The field number is defined by the type of the encoded struct: the first field of the encoded type is field 0, the second is field 1, -etc. When encoding a value, the field numbers are delta encoded for efficiency +etc. When encoding a value, the field numbers are delta encoded for efficiency and the fields are always sent in order of increasing field number; the deltas are -therefore unsigned. The initialization for the delta encoding sets the field +therefore unsigned. The initialization for the delta encoding sets the field number to -1, so an unsigned integer field 0 with value 7 is transmitted as unsigned -delta = 1, unsigned value = 7 or (01 07). Finally, after all the fields have been -sent a terminating mark denotes the end of the struct. That mark is a delta=0 +delta = 1, unsigned value = 7 or (01 07). Finally, after all the fields have been +sent a terminating mark denotes the end of the struct. That mark is a delta=0 value, which has representation (00). Interface types are not checked for compatibility; all interface types are treated, for transmission, as members of a single "interface" type, analogous to -int or []byte - in effect they're all treated as interface{}. Interface values +int or []byte - in effect they're all treated as interface{}. Interface values are transmitted as a string identifying the concrete type being sent (a name that must be pre-defined by calling Register), followed by a byte count of the length of the following data (so the value can be skipped if it cannot be stored), followed by the usual encoding of concrete (dynamic) value stored in -the interface value. (A nil interface value is identified by the empty string +the interface value. (A nil interface value is identified by the empty string and transmits no value.) Upon receipt, the decoder verifies that the unpacked concrete item satisfies the interface of the receiving variable. -The representation of types is described below. When a type is defined on a given +If a value is passed to Encode and the type is not a struct (or pointer to struct, +etc.), for simplicity of processing it is represented as a struct of one field. +The only visible effect of this is to encode a zero byte after the value, just as +after the last field of an encoded struct, so that the decode algorithm knows when +the top-level value is complete. + +The representation of types is described below. When a type is defined on a given connection between an Encoder and Decoder, it is assigned a signed integer type -id. When Encoder.Encode(v) is called, it makes sure there is an id assigned for +id. When Encoder.Encode(v) is called, it makes sure there is an id assigned for the type of v and all its elements and then it sends the pair (typeid, encoded-v) where typeid is the type id of the encoded type of v and encoded-v is the gob encoding of the value v. @@ -223,7 +229,7 @@ If there are nested type ids, the types for all inner type ids must be defined before the top-level type id is used to describe an encoded-v. For simplicity in setup, the connection is defined to understand these types a -priori, as well as the basic gob types int, uint, etc. Their ids are: +priori, as well as the basic gob types int, uint, etc. Their ids are: bool 1 int 2 @@ -244,7 +250,7 @@ priori, as well as the basic gob types int, uint, etc. Their ids are: MapType 23 Finally, each message created by a call to Encode is preceded by an encoded -unsigned integer count of the number of bytes remaining in the message. After +unsigned integer count of the number of bytes remaining in the message. After the initial type name, interface values are wrapped the same way; in effect, the interface value acts like a recursive invocation of Encode. @@ -256,7 +262,7 @@ where * signifies zero or more repetitions and the type id of a value must be predefined or be defined before the value in the stream. Compatibility: Any future changes to the package will endeavor to maintain -compatibility with streams encoded using previous versions. That is, any released +compatibility with streams encoded using previous versions. That is, any released version of this package should be able to decode data written with any previously released version, subject to issues such as security fixes. See the Go compatibility document for background: https://golang.org/doc/go1compat @@ -315,7 +321,7 @@ StructValue: */ /* -For implementers and the curious, here is an encoded example. Given +For implementers and the curious, here is an encoded example. Given type Point struct {X, Y int} and the value p := Point{22, 33} @@ -326,14 +332,14 @@ the bytes transmitted that encode p will be: They are determined as follows. Since this is the first transmission of type Point, the type descriptor -for Point itself must be sent before the value. This is the first type +for Point itself must be sent before the value. This is the first type we've sent on this Encoder, so it has type id 65 (0 through 64 are reserved). 1f // This item (a type descriptor) is 31 bytes long. ff 81 // The negative of the id for the type we're defining, -65. // This is one byte (indicated by FF = -1) followed by - // ^-65<<1 | 1. The low 1 bit signals to complement the + // ^-65<<1 | 1. The low 1 bit signals to complement the // rest upon receipt. // Now we send a type descriptor, which is itself a struct (wireType). @@ -370,7 +376,7 @@ reserved). 00 // end of wireType.structType structure 00 // end of wireType structure -Now we can send the Point value. Again the field number resets to -1: +Now we can send the Point value. Again the field number resets to -1: 07 // this value is 7 bytes long ff 82 // the type number, 65 (1 byte (-FF) followed by 65<<1) @@ -387,7 +393,7 @@ output will be just: 07 ff 82 01 2c 01 42 00 A single non-struct value at top level is transmitted like a field with -delta tag 0. For instance, a signed integer with value 3 presented as +delta tag 0. For instance, a signed integer with value 3 presented as the argument to Encode will emit: 03 04 00 06 diff --git a/libgo/go/encoding/gob/encode.go b/libgo/go/encoding/gob/encode.go index 50cd6adb463..5371e7245f4 100644 --- a/libgo/go/encoding/gob/encode.go +++ b/libgo/go/encoding/gob/encode.go @@ -8,7 +8,9 @@ package gob import ( "encoding" + "encoding/binary" "math" + "math/bits" "reflect" "sync" ) @@ -107,14 +109,12 @@ func (state *encoderState) encodeUint(x uint64) { state.b.WriteByte(uint8(x)) return } - i := uint64Size - for x > 0 { - state.buf[i] = uint8(x) - x >>= 8 - i-- - } - state.buf[i] = uint8(i - uint64Size) // = loop count, negated - state.b.Write(state.buf[i : uint64Size+1]) + + binary.BigEndian.PutUint64(state.buf[1:], x) + bc := bits.LeadingZeros64(x) >> 3 // 8 - bytelen(x) + state.buf[bc] = uint8(bc - uint64Size) // and then we subtract 8 to get -bytelen(x) + + state.b.Write(state.buf[bc : uint64Size+1]) } // encodeInt writes an encoded signed integer to state.w. @@ -209,13 +209,7 @@ func encUint(i *encInstr, state *encoderState, v reflect.Value) { // swizzling. func floatBits(f float64) uint64 { u := math.Float64bits(f) - var v uint64 - for i := 0; i < 8; i++ { - v <<= 8 - v |= u & 0xFF - u >>= 8 - } - return v + return bits.ReverseBytes64(u) } // encFloat encodes the floating point value (float32 float64) referenced by v. @@ -404,12 +398,12 @@ func (enc *Encoder) encodeInterface(b *encBuffer, iv reflect.Value) { } ut := userType(iv.Elem().Type()) - registerLock.RLock() - name, ok := concreteTypeToName[ut.base] - registerLock.RUnlock() + namei, ok := concreteTypeToName.Load(ut.base) if !ok { errorf("type not registered for interface: %s", ut.base) } + name := namei.(string) + // Send the name. state.encodeUint(uint64(len(name))) state.b.WriteString(name) diff --git a/libgo/go/encoding/gob/encoder_test.go b/libgo/go/encoding/gob/encoder_test.go index 9256848b50e..a1ca252ccd1 100644 --- a/libgo/go/encoding/gob/encoder_test.go +++ b/libgo/go/encoding/gob/encoder_test.go @@ -55,6 +55,71 @@ func TestBasicEncoderDecoder(t *testing.T) { } } +func TestEncodeIntSlice(t *testing.T) { + + s8 := []int8{1, 5, 12, 22, 35, 51, 70, 92, 117} + s16 := []int16{145, 176, 210, 247, 287, 330, 376, 425, 477} + s32 := []int32{532, 590, 651, 715, 782, 852, 925, 1001, 1080} + s64 := []int64{1162, 1247, 1335, 1426, 1520, 1617, 1717, 1820, 1926} + + t.Run("int8", func(t *testing.T) { + var sink bytes.Buffer + enc := NewEncoder(&sink) + enc.Encode(s8) + + dec := NewDecoder(&sink) + res := make([]int8, 9) + dec.Decode(&res) + + if !reflect.DeepEqual(s8, res) { + t.Fatalf("EncodeIntSlice: expected %v, got %v", s8, res) + } + }) + + t.Run("int16", func(t *testing.T) { + var sink bytes.Buffer + enc := NewEncoder(&sink) + enc.Encode(s16) + + dec := NewDecoder(&sink) + res := make([]int16, 9) + dec.Decode(&res) + + if !reflect.DeepEqual(s16, res) { + t.Fatalf("EncodeIntSlice: expected %v, got %v", s16, res) + } + }) + + t.Run("int32", func(t *testing.T) { + var sink bytes.Buffer + enc := NewEncoder(&sink) + enc.Encode(s32) + + dec := NewDecoder(&sink) + res := make([]int32, 9) + dec.Decode(&res) + + if !reflect.DeepEqual(s32, res) { + t.Fatalf("EncodeIntSlice: expected %v, got %v", s32, res) + } + }) + + t.Run("int64", func(t *testing.T) { + var sink bytes.Buffer + enc := NewEncoder(&sink) + enc.Encode(s64) + + dec := NewDecoder(&sink) + res := make([]int64, 9) + dec.Decode(&res) + + if !reflect.DeepEqual(s64, res) { + t.Fatalf("EncodeIntSlice: expected %v, got %v", s64, res) + } + }) + +} + type ET0 struct { A int B string diff --git a/libgo/go/encoding/gob/error.go b/libgo/go/encoding/gob/error.go index 8b5265c278f..949333bc037 100644 --- a/libgo/go/encoding/gob/error.go +++ b/libgo/go/encoding/gob/error.go @@ -39,5 +39,4 @@ func catchError(err *error) { } *err = ge.err } - return } diff --git a/libgo/go/encoding/gob/gobencdec_test.go b/libgo/go/encoding/gob/gobencdec_test.go index ecc91eef1f8..41a06b26c87 100644 --- a/libgo/go/encoding/gob/gobencdec_test.go +++ b/libgo/go/encoding/gob/gobencdec_test.go @@ -746,7 +746,7 @@ func (i *isZeroBugInterface) GobDecode(data []byte) error { } func TestGobEncodeIsZero(t *testing.T) { - x := isZeroBug{time.Now(), "hello", -55, isZeroBugArray{1, 2}, isZeroBugInterface{}} + x := isZeroBug{time.Unix(1e9, 0), "hello", -55, isZeroBugArray{1, 2}, isZeroBugInterface{}} b := new(bytes.Buffer) enc := NewEncoder(b) err := enc.Encode(x) diff --git a/libgo/go/encoding/gob/timing_test.go b/libgo/go/encoding/gob/timing_test.go index 424b7e6ea8e..3478bd247ed 100644 --- a/libgo/go/encoding/gob/timing_test.go +++ b/libgo/go/encoding/gob/timing_test.go @@ -8,6 +8,7 @@ import ( "bytes" "io" "os" + "reflect" "runtime" "testing" ) @@ -132,89 +133,60 @@ func TestCountDecodeMallocs(t *testing.T) { } } +func benchmarkEncodeSlice(b *testing.B, a interface{}) { + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + + for pb.Next() { + buf.Reset() + err := enc.Encode(a) + if err != nil { + b.Fatal(err) + } + } + }) +} + func BenchmarkEncodeComplex128Slice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]complex128, 1000) for i := range a { a[i] = 1.2 + 3.4i } - b.ResetTimer() - for i := 0; i < b.N; i++ { - buf.Reset() - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } - } + benchmarkEncodeSlice(b, a) } func BenchmarkEncodeFloat64Slice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]float64, 1000) for i := range a { a[i] = 1.23e4 } - b.ResetTimer() - for i := 0; i < b.N; i++ { - buf.Reset() - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } - } + benchmarkEncodeSlice(b, a) } func BenchmarkEncodeInt32Slice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]int32, 1000) for i := range a { - a[i] = 1234 - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - buf.Reset() - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } + a[i] = int32(i * 100) } + benchmarkEncodeSlice(b, a) } func BenchmarkEncodeStringSlice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]string, 1000) for i := range a { a[i] = "now is the time" } - b.ResetTimer() - for i := 0; i < b.N; i++ { - buf.Reset() - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } - } + benchmarkEncodeSlice(b, a) } func BenchmarkEncodeInterfaceSlice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]interface{}, 1000) for i := range a { a[i] = "now is the time" } - b.ResetTimer() - for i := 0; i < b.N; i++ { - buf.Reset() - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } - } + benchmarkEncodeSlice(b, a) } // benchmarkBuf is a read buffer we can reset @@ -245,120 +217,96 @@ func (b *benchmarkBuf) reset() { b.offset = 0 } -func BenchmarkDecodeComplex128Slice(b *testing.B) { +func benchmarkDecodeSlice(b *testing.B, a interface{}) { var buf bytes.Buffer enc := NewEncoder(&buf) - a := make([]complex128, 1000) - for i := range a { - a[i] = 1.2 + 3.4i - } err := enc.Encode(a) if err != nil { b.Fatal(err) } - x := make([]complex128, 1000) - bbuf := benchmarkBuf{data: buf.Bytes()} + + ra := reflect.ValueOf(a) + rt := ra.Type() b.ResetTimer() - for i := 0; i < b.N; i++ { - bbuf.reset() - dec := NewDecoder(&bbuf) - err := dec.Decode(&x) - if err != nil { - b.Fatal(i, err) + + b.RunParallel(func(pb *testing.PB) { + // TODO(#19025): Move per-thread allocation before ResetTimer. + rp := reflect.New(rt) + rp.Elem().Set(reflect.MakeSlice(rt, ra.Len(), ra.Cap())) + p := rp.Interface() + + bbuf := benchmarkBuf{data: buf.Bytes()} + + for pb.Next() { + bbuf.reset() + dec := NewDecoder(&bbuf) + err := dec.Decode(p) + if err != nil { + b.Fatal(err) + } } + }) +} + +func BenchmarkDecodeComplex128Slice(b *testing.B) { + a := make([]complex128, 1000) + for i := range a { + a[i] = 1.2 + 3.4i } + benchmarkDecodeSlice(b, a) } func BenchmarkDecodeFloat64Slice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]float64, 1000) for i := range a { a[i] = 1.23e4 } - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } - x := make([]float64, 1000) - bbuf := benchmarkBuf{data: buf.Bytes()} - b.ResetTimer() - for i := 0; i < b.N; i++ { - bbuf.reset() - dec := NewDecoder(&bbuf) - err := dec.Decode(&x) - if err != nil { - b.Fatal(i, err) - } - } + benchmarkDecodeSlice(b, a) } func BenchmarkDecodeInt32Slice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]int32, 1000) for i := range a { a[i] = 1234 } - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } - x := make([]int32, 1000) - bbuf := benchmarkBuf{data: buf.Bytes()} - b.ResetTimer() - for i := 0; i < b.N; i++ { - bbuf.reset() - dec := NewDecoder(&bbuf) - err := dec.Decode(&x) - if err != nil { - b.Fatal(i, err) - } - } + benchmarkDecodeSlice(b, a) } func BenchmarkDecodeStringSlice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]string, 1000) for i := range a { a[i] = "now is the time" } - err := enc.Encode(a) - if err != nil { - b.Fatal(err) - } - x := make([]string, 1000) - bbuf := benchmarkBuf{data: buf.Bytes()} - b.ResetTimer() - for i := 0; i < b.N; i++ { - bbuf.reset() - dec := NewDecoder(&bbuf) - err := dec.Decode(&x) - if err != nil { - b.Fatal(i, err) - } - } + benchmarkDecodeSlice(b, a) } func BenchmarkDecodeInterfaceSlice(b *testing.B) { - var buf bytes.Buffer - enc := NewEncoder(&buf) a := make([]interface{}, 1000) for i := range a { a[i] = "now is the time" } - err := enc.Encode(a) + benchmarkDecodeSlice(b, a) +} + +func BenchmarkDecodeMap(b *testing.B) { + count := 1000 + m := make(map[int]int, count) + for i := 0; i < count; i++ { + m[i] = i + } + var buf bytes.Buffer + enc := NewEncoder(&buf) + err := enc.Encode(m) if err != nil { b.Fatal(err) } - x := make([]interface{}, 1000) bbuf := benchmarkBuf{data: buf.Bytes()} b.ResetTimer() for i := 0; i < b.N; i++ { + var rm map[int]int bbuf.reset() dec := NewDecoder(&bbuf) - err := dec.Decode(&x) + err := dec.Decode(&rm) if err != nil { b.Fatal(i, err) } diff --git a/libgo/go/encoding/gob/type.go b/libgo/go/encoding/gob/type.go index c27f7e9707e..31c0ef7af15 100644 --- a/libgo/go/encoding/gob/type.go +++ b/libgo/go/encoding/gob/type.go @@ -36,31 +36,21 @@ const ( xText // encoding.TextMarshaler or encoding.TextUnmarshaler ) -var ( - // Protected by an RWMutex because we read it a lot and write - // it only when we see a new type, typically when compiling. - userTypeLock sync.RWMutex - userTypeCache = make(map[reflect.Type]*userTypeInfo) -) +var userTypeCache sync.Map // map[reflect.Type]*userTypeInfo // validType returns, and saves, the information associated with user-provided type rt. // If the user type is not valid, err will be non-nil. To be used when the error handler // is not set up. -func validUserType(rt reflect.Type) (ut *userTypeInfo, err error) { - userTypeLock.RLock() - ut = userTypeCache[rt] - userTypeLock.RUnlock() - if ut != nil { - return - } - // Now set the value under the write lock. - userTypeLock.Lock() - defer userTypeLock.Unlock() - if ut = userTypeCache[rt]; ut != nil { - // Lost the race; not a problem. - return +func validUserType(rt reflect.Type) (*userTypeInfo, error) { + if ui, ok := userTypeCache.Load(rt); ok { + return ui.(*userTypeInfo), nil } - ut = new(userTypeInfo) + + // Construct a new userTypeInfo and atomically add it to the userTypeCache. + // If we lose the race, we'll waste a little CPU and create a little garbage + // but return the existing value anyway. + + ut := new(userTypeInfo) ut.base = rt ut.user = rt // A type that is just a cycle of pointers (such as type T *T) cannot @@ -108,8 +98,8 @@ func validUserType(rt reflect.Type) (ut *userTypeInfo, err error) { // ut.externalDec, ut.decIndir = xText, indir // } - userTypeCache[rt] = ut - return + ui, _ := userTypeCache.LoadOrStore(rt, ut) + return ui.(*userTypeInfo), nil } var ( @@ -808,9 +798,8 @@ type GobDecoder interface { } var ( - registerLock sync.RWMutex - nameToConcreteType = make(map[string]reflect.Type) - concreteTypeToName = make(map[reflect.Type]string) + nameToConcreteType sync.Map // map[string]reflect.Type + concreteTypeToName sync.Map // map[reflect.Type]string ) // RegisterName is like Register but uses the provided name rather than the @@ -820,21 +809,22 @@ func RegisterName(name string, value interface{}) { // reserved for nil panic("attempt to register empty name") } - registerLock.Lock() - defer registerLock.Unlock() + ut := userType(reflect.TypeOf(value)) + // Check for incompatible duplicates. The name must refer to the // same user type, and vice versa. - if t, ok := nameToConcreteType[name]; ok && t != ut.user { + + // Store the name and type provided by the user.... + if t, dup := nameToConcreteType.LoadOrStore(name, reflect.TypeOf(value)); dup && t != ut.user { panic(fmt.Sprintf("gob: registering duplicate types for %q: %s != %s", name, t, ut.user)) } - if n, ok := concreteTypeToName[ut.base]; ok && n != name { + + // but the flattened type in the type table, since that's what decode needs. + if n, dup := concreteTypeToName.LoadOrStore(ut.base, name); dup && n != name { + nameToConcreteType.Delete(name) panic(fmt.Sprintf("gob: registering duplicate names for %s: %q != %q", ut.user, n, name)) } - // Store the name and type provided by the user.... - nameToConcreteType[name] = reflect.TypeOf(value) - // but the flattened type in the type table, since that's what decode needs. - concreteTypeToName[ut.base] = name } // Register records a type, identified by a value for that type, under its diff --git a/libgo/go/encoding/gob/type_test.go b/libgo/go/encoding/gob/type_test.go index e230d22d431..14f25d8ac4c 100644 --- a/libgo/go/encoding/gob/type_test.go +++ b/libgo/go/encoding/gob/type_test.go @@ -178,9 +178,7 @@ func TestRegistrationNaming(t *testing.T) { Register(tc.t) tct := reflect.TypeOf(tc.t) - registerLock.RLock() - ct := nameToConcreteType[tc.name] - registerLock.RUnlock() + ct, _ := nameToConcreteType.Load(tc.name) if ct != tct { t.Errorf("nameToConcreteType[%q] = %v, want %v", tc.name, ct, tct) } @@ -188,7 +186,7 @@ func TestRegistrationNaming(t *testing.T) { if tct.Kind() == reflect.Ptr { tct = tct.Elem() } - if n := concreteTypeToName[tct]; n != tc.name { + if n, _ := concreteTypeToName.Load(tct); n != tc.name { t.Errorf("concreteTypeToName[%v] got %v, want %v", tct, n, tc.name) } } |