diff options
Diffstat (limited to 'src/mongo/gotools/common/json')
64 files changed, 11952 insertions, 0 deletions
diff --git a/src/mongo/gotools/common/json/bench_test.go b/src/mongo/gotools/common/json/bench_test.go new file mode 100644 index 00000000000..29dbc26d417 --- /dev/null +++ b/src/mongo/gotools/common/json/bench_test.go @@ -0,0 +1,189 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Large data benchmark. +// The JSON data is a summary of agl's changes in the +// go, webkit, and chromium open source projects. +// We benchmark converting between the JSON form +// and in-memory data structures. + +package json + +import ( + "bytes" + "compress/gzip" + "io/ioutil" + "os" + "testing" +) + +type codeResponse struct { + Tree *codeNode `json:"tree"` + Username string `json:"username"` +} + +type codeNode struct { + Name string `json:"name"` + Kids []*codeNode `json:"kids"` + CLWeight float64 `json:"cl_weight"` + Touches int `json:"touches"` + MinT int64 `json:"min_t"` + MaxT int64 `json:"max_t"` + MeanT int64 `json:"mean_t"` +} + +var codeJSON []byte +var codeStruct codeResponse + +func codeInit() { + f, err := os.Open("testdata/code.json.gz") + if err != nil { + panic(err) + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + panic(err) + } + data, err := ioutil.ReadAll(gz) + if err != nil { + panic(err) + } + + codeJSON = data + + if err := Unmarshal(codeJSON, &codeStruct); err != nil { + panic("unmarshal code.json: " + err.Error()) + } + + if data, err = Marshal(&codeStruct); err != nil { + panic("marshal code.json: " + err.Error()) + } + + if !bytes.Equal(data, codeJSON) { + println("different lengths", len(data), len(codeJSON)) + for i := 0; i < len(data) && i < len(codeJSON); i++ { + if data[i] != codeJSON[i] { + println("re-marshal: changed at byte", i) + println("orig: ", string(codeJSON[i-10:i+10])) + println("new: ", string(data[i-10:i+10])) + break + } + } + panic("re-marshal code.json: different result") + } +} + +func BenchmarkCodeEncoder(b *testing.B) { + if codeJSON == nil { + b.StopTimer() + codeInit() + b.StartTimer() + } + enc := NewEncoder(ioutil.Discard) + for i := 0; i < b.N; i++ { + if err := enc.Encode(&codeStruct); err != nil { + b.Fatal("Encode:", err) + } + } + b.SetBytes(int64(len(codeJSON))) +} + +func BenchmarkCodeMarshal(b *testing.B) { + if codeJSON == nil { + b.StopTimer() + codeInit() + b.StartTimer() + } + for i := 0; i < b.N; i++ { + if _, err := Marshal(&codeStruct); err != nil { + b.Fatal("Marshal:", err) + } + } + b.SetBytes(int64(len(codeJSON))) +} + +func BenchmarkCodeDecoder(b *testing.B) { + if codeJSON == nil { + b.StopTimer() + codeInit() + b.StartTimer() + } + var buf bytes.Buffer + dec := NewDecoder(&buf) + var r codeResponse + for i := 0; i < b.N; i++ { + buf.Write(codeJSON) + // hide EOF + buf.WriteByte('\n') + buf.WriteByte('\n') + buf.WriteByte('\n') + if err := dec.Decode(&r); err != nil { + b.Fatal("Decode:", err) + } + } + b.SetBytes(int64(len(codeJSON))) +} + +func BenchmarkCodeUnmarshal(b *testing.B) { + if codeJSON == nil { + b.StopTimer() + codeInit() + b.StartTimer() + } + for i := 0; i < b.N; i++ { + var r codeResponse + if err := Unmarshal(codeJSON, &r); err != nil { + b.Fatal("Unmmarshal:", err) + } + } + b.SetBytes(int64(len(codeJSON))) +} + +func BenchmarkCodeUnmarshalReuse(b *testing.B) { + if codeJSON == nil { + b.StopTimer() + codeInit() + b.StartTimer() + } + var r codeResponse + for i := 0; i < b.N; i++ { + if err := Unmarshal(codeJSON, &r); err != nil { + b.Fatal("Unmmarshal:", err) + } + } +} + +func BenchmarkUnmarshalString(b *testing.B) { + data := []byte(`"hello, world"`) + var s string + + for i := 0; i < b.N; i++ { + if err := Unmarshal(data, &s); err != nil { + b.Fatal("Unmarshal:", err) + } + } +} + +func BenchmarkUnmarshalFloat64(b *testing.B) { + var f float64 + data := []byte(`3.14`) + + for i := 0; i < b.N; i++ { + if err := Unmarshal(data, &f); err != nil { + b.Fatal("Unmarshal:", err) + } + } +} + +func BenchmarkUnmarshalInt64(b *testing.B) { + var x int64 + data := []byte(`3`) + + for i := 0; i < b.N; i++ { + if err := Unmarshal(data, &x); err != nil { + b.Fatal("Unmarshal:", err) + } + } +} diff --git a/src/mongo/gotools/common/json/bindata.go b/src/mongo/gotools/common/json/bindata.go new file mode 100644 index 00000000000..a7357675ba8 --- /dev/null +++ b/src/mongo/gotools/common/json/bindata.go @@ -0,0 +1,67 @@ +package json + +import ( + "fmt" + "reflect" +) + +// Transition functions for recognizing BinData. +// Adapted from encoding/json/scanner.go. + +// stateBi is the state after reading `Bi`. +func stateBi(s *scanner, c int) int { + if c == 'n' { + s.step = generateState("BinData", []byte("Data"), stateConstructor) + return scanContinue + } + return s.error(c, "in literal BinData (expecting 'n')") +} + +// Decodes a BinData literal stored in the underlying byte data into v. +func (d *decodeState) storeBinData(v reflect.Value) { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginCtor { + d.error(fmt.Errorf("expected beginning of constructor")) + } + + args, err := d.ctor("BinData", []reflect.Type{byteType, stringType}) + if err != nil { + d.error(err) + } + switch kind := v.Kind(); kind { + case reflect.Interface: + arg0 := byte(args[0].Uint()) + arg1 := args[1].String() + v.Set(reflect.ValueOf(BinData{arg0, arg1})) + default: + d.error(fmt.Errorf("cannot store %v value into %v type", binDataType, kind)) + } +} + +// Returns a BinData literal from the underlying byte data. +func (d *decodeState) getBinData() interface{} { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginCtor { + d.error(fmt.Errorf("expected beginning of constructor")) + } + + // Prevent d.convertNumber() from parsing the argument as a float64. + useNumber := d.useNumber + d.useNumber = true + + args := d.ctorInterface() + if err := ctorNumArgsMismatch("BinData", 2, len(args)); err != nil { + d.error(err) + } + arg0, err := args[0].(Number).Uint8() + if err != nil { + d.error(fmt.Errorf("expected byte for first argument of BinData constructor")) + } + arg1, ok := args[1].(string) + if !ok { + d.error(fmt.Errorf("expected string for second argument of BinData constructor")) + } + + d.useNumber = useNumber + return BinData{arg0, arg1} +} diff --git a/src/mongo/gotools/common/json/bindata_test.go b/src/mongo/gotools/common/json/bindata_test.go new file mode 100644 index 00000000000..97d3dddee24 --- /dev/null +++ b/src/mongo/gotools/common/json/bindata_test.go @@ -0,0 +1,89 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestBinDataValue(t *testing.T) { + + Convey("When unmarshalling JSON with BinData values", t, func() { + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `BinData(1, "xyz")` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(BinData) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, BinData{1, "xyz"}) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + value1, value2, value3 := `BinData(1, "abc")`, + `BinData(2, "def")`, `BinData(3, "ghi")` + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value1, key2, value2, key3, value3) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue1, ok := jsonMap[key1].(BinData) + So(ok, ShouldBeTrue) + So(jsonValue1, ShouldResemble, BinData{1, "abc"}) + + jsonValue2, ok := jsonMap[key2].(BinData) + So(ok, ShouldBeTrue) + So(jsonValue2, ShouldResemble, BinData{2, "def"}) + + jsonValue3, ok := jsonMap[key3].(BinData) + So(ok, ShouldBeTrue) + So(jsonValue3, ShouldResemble, BinData{3, "ghi"}) + }) + + Convey("works in an array", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `BinData(42, "10")` + data := fmt.Sprintf(`{"%v":[%v,%v,%v]}`, + key, value, value, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonArray, ok := jsonMap[key].([]interface{}) + So(ok, ShouldBeTrue) + + for _, _jsonValue := range jsonArray { + jsonValue, ok := _jsonValue.(BinData) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, BinData{42, "10"}) + } + }) + + Convey("can specify type argument using hexadecimal", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `BinData(0x5f, "xyz")` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(BinData) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, BinData{0x5f, "xyz"}) + }) + }) +} diff --git a/src/mongo/gotools/common/json/boolean.go b/src/mongo/gotools/common/json/boolean.go new file mode 100644 index 00000000000..b40ba5204da --- /dev/null +++ b/src/mongo/gotools/common/json/boolean.go @@ -0,0 +1,73 @@ +package json + +import ( + "fmt" + "reflect" +) + +// Transition functions for recognizing Boolean. +// Adapted from encoding/json/scanner.go. + +// stateBo is the state after reading `Bo`. +func stateBo(s *scanner, c int) int { + if c == 'o' { + s.step = generateState("Boolean", []byte("lean"), stateConstructor) + return scanContinue + } + return s.error(c, "in literal Boolean (expecting 'o')") +} + +// Decodes a Boolean literal stored in the underlying byte data into v. +func (d *decodeState) storeBoolean(v reflect.Value) { + res := d.getBoolean() + switch kind := v.Kind(); kind { + case reflect.Interface, reflect.Bool: + v.Set(reflect.ValueOf(res)) + default: + d.error(fmt.Errorf("cannot store bool value into %v type", kind)) + } +} + +// Returns a Boolean literal from the underlying byte data. +func (d *decodeState) getBoolean() interface{} { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginCtor { + d.error(fmt.Errorf("expected beginning of constructor")) + } + + // Prevent d.convertNumber() from parsing the argument as a float64. + useNumber := d.useNumber + d.useNumber = true + + args := d.ctorInterface() + if len(args) == 0 { + return false + } + + // Ignore all but the first argument. + switch v := args[0].(type) { + case bool: + return v + case Number: + d.useNumber = useNumber + + // First try Int64 so hex numbers work, then if that fails try Float64. + num, err := v.Int64() + if err == nil { + return (num != 0) + } + + numF, err := v.Float64() + if err != nil { + d.error(fmt.Errorf("expected float64 for numeric argument of Boolean constructor, got err: %v", err)) + } + return (numF != 0) + case string: + return (v != "") + case Undefined, nil: + return false + // Parameter values of any other types should yield true. + default: + return true + } +} diff --git a/src/mongo/gotools/common/json/boolean_test.go b/src/mongo/gotools/common/json/boolean_test.go new file mode 100644 index 00000000000..5b5cae6e2ed --- /dev/null +++ b/src/mongo/gotools/common/json/boolean_test.go @@ -0,0 +1,368 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestBooleanValue(t *testing.T) { + + Convey("When unmarshalling JSON with Boolean values", t, func() { + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "Boolean(123)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + jsonValue, ok := jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, true) + }) + + Convey("works for no args", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "Boolean()" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + jsonValue, ok := jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, false) + }) + + Convey("works for a struct of a specific type", func() { + type TestStruct struct { + A bool + b int + } + var jsonStruct TestStruct + + key := "A" + value := "Boolean(123)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonStruct) + So(err, ShouldBeNil) + So(jsonStruct.A, ShouldEqual, true) + + key = "A" + value = "Boolean(0)" + data = fmt.Sprintf(`{"%v":%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonStruct) + So(err, ShouldBeNil) + So(jsonStruct.A, ShouldEqual, false) + }) + + Convey("works for bool", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "Boolean(true)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, true) + + value = "Boolean(false)" + data = fmt.Sprintf(`{"%v":%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok = jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, false) + }) + + Convey("works for numbers", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "Boolean(1)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, true) + + value = "Boolean(0)" + data = fmt.Sprintf(`{"%v":%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok = jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, false) + + value = "Boolean(0.0)" + data = fmt.Sprintf(`{"%v":%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok = jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, false) + + value = "Boolean(2.0)" + data = fmt.Sprintf(`{"%v":%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok = jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, true) + + value = "Boolean(-15.4)" + data = fmt.Sprintf(`{"%v":%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok = jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, true) + }) + + Convey("works for strings", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "Boolean('hello')" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, true) + + value = "Boolean('')" + data = fmt.Sprintf(`{"%v":%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok = jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, false) + }) + + Convey("works for undefined", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "Boolean(undefined)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, false) + }) + + Convey("works for null", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "Boolean(null)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, false) + }) + + Convey("works when given too many args", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "Boolean(true, false)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, true) + + key = "key" + value = "Boolean(false, true)" + data = fmt.Sprintf(`{"%v":%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok = jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, false) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + value1, value2, value3 := "Boolean(123)", "Boolean(0)", "Boolean(true)" + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value1, key2, value2, key3, value3) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue1, ok := jsonMap[key1].(bool) + So(ok, ShouldBeTrue) + So(jsonValue1, ShouldEqual, true) + + jsonValue2, ok := jsonMap[key2].(bool) + So(ok, ShouldBeTrue) + So(jsonValue2, ShouldEqual, false) + + jsonValue3, ok := jsonMap[key3].(bool) + So(ok, ShouldBeTrue) + So(jsonValue3, ShouldEqual, true) + }) + + Convey("works for other types", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "Boolean(new Date (0))" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, true) + + value = "Boolean(ObjectId('56609335028bd7dc5c36cb9f'))" + data = fmt.Sprintf(`{"%v":%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok = jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, true) + + value = "Boolean([])" + data = fmt.Sprintf(`{"%v":%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok = jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, true) + }) + + Convey("works for nested booleans", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "Boolean(Boolean(5))" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, true) + + value = "Boolean(Boolean(Boolean(0)))" + data = fmt.Sprintf(`{"%v":%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok = jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, false) + }) + + Convey("works in an array", func() { + var jsonMap map[string]interface{} + + key := "key" + value1 := "Boolean(42)" + value2 := "Boolean(0)" + data := fmt.Sprintf(`{"%v":[%v,%v,%v]}`, + key, value1, value2, value1) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonArray, ok := jsonMap[key].([]interface{}) + So(ok, ShouldBeTrue) + + jsonValue, ok := jsonArray[0].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, true) + + jsonValue, ok = jsonArray[1].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, false) + + jsonValue, ok = jsonArray[2].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, true) + }) + + Convey("can specify argument in hexadecimal (true)", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "Boolean(0x5f)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, true) + }) + + Convey("can specify argument in hexadecimal (false)", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "Boolean(0x0)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, false) + }) + }) +} diff --git a/src/mongo/gotools/common/json/constructor.go b/src/mongo/gotools/common/json/constructor.go new file mode 100644 index 00000000000..b2bb91165fc --- /dev/null +++ b/src/mongo/gotools/common/json/constructor.go @@ -0,0 +1,117 @@ +package json + +import ( + "fmt" + "reflect" +) + +const CtorNumArgsErrorf = "expected %v argument%v to %v constructor, but %v received" + +// Transition functions for recognizing object constructors. +// Adapted from encoding/json/scanner.go. + +// stateConstructor is the state after reading a constructor name. +func stateConstructor(s *scanner, c int) int { + if c <= ' ' && isSpace(rune(c)) { + return scanSkipSpace + } + if c == '(' { + s.step = stateBeginCtorOrEmpty + s.pushParseState(parseCtorArg) + return scanBeginCtor + } + return s.error(c, "expected '('") +} + +// stateBeginCtorOrEmpty is the state after reading `(`. +func stateBeginCtorOrEmpty(s *scanner, c int) int { + if c <= ' ' && isSpace(rune(c)) { + return scanSkipSpace + } + if c == ')' { + return stateEndValue(s, c) + } + return stateBeginValue(s, c) +} + +// ctor consumes a constructor from d.data[d.off-1:], given a type specification t. +// the first byte of the constructor ('(') has been read already. +func (d *decodeState) ctor(name string, t []reflect.Type) ([]reflect.Value, error) { + result := make([]reflect.Value, 0, len(t)) + + i := 0 + for { + // Look ahead for ) - can only happen on first iteration. + op := d.scanWhile(scanSkipSpace) + if op == scanEndCtor { + break + } + + // Back up so d.value can have the byte we just read. + d.off-- + d.scan.undo(op) + + if i < len(t) { + v := reflect.New(t[i]).Elem() + + // Get argument of constructor + d.value(v) + + result = append(result, v) + i++ + } + + // Next token must be , or ). + op = d.scanWhile(scanSkipSpace) + if op == scanEndCtor { + break + } + if op != scanCtorArg { + d.error(errPhase) + } + } + + return result, ctorNumArgsMismatch(name, len(t), i) +} + +// ctorInterface is like ctor but returns []interface{}. +func (d *decodeState) ctorInterface() []interface{} { + var v = make([]interface{}, 0) + for { + // Look ahead for ) - can only happen on first iteration. + op := d.scanWhile(scanSkipSpace) + if op == scanEndCtor { + break + } + + // Back up so d.value can have the byte we just read. + d.off-- + d.scan.undo(op) + + v = append(v, d.valueInterface(false)) + + // Next token must be , or ). + op = d.scanWhile(scanSkipSpace) + if op == scanEndCtor { + break + } + if op != scanCtorArg { + d.error(errPhase) + } + } + return v +} + +// Returns a descriptive error message if the number of arguments given +// to the constructor do not match what is expected. +func ctorNumArgsMismatch(name string, expected, actual int) error { + if expected == actual { + return nil + } + + quantifier := "" + if expected > 1 { + quantifier = "s" + } + return fmt.Errorf(CtorNumArgsErrorf, expected, quantifier, name, actual) +} diff --git a/src/mongo/gotools/common/json/consts.go b/src/mongo/gotools/common/json/consts.go new file mode 100644 index 00000000000..ae7f7e6d31a --- /dev/null +++ b/src/mongo/gotools/common/json/consts.go @@ -0,0 +1,7 @@ +package json + +const ( + ArrayStart = '[' + ArraySep = ',' + ArrayEnd = ']' +) diff --git a/src/mongo/gotools/common/json/csv_format.go b/src/mongo/gotools/common/json/csv_format.go new file mode 100644 index 00000000000..2228d227393 --- /dev/null +++ b/src/mongo/gotools/common/json/csv_format.go @@ -0,0 +1,84 @@ +package json + +import ( + "encoding/base64" + "fmt" + "time" +) + +const CSV_DATE_FORMAT = "2006-01-02T15:04:05.000Z" + +func (b BinData) String() string { + data, err := base64.StdEncoding.DecodeString(b.Base64) + if err != nil { + return "" // XXX: panic? + } + if b.Type == 0x02 { + data = data[4:] // skip the first 4 bytes + } + return fmt.Sprintf("%X", data) // use uppercase hexadecimal +} + +func (js JavaScript) String() string { + return js.Code +} + +func (d Date) String() string { + if d.isFormatable() { + n := int64(d) + t := time.Unix(n/1e3, n%1e3*1e6) + return t.UTC().Format(JSON_DATE_FORMAT) + } + // date.MarshalJSON always returns a nil err. + data, _ := d.MarshalJSON() + return string(data) +} + +func (d DBRef) String() string { + return fmt.Sprintf(`{ "$ref": "%v", "$id": %v, "$db": "%v" }`, + d.Collection, d.Id, d.Database) +} + +func (d DBPointer) String() string { + return fmt.Sprintf(`{ "$ref": "%v", "$id": %v }`, + d.Namespace, d.Id) +} + +func (f Float) String() string { + return fmt.Sprintf("%v", float64(f)) +} + +func (_ MinKey) String() string { + return "$MinKey" +} + +func (_ MaxKey) String() string { + return "$MaxKey" +} + +func (n NumberInt) String() string { + return fmt.Sprintf("%v", int32(n)) +} + +func (n NumberLong) String() string { + return fmt.Sprintf("%v", int64(n)) +} + +// Assumes that o represents a valid ObjectId +// (composed of 24 hexadecimal characters). +func (o ObjectId) String() string { + return fmt.Sprintf("ObjectId(%v)", string(o)) +} + +func (r RegExp) String() string { + return fmt.Sprintf("/%v/%v", r.Pattern, r.Options) +} + +func (t Timestamp) String() string { + return fmt.Sprintf(`{ "$timestamp": { "t": %v, "i": %v } }`, + t.Seconds, t.Increment) +} + +func (_ Undefined) String() string { + return `{ "$undefined": true }` +} diff --git a/src/mongo/gotools/common/json/date.go b/src/mongo/gotools/common/json/date.go new file mode 100644 index 00000000000..dd310a675ee --- /dev/null +++ b/src/mongo/gotools/common/json/date.go @@ -0,0 +1,80 @@ +package json + +import ( + "fmt" + "github.com/mongodb/mongo-tools/common/util" + "reflect" +) + +// Transition functions for recognizing Date. +// Adapted from encoding/json/scanner.go. + +// stateDa is the state after reading `Da`. +func stateDa(s *scanner, c int) int { + if c == 't' { + s.step = stateDat + return scanContinue + } + return s.error(c, "in literal Date (expecting 't')") +} + +// stateDat is the state after reading `Dat`. +func stateDat(s *scanner, c int) int { + if c == 'e' { + s.step = stateConstructor + return scanContinue + } + return s.error(c, "in literal Date (expecting 'e')") +} + +// Decodes a Date literal stored in the underlying byte data into v. +func (d *decodeState) storeDate(v reflect.Value) { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginCtor { + d.error(fmt.Errorf("expected beginning of constructor")) + } + args, err := d.ctor("Date", []reflect.Type{dateType}) + if err != nil { + d.error(err) + } + switch kind := v.Kind(); kind { + case reflect.Interface: + v.Set(args[0]) + default: + d.error(fmt.Errorf("cannot store %v value into %v type", dateType, kind)) + } +} + +// Returns a Date literal from the underlying byte data. +func (d *decodeState) getDate() interface{} { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginCtor { + d.error(fmt.Errorf("expected beginning of constructor")) + } + + // Prevent d.convertNumber() from parsing the argument as a float64. + useNumber := d.useNumber + d.useNumber = true + + args := d.ctorInterface() + if err := ctorNumArgsMismatch("Date", 1, len(args)); err != nil { + d.error(err) + } + arg0num, isNumber := args[0].(Number) + if !isNumber { + // validate the date format of the string + _, err := util.FormatDate(args[0].(string)) + if err != nil { + d.error(fmt.Errorf("unexpected ISODate format")) + } + d.useNumber = useNumber + return ISODate(args[0].(string)) + } + arg0, err := arg0num.Int64() + if err != nil { + d.error(fmt.Errorf("expected int64 for first argument of Date constructor")) + } + + d.useNumber = useNumber + return Date(arg0) +} diff --git a/src/mongo/gotools/common/json/date_test.go b/src/mongo/gotools/common/json/date_test.go new file mode 100644 index 00000000000..b606fb62f43 --- /dev/null +++ b/src/mongo/gotools/common/json/date_test.go @@ -0,0 +1,99 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestDateValue(t *testing.T) { + + Convey("When unmarshalling JSON with Date values", t, func() { + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "Date(123)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(Date) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, Date(123)) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + value1, value2, value3 := "Date(123)", "Date(456)", "Date(789)" + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value1, key2, value2, key3, value3) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue1, ok := jsonMap[key1].(Date) + So(ok, ShouldBeTrue) + So(jsonValue1, ShouldEqual, Date(123)) + + jsonValue2, ok := jsonMap[key2].(Date) + So(ok, ShouldBeTrue) + So(jsonValue2, ShouldEqual, Date(456)) + + jsonValue3, ok := jsonMap[key3].(Date) + So(ok, ShouldBeTrue) + So(jsonValue3, ShouldEqual, Date(789)) + }) + + Convey("works in an array", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "Date(42)" + data := fmt.Sprintf(`{"%v":[%v,%v,%v]}`, + key, value, value, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonArray, ok := jsonMap[key].([]interface{}) + So(ok, ShouldBeTrue) + + for _, _jsonValue := range jsonArray { + jsonValue, ok := _jsonValue.(Date) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, Date(42)) + } + }) + + Convey("cannot use string as argument", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `Date("123")` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + }) + + Convey("can specify argument in hexadecimal", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "Date(0x5f)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(Date) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, Date(0x5f)) + }) + }) +} diff --git a/src/mongo/gotools/common/json/dbpointer.go b/src/mongo/gotools/common/json/dbpointer.go new file mode 100644 index 00000000000..24582038576 --- /dev/null +++ b/src/mongo/gotools/common/json/dbpointer.go @@ -0,0 +1,71 @@ +package json + +import ( + "fmt" + "gopkg.in/mgo.v2/bson" + "reflect" +) + +// Transition functions for recognizing DBPointer. +// Adapted from encoding/json/scanner.go. + +// stateDB is the state after reading `DB`. +func stateDBP(s *scanner, c int) int { + if c == 'o' { + s.step = generateState("DBPointer", []byte("inter"), stateConstructor) + return scanContinue + } + return s.error(c, "in literal DBPointer (expecting 'o')") +} + +// Decodes a DBRef literal stored in the underlying byte data into v. +func (d *decodeState) storeDBPointer(v reflect.Value) { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginCtor { + d.error(fmt.Errorf("expected beginning of constructor")) + } + + args := d.ctorInterface() + if len(args) != 2 { + d.error(fmt.Errorf("expected 2 arguments to DBPointer constructor, but %v received", len(args))) + } + switch kind := v.Kind(); kind { + case reflect.Interface: + arg0, ok := args[0].(string) + if !ok { + d.error(fmt.Errorf("expected first argument to DBPointer to be of type string")) + } + arg1, ok := args[1].(ObjectId) + if !ok { + d.error(fmt.Errorf("expected second argument to DBPointer to be of type ObjectId, but ended up being %t", args[1])) + } + id := bson.ObjectIdHex(string(arg1)) + v.Set(reflect.ValueOf(DBPointer{arg0, id})) + default: + d.error(fmt.Errorf("cannot store %v value into %v type", dbPointerType, kind)) + } +} + +// Returns a DBRef literal from the underlying byte data. +func (d *decodeState) getDBPointer() interface{} { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginCtor { + d.error(fmt.Errorf("expected beginning of constructor")) + } + + args := d.ctorInterface() + if err := ctorNumArgsMismatch("DBPointer", 2, len(args)); err != nil { + d.error(err) + } + arg0, ok := args[0].(string) + if !ok { + d.error(fmt.Errorf("expected string for first argument of DBPointer constructor")) + } + arg1, ok := args[1].(ObjectId) + if !ok { + d.error(fmt.Errorf("expected ObjectId for second argument of DBPointer constructor")) + } + id := bson.ObjectIdHex(string(arg1)) + + return DBPointer{arg0, id} +} diff --git a/src/mongo/gotools/common/json/dbpointer_test.go b/src/mongo/gotools/common/json/dbpointer_test.go new file mode 100644 index 00000000000..9831f7028ac --- /dev/null +++ b/src/mongo/gotools/common/json/dbpointer_test.go @@ -0,0 +1,84 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "gopkg.in/mgo.v2/bson" + "testing" +) + +func TestDBPointerValue(t *testing.T) { + + Convey("Unmarshalling JSON with DBPointer values", t, func() { + key := "key" + value := `DBPointer("ref", ObjectId("552ffe9f5739878e73d116a9"))` + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(DBPointer) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, DBPointer{"ref", bson.ObjectIdHex("552ffe9f5739878e73d116a9")}) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + value2 := `DBPointer("ref2", ObjectId("552ffed95739878e73d116aa"))` + value3 := `DBPointer("ref3", ObjectId("552fff215739878e73d116ab"))` + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value, key2, value2, key3, value3) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue1, ok := jsonMap[key1].(DBPointer) + So(ok, ShouldBeTrue) + + So(jsonValue1, ShouldResemble, DBPointer{"ref", bson.ObjectIdHex("552ffe9f5739878e73d116a9")}) + + jsonValue2, ok := jsonMap[key2].(DBPointer) + So(ok, ShouldBeTrue) + So(jsonValue2, ShouldResemble, DBPointer{"ref2", bson.ObjectIdHex("552ffed95739878e73d116aa")}) + + jsonValue3, ok := jsonMap[key3].(DBPointer) + So(ok, ShouldBeTrue) + So(jsonValue3, ShouldResemble, DBPointer{"ref3", bson.ObjectIdHex("552fff215739878e73d116ab")}) + }) + + Convey("works in an array", func() { + var jsonMap map[string]interface{} + + data := fmt.Sprintf(`{"%v":[%v,%v,%v]}`, + key, value, value, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + jsonArray, ok := jsonMap[key].([]interface{}) + So(ok, ShouldBeTrue) + + for _, _jsonValue := range jsonArray { + jsonValue, ok := _jsonValue.(DBPointer) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, DBPointer{"ref", bson.ObjectIdHex("552ffe9f5739878e73d116a9")}) + } + }) + + Convey("will not accept an $id type that is not an ObjectId", func() { + value := `DBPointer("ref", 4)` + var jsonMap map[string]interface{} + + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + }) + + }) +} diff --git a/src/mongo/gotools/common/json/dbref.go b/src/mongo/gotools/common/json/dbref.go new file mode 100644 index 00000000000..23ee1eb2d50 --- /dev/null +++ b/src/mongo/gotools/common/json/dbref.go @@ -0,0 +1,69 @@ +package json + +import ( + "fmt" + "reflect" +) + +// Transition functions for recognizing DBRef and Dbref. +// Adapted from encoding/json/scanner.go. + +// stateDB is the state after reading `DB`. +func stateDBR(s *scanner, c int) int { + if c == 'e' { + s.step = generateState("DBRef", []byte("f"), stateConstructor) + return scanContinue + } + return s.error(c, "in literal DBRef (expecting 'e')") +} + +// stateDb is the state after reading `Db`. +func stateDb(s *scanner, c int) int { + if c == 'r' { + s.step = generateState("Dbref", []byte("ef"), stateConstructor) + return scanContinue + } + return s.error(c, "in literal Dbref (expecting 'r')") +} + +// Decodes a DBRef literal stored in the underlying byte data into v. +func (d *decodeState) storeDBRef(v reflect.Value) { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginCtor { + d.error(fmt.Errorf("expected beginning of constructor")) + } + + args := d.ctorInterface() + if len(args) != 2 { + d.error(fmt.Errorf("expected 2 arguments to DBRef constructor, but %v received", len(args))) + } + switch kind := v.Kind(); kind { + case reflect.Interface: + arg0, ok := args[0].(string) + if !ok { + d.error(fmt.Errorf("expected first argument to DBRef to be of type string")) + } + arg1 := args[1] + v.Set(reflect.ValueOf(DBRef{arg0, arg1, ""})) + default: + d.error(fmt.Errorf("cannot store %v value into %v type", dbRefType, kind)) + } +} + +// Returns a DBRef literal from the underlying byte data. +func (d *decodeState) getDBRef() interface{} { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginCtor { + d.error(fmt.Errorf("expected beginning of constructor")) + } + + args := d.ctorInterface() + if err := ctorNumArgsMismatch("DBRef", 2, len(args)); err != nil { + d.error(err) + } + arg0, ok := args[0].(string) + if !ok { + d.error(fmt.Errorf("expected string for first argument of DBRef constructor")) + } + return DBRef{arg0, args[1], ""} +} diff --git a/src/mongo/gotools/common/json/dbref_test.go b/src/mongo/gotools/common/json/dbref_test.go new file mode 100644 index 00000000000..e1f4c6bb287 --- /dev/null +++ b/src/mongo/gotools/common/json/dbref_test.go @@ -0,0 +1,347 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "math" + "testing" +) + +func TestDBRefValue(t *testing.T) { + + Convey("When unmarshalling JSON with DBRef values", t, func() { + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `DBRef("ref", "123")` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, DBRef{"ref", "123", ""}) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + value1, value2, value3 := `DBRef("ref1", "123")`, + `DBRef("ref2", "456")`, `DBRef("ref3", "789")` + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value1, key2, value2, key3, value3) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue1, ok := jsonMap[key1].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue1, ShouldResemble, DBRef{"ref1", "123", ""}) + + jsonValue2, ok := jsonMap[key2].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue2, ShouldResemble, DBRef{"ref2", "456", ""}) + + jsonValue3, ok := jsonMap[key3].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue3, ShouldResemble, DBRef{"ref3", "789", ""}) + }) + + Convey("works in an array", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `DBRef("ref", "42")` + data := fmt.Sprintf(`{"%v":[%v,%v,%v]}`, + key, value, value, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonArray, ok := jsonMap[key].([]interface{}) + So(ok, ShouldBeTrue) + + for _, _jsonValue := range jsonArray { + jsonValue, ok := _jsonValue.(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, DBRef{"ref", "42", ""}) + } + }) + + Convey("can use alternative capitalization ('Dbref')", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `Dbref("ref", "123")` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, DBRef{"ref", "123", ""}) + }) + + Convey("can have any extended JSON value for id parameter", func() { + + Convey("a null literal", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `DBRef("ref", null)` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, DBRef{"ref", nil, ""}) + }) + + Convey("a true literal", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `DBRef("ref", true)` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, DBRef{"ref", true, ""}) + }) + + Convey("a false literal", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `DBRef("ref", false)` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, DBRef{"ref", false, ""}) + }) + + Convey("an undefined literal", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `DBRef("ref", undefined)` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, DBRef{"ref", Undefined{}, ""}) + }) + + Convey("a NaN literal", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `DBRef("ref", NaN)` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue.Collection, ShouldEqual, "ref") + + id, ok := jsonValue.Id.(float64) + So(ok, ShouldBeTrue) + So(math.IsNaN(id), ShouldBeTrue) + + }) + + Convey("an Infinity literal", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `DBRef("ref", Infinity)` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue.Collection, ShouldEqual, "ref") + + id, ok := jsonValue.Id.(float64) + So(ok, ShouldBeTrue) + So(math.IsInf(id, 1), ShouldBeTrue) + + }) + + Convey("a MinKey literal", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `DBRef("ref", MinKey)` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, DBRef{"ref", MinKey{}, ""}) + }) + + Convey("a MaxKey literal", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `DBRef("ref", MaxKey)` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, DBRef{"ref", MaxKey{}, ""}) + }) + + Convey("an ObjectId object", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `DBRef("ref", ObjectId("123"))` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, DBRef{"ref", ObjectId("123"), ""}) + }) + + Convey("a NumberInt object", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `DBRef("ref", NumberInt(123))` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, DBRef{"ref", NumberInt(123), ""}) + }) + + Convey("a NumberLong object", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `DBRef("ref", NumberLong(123))` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, DBRef{"ref", NumberLong(123), ""}) + }) + + Convey("a RegExp object", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `DBRef("ref", RegExp("xyz", "i"))` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, DBRef{"ref", RegExp{"xyz", "i"}, ""}) + }) + + Convey("a regular expression literal", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `DBRef("ref", /xyz/i)` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, DBRef{"ref", RegExp{"xyz", "i"}, ""}) + }) + + Convey("a Timestamp object", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `DBRef("ref", Timestamp(123, 321))` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, DBRef{"ref", Timestamp{123, 321}, ""}) + }) + + Convey("a string literal", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `DBRef("ref", "xyz")` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, DBRef{"ref", "xyz", ""}) + }) + + Convey("a numeric literal", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `DBRef("ref", 123)` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue.Collection, ShouldEqual, "ref") + + id, ok := jsonValue.Id.(int32) + So(ok, ShouldBeTrue) + So(id, ShouldAlmostEqual, 123) + }) + }) + }) +} diff --git a/src/mongo/gotools/common/json/decode.go b/src/mongo/gotools/common/json/decode.go new file mode 100644 index 00000000000..e2e6418ed5f --- /dev/null +++ b/src/mongo/gotools/common/json/decode.go @@ -0,0 +1,1273 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Represents JSON data structure using native Go types: booleans, floats, +// strings, arrays, and maps. + +package json + +import ( + "bytes" + "encoding" + "encoding/base64" + "errors" + "fmt" + "gopkg.in/mgo.v2/bson" + "math" + "reflect" + "runtime" + "strconv" + "unicode" + "unicode/utf16" + "unicode/utf8" +) + +// Unmarshal parses the JSON-encoded data and stores the result +// in the value pointed to by v. +// +// Unmarshal uses the inverse of the encodings that +// Marshal uses, allocating maps, slices, and pointers as necessary, +// with the following additional rules: +// +// To unmarshal JSON into a pointer, Unmarshal first handles the case of +// the JSON being the JSON literal null. In that case, Unmarshal sets +// the pointer to nil. Otherwise, Unmarshal unmarshals the JSON into +// the value pointed at by the pointer. If the pointer is nil, Unmarshal +// allocates a new value for it to point to. +// +// To unmarshal JSON into a struct, Unmarshal matches incoming object +// keys to the keys used by Marshal (either the struct field name or its tag), +// preferring an exact match but also accepting a case-insensitive match. +// +// To unmarshal JSON into an interface value, +// Unmarshal stores one of these in the interface value: +// +// bool, for JSON booleans +// float64, for JSON numbers +// string, for JSON strings +// []interface{}, for JSON arrays +// map[string]interface{}, for JSON objects +// nil for JSON null +// +// If a JSON value is not appropriate for a given target type, +// or if a JSON number overflows the target type, Unmarshal +// skips that field and completes the unmarshalling as best it can. +// If no more serious errors are encountered, Unmarshal returns +// an UnmarshalTypeError describing the earliest such error. +// +// The JSON null value unmarshals into an interface, map, pointer, or slice +// by setting that Go value to nil. Because null is often used in JSON to mean +// ``not present,'' unmarshaling a JSON null into any other Go type has no effect +// on the value and produces no error. +// +// When unmarshaling quoted strings, invalid UTF-8 or +// invalid UTF-16 surrogate pairs are not treated as an error. +// Instead, they are replaced by the Unicode replacement +// character U+FFFD. +// +func Unmarshal(data []byte, v interface{}) error { + // Check for well-formedness. + // Avoids filling out half a data structure + // before discovering a JSON syntax error. + var d decodeState + err := checkValid(data, &d.scan) + if err != nil { + return err + } + + d.init(data) + return d.unmarshal(v) +} + +func UnmarshalMap(data []byte) (map[string]interface{}, error) { + // Check for well-formedness. + // Avoids filling out half a data structure + // before discovering a JSON syntax error. + var d decodeState + err := checkValid(data, &d.scan) + if err != nil { + return nil, err + } + + d.init(data) + return d.unmarshalMap() +} + +func UnmarshalBsonD(data []byte) (bson.D, error) { + // Check for well-formedness. + // Avoids filling out half a data structure + // before discovering a JSON syntax error. + var d decodeState + err := checkValid(data, &d.scan) + if err != nil { + return nil, err + } + + d.init(data) + return d.unmarshalBsonD() +} + +// Unmarshaler is the interface implemented by objects +// that can unmarshal a JSON description of themselves. +// The input can be assumed to be a valid encoding of +// a JSON value. UnmarshalJSON must copy the JSON data +// if it wishes to retain the data after returning. +type Unmarshaler interface { + UnmarshalJSON([]byte) error +} + +// An UnmarshalTypeError describes a JSON value that was +// not appropriate for a value of a specific Go type. +type UnmarshalTypeError struct { + Value string // description of JSON value - "bool", "array", "number -5" + Type reflect.Type // type of Go value it could not be assigned to +} + +func (e *UnmarshalTypeError) Error() string { + return "json: cannot unmarshal " + e.Value + " into Go value of type " + e.Type.String() +} + +// An UnmarshalFieldError describes a JSON object key that +// led to an unexported (and therefore unwritable) struct field. +// (No longer used; kept for compatibility.) +type UnmarshalFieldError struct { + Key string + Type reflect.Type + Field reflect.StructField +} + +func (e *UnmarshalFieldError) Error() string { + return "json: cannot unmarshal object key " + strconv.Quote(e.Key) + " into unexported field " + e.Field.Name + " of type " + e.Type.String() +} + +// An InvalidUnmarshalError describes an invalid argument passed to Unmarshal. +// (The argument to Unmarshal must be a non-nil pointer.) +type InvalidUnmarshalError struct { + Type reflect.Type +} + +func (e *InvalidUnmarshalError) Error() string { + if e.Type == nil { + return "json: Unmarshal(nil)" + } + + if e.Type.Kind() != reflect.Ptr { + return "json: Unmarshal(non-pointer " + e.Type.String() + ")" + } + return "json: Unmarshal(nil " + e.Type.String() + ")" +} + +func (d *decodeState) unmarshalMap() (out map[string]interface{}, err error) { + defer func() { + if r := recover(); r != nil { + if _, ok := r.(runtime.Error); ok { + panic(r) + } + err = r.(error) + } + }() + + d.scan.reset() + // We decode rv not rv.Elem because the Unmarshaler interface + // test must be applied at the top level of the value. + out = d.document() + return out, d.savedError +} + +func (d *decodeState) unmarshalBsonD() (out bson.D, err error) { + defer func() { + if r := recover(); r != nil { + if _, ok := r.(runtime.Error); ok { + panic(r) + } + err = r.(error) + } + }() + + d.scan.reset() + // We decode rv not rv.Elem because the Unmarshaler interface + // test must be applied at the top level of the value. + out = d.bsonDocument() + return out, d.savedError +} + +func (d *decodeState) unmarshal(v interface{}) (err error) { + defer func() { + if r := recover(); r != nil { + if _, ok := r.(runtime.Error); ok { + panic(r) + } + err = r.(error) + } + }() + + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Ptr || rv.IsNil() { + return &InvalidUnmarshalError{reflect.TypeOf(v)} + } + + d.scan.reset() + // We decode rv not rv.Elem because the Unmarshaler interface + // test must be applied at the top level of the value. + d.value(rv) + return d.savedError +} + +// A Number represents a JSON number literal. +type Number string + +// String returns the literal text of the number. +func (n Number) String() string { return string(n) } + +// Float64 returns the number as a float64. +func (n Number) Float64() (float64, error) { + return strconv.ParseFloat(string(n), 64) +} + +// Int32 returns the number as an int32. +func (n Number) Int32() (int32, error) { + x, err := n.Int64() + return int32(x), err +} + +// Int64 returns the number as an int64. +func (n Number) Int64() (int64, error) { + base := 10 + if isHexPrefix(string(n)) { + base = 0 // strconv.ParseInt will infer base 16 + } + return strconv.ParseInt(string(n), base, 64) +} + +// Uint8 returns the number as an uint8. +func (n Number) Uint8() (uint8, error) { + x, err := n.Uint64() + return uint8(x), err +} + +// Uint32 returns the number as an uint32. +func (n Number) Uint32() (uint32, error) { + x, err := n.Uint64() + return uint32(x), err +} + +// Uint64 returns the number as an uint64. +func (n Number) Uint64() (uint64, error) { + base := 10 + if isHexPrefix(string(n)) { + base = 0 // strconv.ParseUint will infer base 16 + } + return strconv.ParseUint(string(n), base, 64) +} + +// decodeState represents the state while decoding a JSON value. +type decodeState struct { + data []byte + off int // read offset in data + scan scanner + nextscan scanner // for calls to nextValue + savedError error + tempstr string // scratch space to avoid some allocations + useNumber bool +} + +// errPhase is used for errors that should not happen unless +// there is a bug in the JSON decoder or something is editing +// the data slice while the decoder executes. +var errPhase = errors.New("JSON decoder out of sync - data changing underfoot?") + +func (d *decodeState) init(data []byte) *decodeState { + d.data = data + d.off = 0 + d.savedError = nil + return d +} + +// error aborts the decoding by panicking with err. +func (d *decodeState) error(err error) { + panic(err) +} + +// saveError saves the first err it is called with, +// for reporting at the end of the unmarshal. +func (d *decodeState) saveError(err error) { + if d.savedError == nil { + d.savedError = err + } +} + +// next cuts off and returns the next full JSON value in d.data[d.off:]. +// The next value is known to be an object or array, not a literal. +func (d *decodeState) next() []byte { + c := d.data[d.off] + item, rest, err := nextValue(d.data[d.off:], &d.nextscan) + if err != nil { + d.error(err) + } + d.off = len(d.data) - len(rest) + + // Our scanner has seen the opening brace/bracket + // and thinks we're still in the middle of the object. + // invent a closing brace/bracket to get it out. + if c == '{' { + d.scan.step(&d.scan, '}') + } else { + d.scan.step(&d.scan, ']') + } + + return item +} + +// scanWhile processes bytes in d.data[d.off:] until it +// receives a scan code not equal to op. +// It updates d.off and returns the new scan code. +func (d *decodeState) scanWhile(op int) int { + var newOp int + for { + if d.off >= len(d.data) { + newOp = d.scan.eof() + d.off = len(d.data) + 1 // mark processed EOF with len+1 + } else { + c := int(d.data[d.off]) + d.off++ + newOp = d.scan.step(&d.scan, c) + } + if newOp != op { + break + } + } + return newOp +} + +func (d *decodeState) document() map[string]interface{} { + switch op := d.scanWhile(scanSkipSpace); op { + default: + d.error(errPhase) + return nil + case scanBeginObject: + return d.objectInterface() + } +} + +func (d *decodeState) bsonDocument() bson.D { + switch op := d.scanWhile(scanSkipSpace); op { + default: + d.error(errPhase) + return nil + case scanBeginObject: + return d.bsonDInterface() + } +} + +// value decodes a JSON value from d.data[d.off:] into the value. +// it updates d.off to point past the decoded value. +func (d *decodeState) value(v reflect.Value) { + if !v.IsValid() { + _, rest, err := nextValue(d.data[d.off:], &d.nextscan) + if err != nil { + d.error(err) + } + d.off = len(d.data) - len(rest) + + // d.scan thinks we're still at the beginning of the item. + // Feed in an empty string - the shortest, simplest value - + // so that it knows we got to the end of the value. + if d.scan.redo { + // rewind. + d.scan.redo = false + d.scan.step = stateBeginValue + } + d.scan.step(&d.scan, '"') + d.scan.step(&d.scan, '"') + + n := len(d.scan.parseState) + if n > 0 && d.scan.parseState[n-1] == parseObjectKey { + // d.scan thinks we just read an object key; finish the object + d.scan.step(&d.scan, ':') + d.scan.step(&d.scan, '"') + d.scan.step(&d.scan, '"') + d.scan.step(&d.scan, '}') + } + + return + } + + switch op := d.scanWhile(scanSkipSpace); op { + default: + d.error(errPhase) + + case scanBeginArray: + d.array(v) + + case scanBeginObject: + d.object(v) + + case scanBeginLiteral: + d.literal(v) + } +} + +// indirect walks down v allocating pointers as needed, +// until it gets to a non-pointer. +// if it encounters an Unmarshaler, indirect stops and returns that. +// if decodingNull is true, indirect stops at the last pointer so it can be set to nil. +func (d *decodeState) indirect(v reflect.Value, decodingNull bool) (Unmarshaler, encoding.TextUnmarshaler, reflect.Value) { + // If v is a named type and is addressable, + // start with its address, so that if the type has pointer methods, + // we find them. + if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() { + v = v.Addr() + } + for { + // Load value from interface, but only if the result will be + // usefully addressable. + if v.Kind() == reflect.Interface && !v.IsNil() { + e := v.Elem() + if e.Kind() == reflect.Ptr && !e.IsNil() && (!decodingNull || e.Elem().Kind() == reflect.Ptr) { + v = e + continue + } + } + + if v.Kind() != reflect.Ptr { + break + } + + if v.Elem().Kind() != reflect.Ptr && decodingNull && v.CanSet() { + break + } + if v.IsNil() { + v.Set(reflect.New(v.Type().Elem())) + } + if v.Type().NumMethod() > 0 { + if u, ok := v.Interface().(Unmarshaler); ok { + return u, nil, reflect.Value{} + } + if u, ok := v.Interface().(encoding.TextUnmarshaler); ok { + return nil, u, reflect.Value{} + } + } + v = v.Elem() + } + return nil, nil, v +} + +// array consumes an array from d.data[d.off-1:], decoding into the value v. +// the first byte of the array ('[') has been read already. +func (d *decodeState) array(v reflect.Value) { + // Check for unmarshaler. + u, ut, pv := d.indirect(v, false) + if u != nil { + d.off-- + err := u.UnmarshalJSON(d.next()) + if err != nil { + d.error(err) + } + return + } + if ut != nil { + d.saveError(&UnmarshalTypeError{"array", v.Type()}) + d.off-- + d.next() + return + } + + v = pv + + // Check type of target. + switch v.Kind() { + case reflect.Interface: + if v.NumMethod() == 0 { + // Decoding into nil interface? Switch to non-reflect code. + v.Set(reflect.ValueOf(d.arrayInterface(false))) + return + } + // Otherwise it's invalid. + fallthrough + default: + d.saveError(&UnmarshalTypeError{"array", v.Type()}) + d.off-- + d.next() + return + case reflect.Array: + case reflect.Slice: + break + } + + i := 0 + for { + // Look ahead for ] - can only happen on first iteration. + op := d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + + // Back up so d.value can have the byte we just read. + d.off-- + d.scan.undo(op) + + // Get element of array, growing if necessary. + if v.Kind() == reflect.Slice { + // Grow slice if necessary + if i >= v.Cap() { + newcap := v.Cap() + v.Cap()/2 + if newcap < 4 { + newcap = 4 + } + newv := reflect.MakeSlice(v.Type(), v.Len(), newcap) + reflect.Copy(newv, v) + v.Set(newv) + } + if i >= v.Len() { + v.SetLen(i + 1) + } + } + + if i < v.Len() { + // Decode into element. + d.value(v.Index(i)) + } else { + // Ran out of fixed array: skip. + d.value(reflect.Value{}) + } + i++ + + // Next token must be , or ]. + op = d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + if op != scanArrayValue { + d.error(errPhase) + } + } + + if i < v.Len() { + if v.Kind() == reflect.Array { + // Array. Zero the rest. + z := reflect.Zero(v.Type().Elem()) + for ; i < v.Len(); i++ { + v.Index(i).Set(z) + } + } else { + v.SetLen(i) + } + } + if i == 0 && v.Kind() == reflect.Slice { + v.Set(reflect.MakeSlice(v.Type(), 0, 0)) + } +} + +// object consumes an object from d.data[d.off-1:], decoding into the value v. +// the first byte of the object ('{') has been read already. +func (d *decodeState) object(v reflect.Value) { + + // Check for unmarshaler. + u, ut, pv := d.indirect(v, false) + if u != nil { + d.off-- + err := u.UnmarshalJSON(d.next()) + if err != nil { + d.error(err) + } + return + } + if ut != nil { + d.saveError(&UnmarshalTypeError{"object", v.Type()}) + d.off-- + d.next() // skip over { } in input + return + } + v = pv + + // Decoding into nil interface? Switch to non-reflect code. + if v.Kind() == reflect.Interface && v.NumMethod() == 0 { + v.Set(reflect.ValueOf(d.objectInterface())) + return + } + + // Check type of target: struct or map[string]T + switch v.Kind() { + case reflect.Map: + // map must have string kind + t := v.Type() + if t.Key().Kind() != reflect.String { + d.saveError(&UnmarshalTypeError{"object", v.Type()}) + break + } + if v.IsNil() { + v.Set(reflect.MakeMap(t)) + } + + case reflect.Struct: + // do nothing + + case reflect.Slice: + // this is only a valid case if the output type is a bson.D + t := v.Type() + if t == orderedBSONType { + v.Set(reflect.ValueOf(d.bsonDInterface())) + return + } + fallthrough // can't unmarshal into a regular slice, goto "default" error case + + default: + d.saveError(&UnmarshalTypeError{"object", v.Type()}) + d.off-- + d.next() // skip over { } in input + return + } + + var mapElem reflect.Value + + for { + // Read opening " of string key or closing }. + op := d.scanWhile(scanSkipSpace) + if op == scanEndObject { + // closing } - can only happen on first iteration. + break + } + if op != scanBeginLiteral { + d.error(errPhase) + } + + // Read key. + start := d.off - 1 + op = d.scanWhile(scanContinue) + item := d.data[start : d.off-1] + key, ok := maybeUnquoteBytes(item) + if !ok { + d.error(errPhase) + } + + // Figure out field corresponding to key. + var subv reflect.Value + destring := false // whether the value is wrapped in a string to be decoded first + + if v.Kind() == reflect.Map { + elemType := v.Type().Elem() + if !mapElem.IsValid() { + mapElem = reflect.New(elemType).Elem() + } else { + mapElem.Set(reflect.Zero(elemType)) + } + subv = mapElem + } else { + var f *field + fields := cachedTypeFields(v.Type()) + for i := range fields { + ff := &fields[i] + if bytes.Equal(ff.nameBytes, key) { + f = ff + break + } + if f == nil && ff.equalFold(ff.nameBytes, key) { + f = ff + } + } + if f != nil { + subv = v + destring = f.quoted + for _, i := range f.index { + if subv.Kind() == reflect.Ptr { + if subv.IsNil() { + subv.Set(reflect.New(subv.Type().Elem())) + } + subv = subv.Elem() + } + subv = subv.Field(i) + } + } + } + + // Read : before value. + if op == scanSkipSpace { + op = d.scanWhile(scanSkipSpace) + } + if op != scanObjectKey { + d.error(errPhase) + } + // Read value. + if destring { + d.value(reflect.ValueOf(&d.tempstr)) + d.literalStore([]byte(d.tempstr), subv, true) + d.tempstr = "" // Zero scratch space for successive values. + } else { + d.value(subv) + } + + // Write value back to map; + // if using struct, subv points into struct already. + if v.Kind() == reflect.Map { + kv := reflect.ValueOf(key).Convert(v.Type().Key()) + v.SetMapIndex(kv, subv) + } + + // Next token must be , or }. + op = d.scanWhile(scanSkipSpace) + if op == scanEndObject { + break + } + if op != scanObjectValue { + d.error(errPhase) + } + } +} + +// literal consumes a literal from d.data[d.off-1:], decoding into the value v. +// The first byte of the literal has been read already +// (that's how the caller knows it's a literal). +func (d *decodeState) literal(v reflect.Value) { + // All bytes inside literal return scanContinue op code. + start := d.off - 1 + op := d.scanWhile(scanContinue) + + // Scan read one byte too far; back up. + d.off-- + d.scan.undo(op) + + d.literalStore(d.data[start:d.off], v, false) +} + +// convertNumber converts the number literal s to an int32, int64, a float64, +// or a Number depending on the setting of d.useNumber and whether the +// string is specified in hexadecimal. It does this by parsing the string to see if it +// can an integer, if not it is treated as a float. If the integer is within the bounds of an int32 it +// is returned as an int32. +func (d *decodeState) convertNumber(s string) (interface{}, error) { + if d.useNumber { + return Number(s), nil + } + parsedInteger, err := strconv.ParseInt(s, 0, 64) + if err != nil { + parsedFloat, err := strconv.ParseFloat(s, 64) + if err != nil { + return nil, &UnmarshalTypeError{"number " + s, reflect.TypeOf(0.0)} + } + return parsedFloat, nil + } + + if parsedInteger <= math.MaxInt32 && parsedInteger >= math.MinInt32 { + return int32(parsedInteger), nil + } + return int64(parsedInteger), nil + +} + +var numberType = reflect.TypeOf(Number("")) + +// literalStore decodes a literal stored in item into v. +// +// fromQuoted indicates whether this literal came from unwrapping a +// string from the ",string" struct tag option. this is used only to +// produce more helpful error messages. +func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool) { + // Check for unmarshaler. + if len(item) == 0 { + // Empty string given + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + return + } + wantptr := isNull(item) // null + u, ut, pv := d.indirect(v, wantptr) + if u != nil { + err := u.UnmarshalJSON(item) + if err != nil { + d.error(err) + } + return + } + if ut != nil { + if item[0] != '"' { + if fromQuoted { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.saveError(&UnmarshalTypeError{"string", v.Type()}) + } + } + s, ok := unquoteBytes(item) + if !ok { + if fromQuoted { + d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.error(errPhase) + } + } + err := ut.UnmarshalText(s) + if err != nil { + d.error(err) + } + return + } + + v = pv + + switch c := item[0]; { + case isNull(item): // null + switch v.Kind() { + case reflect.Interface, reflect.Ptr, reflect.Map, reflect.Slice: + v.Set(reflect.Zero(v.Type())) + // otherwise, ignore null for primitives/string + } + case c == 't', c == 'f': // true, false + value := c == 't' + switch v.Kind() { + default: + if fromQuoted { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.saveError(&UnmarshalTypeError{"bool", v.Type()}) + } + case reflect.Bool: + v.SetBool(value) + case reflect.Interface: + if v.NumMethod() == 0 { + v.Set(reflect.ValueOf(value)) + } else { + d.saveError(&UnmarshalTypeError{"bool", v.Type()}) + } + } + + case c == '"', c == '\'': // string + + s, ok := unquoteBytes(item) + if !ok { + if fromQuoted { + d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.error(errPhase) + } + } + switch v.Kind() { + default: + d.saveError(&UnmarshalTypeError{"string", v.Type()}) + case reflect.Slice: + if v.Type() != byteSliceType { + d.saveError(&UnmarshalTypeError{"string", v.Type()}) + break + } + b := make([]byte, base64.StdEncoding.DecodedLen(len(s))) + n, err := base64.StdEncoding.Decode(b, s) + if err != nil { + d.saveError(err) + break + } + v.Set(reflect.ValueOf(b[0:n])) + case reflect.String: + v.SetString(string(s)) + case reflect.Interface: + if v.NumMethod() == 0 { + v.Set(reflect.ValueOf(string(s))) + } else { + d.saveError(&UnmarshalTypeError{"string", v.Type()}) + } + } + + case isNumber(item): + s := string(item) + switch v.Kind() { + default: + + if v.Kind() == reflect.String && v.Type() == numberType { + v.SetString(s) + break + } + if fromQuoted { + d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.error(&UnmarshalTypeError{"number", v.Type()}) + } + case reflect.Interface: + n, err := d.convertNumber(s) + if err != nil { + d.saveError(err) + break + } + if v.NumMethod() != 0 { + d.saveError(&UnmarshalTypeError{"number", v.Type()}) + break + } + v.Set(reflect.ValueOf(n)) + + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + base := 10 + if isHexPrefix(s) { + base = 0 // strconv.ParseInt will infer base 16 + } + n, err := strconv.ParseInt(s, base, 64) + if err != nil || v.OverflowInt(n) { + d.saveError(&UnmarshalTypeError{"number " + s, v.Type()}) + break + } + v.SetInt(n) + + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + base := 10 + if isHexPrefix(s) { + base = 0 // strconv.ParseUint will infer base 16 + } + n, err := strconv.ParseUint(s, base, 64) + if err != nil || v.OverflowUint(n) { + d.saveError(&UnmarshalTypeError{"number " + s, v.Type()}) + break + } + v.SetUint(n) + + case reflect.Float32, reflect.Float64: + n, err := strconv.ParseFloat(s, v.Type().Bits()) + if err != nil || v.OverflowFloat(n) { + d.saveError(&UnmarshalTypeError{"number " + s, v.Type()}) + break + } + v.SetFloat(n) + } + + default: + if ok := d.storeExtendedLiteral(item, v, fromQuoted); !ok { + if fromQuoted { + d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + } else { + d.error(errPhase) + } + } + } + +} + +// The xxxInterface routines build up a value to be stored +// in an empty interface. They are not strictly necessary, +// but they avoid the weight of reflection in this common case. + +// valueInterface is like value but returns interface{}. It takes a boolean +// parameter denoting whether or not the value is being unmarshalled within +// a bson.D, so that bson.Ds can be the default object type when +// they are inside other bson.D documents. +func (d *decodeState) valueInterface(insideBSOND bool) interface{} { + switch d.scanWhile(scanSkipSpace) { + default: + d.error(errPhase) + panic("unreachable") + case scanBeginArray: + return d.arrayInterface(insideBSOND) + case scanBeginObject: + if insideBSOND { + return d.bsonDInterface() + } + return d.objectInterface() + case scanBeginLiteral: + return d.literalInterface() + } +} + +// arrayInterface is like array but returns []interface{}. It takes a boolean +// parameter denoting whether or not the value is being unmarshalled within +// a bson.D, so that bson.Ds can be the default object type when +// they are inside other bson.D documents. +func (d *decodeState) arrayInterface(insideBSOND bool) []interface{} { + var v = make([]interface{}, 0) + for { + // Look ahead for ] - can only happen on first iteration. + op := d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + + // Back up so d.value can have the byte we just read. + d.off-- + d.scan.undo(op) + + v = append(v, d.valueInterface(insideBSOND)) + + // Next token must be , or ]. + op = d.scanWhile(scanSkipSpace) + if op == scanEndArray { + break + } + if op != scanArrayValue { + d.error(errPhase) + } + } + return v +} + +// bsonDInterface is like object but returns bson.D{}. +func (d *decodeState) bsonDInterface() bson.D { + m := bson.D{} + for { + // Read opening " of string key or closing }. + op := d.scanWhile(scanSkipSpace) + if op == scanEndObject { + // closing } - can only happen on first iteration. + break + } + if op != scanBeginLiteral { + d.error(errPhase) + } + + // Read string key. + start := d.off - 1 + op = d.scanWhile(scanContinue) + item := d.data[start : d.off-1] + key, ok := maybeUnquote(item) + if !ok { + d.error(errPhase) + } + + // Read : before value. + if op == scanSkipSpace { + op = d.scanWhile(scanSkipSpace) + } + if op != scanObjectKey { + d.error(errPhase) + } + + // Read value. + m = append(m, bson.DocElem{Name: key, Value: d.valueInterface(true)}) + + // Next token must be , or }. + op = d.scanWhile(scanSkipSpace) + if op == scanEndObject { + break + } + if op != scanObjectValue { + d.error(errPhase) + } + } + return m +} + +// objectInterface is like object but returns map[string]interface{}. +func (d *decodeState) objectInterface() map[string]interface{} { + m := make(map[string]interface{}) + for { + // Read opening " of string key or closing }. + op := d.scanWhile(scanSkipSpace) + if op == scanEndObject { + // closing } - can only happen on first iteration. + break + } + if op != scanBeginLiteral { + d.error(errPhase) + } + + // Read string key. + start := d.off - 1 + op = d.scanWhile(scanContinue) + item := d.data[start : d.off-1] + key, ok := maybeUnquote(item) + if !ok { + d.error(errPhase) + } + + // Read : before value. + if op == scanSkipSpace { + op = d.scanWhile(scanSkipSpace) + } + if op != scanObjectKey { + d.error(errPhase) + } + + // Read value. + m[key] = d.valueInterface(false) + + // Next token must be , or }. + op = d.scanWhile(scanSkipSpace) + if op == scanEndObject { + break + } + if op != scanObjectValue { + d.error(errPhase) + } + } + return m +} + +// literalInterface is like literal but returns an interface value. +func (d *decodeState) literalInterface() interface{} { + // All bytes inside literal return scanContinue op code. + start := d.off - 1 + op := d.scanWhile(scanContinue) + + // Scan read one byte too far; back up. + d.off-- + d.scan.undo(op) + item := d.data[start:d.off] + + switch c := item[0]; { + case isNull(item): // null + return nil + + case c == 't', c == 'f': // true, false + return c == 't' + + case c == '"', c == '\'': // string + s, ok := unquote(item) + if !ok { + d.error(errPhase) + } + return s + + case isNumber(item): // number + n, err := d.convertNumber(string(item)) + if err != nil { + d.saveError(err) + } + return n + + default: + if value, ok := d.getExtendedLiteral(item); ok { + return value + } + d.error(errPhase) + panic("unreachable") + } +} + +// getu4 decodes \uXXXX from the beginning of s, returning the hex value, +// or it returns -1. +func getu4(s []byte) rune { + if len(s) < 6 || s[0] != '\\' || s[1] != 'u' { + return -1 + } + r, err := strconv.ParseUint(string(s[2:6]), 16, 64) + if err != nil { + return -1 + } + return rune(r) +} + +// unquote converts a quoted JSON string literal s into an actual string t. +// The rules are different than for Go, so cannot use strconv.Unquote. +func unquote(s []byte) (t string, ok bool) { + s, ok = unquoteBytes(s) + t = string(s) + return +} + +// unquote converts a quoted JSON string literal s into an actual string t. +// The rules are different than for Go, so cannot use strconv.Unquote. +func maybeUnquote(s []byte) (t string, ok bool) { + s, ok = maybeUnquoteBytes(s) + t = string(s) + return +} + +func unquoteBytes(s []byte) (t []byte, ok bool) { + if len(s) < 2 || s[0] != '"' && s[0] != '\'' || s[len(s)-1] != '"' && s[len(s)-1] != '\'' { + return + } + singleQuoted := s[0] == '\'' + s = s[1 : len(s)-1] + + // Check for unusual characters. If there are none, + // then no unquoting is needed, so return a slice of the + // original bytes. + r := 0 + for r < len(s) { + c := s[r] + if c == '\\' || c == '"' || c == '\'' || c < ' ' { + break + } + if c < utf8.RuneSelf { + r++ + continue + } + rr, size := utf8.DecodeRune(s[r:]) + if rr == utf8.RuneError && size == 1 { + break + } + r += size + } + if r == len(s) { + return s, true + } + + b := make([]byte, len(s)+2*utf8.UTFMax) + w := copy(b, s[0:r]) + for r < len(s) { + // Out of room? Can only happen if s is full of + // malformed UTF-8 and we're replacing each + // byte with RuneError. + if w >= len(b)-2*utf8.UTFMax { + nb := make([]byte, (len(b)+utf8.UTFMax)*2) + copy(nb, b[0:w]) + b = nb + } + switch c := s[r]; { + case c == '\\': + r++ + if r >= len(s) { + return + } + switch s[r] { + default: + return + case '"', '\\', '/', '\'': + b[w] = s[r] + r++ + w++ + case 'b': + b[w] = '\b' + r++ + w++ + case 'f': + b[w] = '\f' + r++ + w++ + case 'n': + b[w] = '\n' + r++ + w++ + case 'r': + b[w] = '\r' + r++ + w++ + case 't': + b[w] = '\t' + r++ + w++ + case 'u': + r-- + rr := getu4(s[r:]) + if rr < 0 { + return + } + r += 6 + if utf16.IsSurrogate(rr) { + rr1 := getu4(s[r:]) + if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar { + // A valid pair; consume. + r += 6 + w += utf8.EncodeRune(b[w:], dec) + break + } + // Invalid surrogate; fall back to replacement rune. + rr = unicode.ReplacementChar + } + w += utf8.EncodeRune(b[w:], rr) + } + + // Double quote, control characters are invalid. + case !singleQuoted && c == '"', c < ' ': + return + + // Single quote characters are invalid. + case singleQuoted && c == '\'': + return + + // ASCII + case c < utf8.RuneSelf: + b[w] = c + r++ + w++ + + // Coerce to well-formed UTF-8. + default: + rr, size := utf8.DecodeRune(s[r:]) + r += size + w += utf8.EncodeRune(b[w:], rr) + } + } + return b[0:w], true +} diff --git a/src/mongo/gotools/common/json/decode_d_test.go b/src/mongo/gotools/common/json/decode_d_test.go new file mode 100644 index 00000000000..84a95424e59 --- /dev/null +++ b/src/mongo/gotools/common/json/decode_d_test.go @@ -0,0 +1,118 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "gopkg.in/mgo.v2/bson" + "testing" +) + +func TestDecodeBsonD(t *testing.T) { + Convey("When unmarshalling JSON into a bson.D", t, func() { + Convey("a document should be stored with keys in the same order", func() { + data := `{"a":1, "b":2, "c":3, "d":4, "e":5, "f":6}` + out := bson.D{} + err := Unmarshal([]byte(data), &out) + So(err, ShouldBeNil) + So(len(out), ShouldEqual, 6) + So(out[0].Name, ShouldEqual, "a") + So(out[1].Name, ShouldEqual, "b") + So(out[2].Name, ShouldEqual, "c") + So(out[3].Name, ShouldEqual, "d") + So(out[4].Name, ShouldEqual, "e") + So(out[5].Name, ShouldEqual, "f") + + }) + + Convey("a nested bson.D should be parsed", func() { + data := `{"a": 17, "b":{"foo":"bar", "baz":"boo"}, c:"wow" }` + out := struct { + A int `json:"a"` + B bson.D `json:"b"` + C string `json:"c"` + }{} + err := Unmarshal([]byte(data), &out) + So(err, ShouldBeNil) + So(out.A, ShouldEqual, 17) + So(out.C, ShouldEqual, "wow") + So(len(out.B), ShouldEqual, 2) + So(out.B[0].Name, ShouldEqual, "foo") + So(out.B[0].Value, ShouldEqual, "bar") + So(out.B[1].Name, ShouldEqual, "baz") + So(out.B[1].Value, ShouldEqual, "boo") + }) + + Convey("objects nested within DocElems should still be parsed", func() { + data := `{"a":["x", "y","z"], "b":{"foo":"bar", "baz":"boo"}}` + out := bson.D{} + err := Unmarshal([]byte(data), &out) + So(err, ShouldBeNil) + So(len(out), ShouldEqual, 2) + So(out[0].Name, ShouldEqual, "a") + So(out[1].Name, ShouldEqual, "b") + So(out[0].Value, ShouldResemble, []interface{}{"x", "y", "z"}) + So(out[1].Value, ShouldResemble, bson.D{{"foo", "bar"}, {"baz", "boo"}}) + }) + + Convey("only subdocuments inside a bson.D should be parsed into a bson.D", func() { + data := `{subA: {a:{b:{c:9}}}, subB:{a:{b:{c:9}}}}` + out := struct { + A interface{} `json:"subA"` + B bson.D `json:"subB"` + }{} + err := Unmarshal([]byte(data), &out) + So(err, ShouldBeNil) + aMap := out.A.(map[string]interface{}) + So(len(aMap), ShouldEqual, 1) + aMapSub := aMap["a"].(map[string]interface{}) + So(len(aMapSub), ShouldEqual, 1) + aMapSubSub := aMapSub["b"].(map[string]interface{}) + So(aMapSubSub["c"], ShouldEqual, 9) + So(len(out.B), ShouldEqual, 1) + // using string comparison for simplicity + c := bson.D{{Name: "c", Value: 9}} + b := bson.D{{Name: "b", Value: c}} + a := bson.D{{Name: "a", Value: b}} + So(fmt.Sprintf("%v", out.B), ShouldEqual, fmt.Sprintf("%v", a)) + }) + + Convey("subdocuments inside arrays inside bson.D should be parsed into a bson.D", func() { + data := `{"a":[1,2,{b:"inner"}]}` + out := bson.D{} + err := Unmarshal([]byte(data), &out) + So(err, ShouldBeNil) + So(len(out), ShouldEqual, 1) + So(out[0].Value, ShouldHaveSameTypeAs, []interface{}{}) + innerArray := out[0].Value.([]interface{}) + So(len(innerArray), ShouldEqual, 3) + So(innerArray[0], ShouldEqual, 1) + So(innerArray[1], ShouldEqual, 2) + So(innerArray[2], ShouldHaveSameTypeAs, bson.D{}) + innerD := innerArray[2].(bson.D) + So(len(innerD), ShouldEqual, 1) + So(innerD[0].Name, ShouldEqual, "b") + So(innerD[0].Value, ShouldEqual, "inner") + }) + + Convey("null should be a valid value", func() { + data := `{"a":true, "b":null, "c": 5}` + out := bson.D{} + err := Unmarshal([]byte(data), &out) + So(err, ShouldBeNil) + So(len(out), ShouldEqual, 3) + So(out[0].Name, ShouldEqual, "a") + So(out[0].Value, ShouldEqual, true) + So(out[1].Name, ShouldEqual, "b") + So(out[1].Value, ShouldBeNil) + So(out[2].Name, ShouldEqual, "c") + So(out[2].Value, ShouldEqual, 5) + }) + + }) + Convey("Unmarshalling to a non-bson.D slice types should fail", t, func() { + data := `{"a":["x", "y","z"], "b":{"foo":"bar", "baz":"boo"}}` + out := []interface{}{} + err := Unmarshal([]byte(data), &out) + So(err, ShouldNotBeNil) + }) +} diff --git a/src/mongo/gotools/common/json/decode_test.go b/src/mongo/gotools/common/json/decode_test.go new file mode 100644 index 00000000000..d32f00b4952 --- /dev/null +++ b/src/mongo/gotools/common/json/decode_test.go @@ -0,0 +1,1364 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import ( + "bytes" + "encoding" + "fmt" + "image" + "reflect" + "strings" + "testing" + "time" +) + +type T struct { + X string + Y int + Z int `json:"-"` +} + +type U struct { + Alphabet string `json:"alpha"` +} + +type V struct { + F1 interface{} + F2 int32 + F3 Number +} + +// ifaceNumAsFloat64/ifaceNumAsNumber are used to test unmarshaling with and +// without UseNumber +var ifaceNumAsFloat64 = map[string]interface{}{ + "k1": float64(1), + "k2": "s", + "k3": []interface{}{float64(1), float64(2.0), float64(3e-3)}, + "k4": map[string]interface{}{"kk1": "s", "kk2": float64(2)}, +} + +// ifaceNumAsMixedTypes is used to test unmarshalling with extended JSON +var ifaceNumAsMixedTypes = map[string]interface{}{ + "k1": int32(1), + "k2": "s", + "k3": []interface{}{int32(1), int32(2), float64(3e-3)}, + "k4": map[string]interface{}{"kk1": "s", "kk2": int32(2)}, +} + +var ifaceNumAsNumber = map[string]interface{}{ + "k1": Number("1"), + "k2": "s", + "k3": []interface{}{Number("1"), Number("2.0"), Number("3e-3")}, + "k4": map[string]interface{}{"kk1": "s", "kk2": Number("2")}, +} + +type tx struct { + x int +} + +// A type that can unmarshal itself. + +type unmarshaler struct { + T bool +} + +func (u *unmarshaler) UnmarshalJSON(b []byte) error { + *u = unmarshaler{true} // All we need to see that UnmarshalJSON is called. + return nil +} + +type ustruct struct { + M unmarshaler +} + +type unmarshalerText struct { + T bool +} + +// needed for re-marshaling tests +func (u *unmarshalerText) MarshalText() ([]byte, error) { + return []byte(""), nil +} + +func (u *unmarshalerText) UnmarshalText(b []byte) error { + *u = unmarshalerText{true} // All we need to see that UnmarshalText is called. + return nil +} + +var _ encoding.TextUnmarshaler = (*unmarshalerText)(nil) + +type ustructText struct { + M unmarshalerText +} + +var ( + um0, um1 unmarshaler // target2 of unmarshaling + ump = &um1 + umtrue = unmarshaler{true} + umslice = []unmarshaler{{true}} + umslicep = new([]unmarshaler) + umstruct = ustruct{unmarshaler{true}} + + um0T, um1T unmarshalerText // target2 of unmarshaling + umpT = &um1T + umtrueT = unmarshalerText{true} + umsliceT = []unmarshalerText{{true}} + umslicepT = new([]unmarshalerText) + umstructT = ustructText{unmarshalerText{true}} +) + +// Test data structures for anonymous fields. + +type Point struct { + Z int +} + +type Top struct { + Level0 int + Embed0 + *Embed0a + *Embed0b `json:"e,omitempty"` // treated as named + Embed0c `json:"-"` // ignored + Loop + Embed0p // has Point with X, Y, used + Embed0q // has Point with Z, used +} + +type Embed0 struct { + Level1a int // overridden by Embed0a's Level1a with json tag + Level1b int // used because Embed0a's Level1b is renamed + Level1c int // used because Embed0a's Level1c is ignored + Level1d int // annihilated by Embed0a's Level1d + Level1e int `json:"x"` // annihilated by Embed0a.Level1e +} + +type Embed0a struct { + Level1a int `json:"Level1a,omitempty"` + Level1b int `json:"LEVEL1B,omitempty"` + Level1c int `json:"-"` + Level1d int // annihilated by Embed0's Level1d + Level1f int `json:"x"` // annihilated by Embed0's Level1e +} + +type Embed0b Embed0 + +type Embed0c Embed0 + +type Embed0p struct { + image.Point +} + +type Embed0q struct { + Point +} + +type Loop struct { + Loop1 int `json:",omitempty"` + Loop2 int `json:",omitempty"` + *Loop +} + +// From reflect test: +// The X in S6 and S7 annihilate, but they also block the X in S8.S9. +type S5 struct { + S6 + S7 + S8 +} + +type S6 struct { + X int +} + +type S7 S6 + +type S8 struct { + S9 +} + +type S9 struct { + X int + Y int +} + +// From reflect test: +// The X in S11.S6 and S12.S6 annihilate, but they also block the X in S13.S8.S9. +type S10 struct { + S11 + S12 + S13 +} + +type S11 struct { + S6 +} + +type S12 struct { + S6 +} + +type S13 struct { + S8 +} + +type unmarshalTest struct { + in string + ptr interface{} + out interface{} + err error + useNumber bool +} + +type Ambig struct { + // Given "hello", the first match should win. + First int `json:"HELLO"` + Second int `json:"Hello"` +} + +type XYZ struct { + X interface{} + Y interface{} + Z interface{} +} + +var unmarshalTests = []unmarshalTest{ + // basic types + {in: `true`, ptr: new(bool), out: true}, + {in: `1`, ptr: new(int), out: 1}, + {in: `1.2`, ptr: new(float64), out: 1.2}, + {in: `-5`, ptr: new(int16), out: int16(-5)}, + {in: `2`, ptr: new(Number), out: Number("2"), useNumber: true}, + {in: `2`, ptr: new(Number), out: Number("2")}, + {in: `2`, ptr: new(interface{}), out: int32(2)}, + {in: `2`, ptr: new(interface{}), out: Number("2"), useNumber: true}, + {in: `"a\u1234"`, ptr: new(string), out: "a\u1234"}, + {in: `"http:\/\/"`, ptr: new(string), out: "http://"}, + {in: `"g-clef: \uD834\uDD1E"`, ptr: new(string), out: "g-clef: \U0001D11E"}, + {in: `"invalid: \uD834x\uDD1E"`, ptr: new(string), out: "invalid: \uFFFDx\uFFFD"}, + {in: "null", ptr: new(interface{}), out: nil}, + {in: `{"X": [1,2,3], "Y": 4}`, ptr: new(T), out: T{Y: 4}, err: &UnmarshalTypeError{"array", reflect.TypeOf("")}}, + {in: `{"x": 1}`, ptr: new(tx), out: tx{}}, + {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: int32(1), F2: int32(2), F3: Number("3")}}, + {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: Number("1"), F2: int32(2), F3: Number("3")}, useNumber: true}, + {in: `{"k1":1,"k2":"s","k3":[1,2,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsMixedTypes}, + {in: `{"k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4":{"kk1":"s","kk2":2}}`, ptr: new(interface{}), out: ifaceNumAsNumber, useNumber: true}, + + // raw values with whitespace + {in: "\n true ", ptr: new(bool), out: true}, + {in: "\t 1 ", ptr: new(int), out: 1}, + {in: "\r 1.2 ", ptr: new(float64), out: 1.2}, + {in: "\t -5 \n", ptr: new(int16), out: int16(-5)}, + {in: "\t \"a\\u1234\" \n", ptr: new(string), out: "a\u1234"}, + + // Z has a "-" tag. + {in: `{"Y": 1, "Z": 2}`, ptr: new(T), out: T{Y: 1}}, + + {in: `{"alpha": "abc", "alphabet": "xyz"}`, ptr: new(U), out: U{Alphabet: "abc"}}, + {in: `{"alpha": "abc"}`, ptr: new(U), out: U{Alphabet: "abc"}}, + {in: `{"alphabet": "xyz"}`, ptr: new(U), out: U{}}, + + // syntax errors + {in: `{"X": "foo", "Y"}`, err: &SyntaxError{"invalid character '}' after object key", 17}}, + {in: `[1, 2, 3+]`, err: &SyntaxError{"invalid character '+' after array element", 9}}, + {in: `{"X":12x}`, err: &SyntaxError{"invalid character 'x' after object key:value pair", 8}, useNumber: true}, + + // raw value errors + {in: "\x01 42", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, + {in: " 42 \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 5}}, + {in: "\x01 true", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, + {in: " false \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 8}}, + {in: "\x01 1.2", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, + {in: " 3.4 \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 6}}, + {in: "\x01 \"string\"", err: &SyntaxError{"invalid character '\\x01' looking for beginning of value", 1}}, + {in: " \"string\" \x01", err: &SyntaxError{"invalid character '\\x01' after top-level value", 11}}, + + // array tests + {in: `[1, 2, 3]`, ptr: new([3]int), out: [3]int{1, 2, 3}}, + {in: `[1, 2, 3]`, ptr: new([1]int), out: [1]int{1}}, + {in: `[1, 2, 3]`, ptr: new([5]int), out: [5]int{1, 2, 3, 0, 0}}, + + // empty array to interface test + {in: `[]`, ptr: new([]interface{}), out: []interface{}{}}, + {in: `null`, ptr: new([]interface{}), out: []interface{}(nil)}, + {in: `{"T":[]}`, ptr: new(map[string]interface{}), out: map[string]interface{}{"T": []interface{}{}}}, + {in: `{"T":null}`, ptr: new(map[string]interface{}), out: map[string]interface{}{"T": interface{}(nil)}}, + + // composite tests + {in: allValueIndent, ptr: new(All), out: allValue}, + {in: allValueCompact, ptr: new(All), out: allValue}, + {in: allValueIndent, ptr: new(*All), out: &allValue}, + {in: allValueCompact, ptr: new(*All), out: &allValue}, + {in: pallValueIndent, ptr: new(All), out: pallValue}, + {in: pallValueCompact, ptr: new(All), out: pallValue}, + {in: pallValueIndent, ptr: new(*All), out: &pallValue}, + {in: pallValueCompact, ptr: new(*All), out: &pallValue}, + + // unmarshal interface test + {in: `{"T":false}`, ptr: &um0, out: umtrue}, // use "false" so test will fail if custom unmarshaler is not called + {in: `{"T":false}`, ptr: &ump, out: &umtrue}, + {in: `[{"T":false}]`, ptr: &umslice, out: umslice}, + {in: `[{"T":false}]`, ptr: &umslicep, out: &umslice}, + {in: `{"M":{"T":false}}`, ptr: &umstruct, out: umstruct}, + + // UnmarshalText interface test + {in: `"X"`, ptr: &um0T, out: umtrueT}, // use "false" so test will fail if custom unmarshaler is not called + {in: `"X"`, ptr: &umpT, out: &umtrueT}, + {in: `["X"]`, ptr: &umsliceT, out: umsliceT}, + {in: `["X"]`, ptr: &umslicepT, out: &umsliceT}, + {in: `{"M":"X"}`, ptr: &umstructT, out: umstructT}, + + { + in: `{ + "Level0": 1, + "Level1b": 2, + "Level1c": 3, + "x": 4, + "Level1a": 5, + "LEVEL1B": 6, + "e": { + "Level1a": 8, + "Level1b": 9, + "Level1c": 10, + "Level1d": 11, + "x": 12 + }, + "Loop1": 13, + "Loop2": 14, + "X": 15, + "Y": 16, + "Z": 17 + }`, + ptr: new(Top), + out: Top{ + Level0: 1, + Embed0: Embed0{ + Level1b: 2, + Level1c: 3, + }, + Embed0a: &Embed0a{ + Level1a: 5, + Level1b: 6, + }, + Embed0b: &Embed0b{ + Level1a: 8, + Level1b: 9, + Level1c: 10, + Level1d: 11, + Level1e: 12, + }, + Loop: Loop{ + Loop1: 13, + Loop2: 14, + }, + Embed0p: Embed0p{ + Point: image.Point{X: 15, Y: 16}, + }, + Embed0q: Embed0q{ + Point: Point{Z: 17}, + }, + }, + }, + { + in: `{"hello": 1}`, + ptr: new(Ambig), + out: Ambig{First: 1}, + }, + + { + in: `{"X": 1,"Y":2}`, + ptr: new(S5), + out: S5{S8: S8{S9: S9{Y: 2}}}, + }, + { + in: `{"X": 1,"Y":2}`, + ptr: new(S10), + out: S10{S13: S13{S8: S8{S9: S9{Y: 2}}}}, + }, + + // invalid UTF-8 is coerced to valid UTF-8. + { + in: "\"hello\xffworld\"", + ptr: new(string), + out: "hello\ufffdworld", + }, + { + in: "\"hello\xc2\xc2world\"", + ptr: new(string), + out: "hello\ufffd\ufffdworld", + }, + { + in: "\"hello\xc2\xffworld\"", + ptr: new(string), + out: "hello\ufffd\ufffdworld", + }, + { + in: "\"hello\\ud800world\"", + ptr: new(string), + out: "hello\ufffdworld", + }, + { + in: "\"hello\\ud800\\ud800world\"", + ptr: new(string), + out: "hello\ufffd\ufffdworld", + }, + { + in: "\"hello\\ud800\\ud800world\"", + ptr: new(string), + out: "hello\ufffd\ufffdworld", + }, + { + in: "\"hello\xed\xa0\x80\xed\xb0\x80world\"", + ptr: new(string), + out: "hello\ufffd\ufffd\ufffd\ufffd\ufffd\ufffdworld", + }, +} + +func TestMarshal(t *testing.T) { + b, err := Marshal(allValue) + if err != nil { + t.Fatalf("Marshal allValue: %v", err) + } + if string(b) != allValueCompact { + t.Errorf("Marshal allValueCompact") + diff(t, b, []byte(allValueCompact)) + return + } + + b, err = Marshal(pallValue) + if err != nil { + t.Fatalf("Marshal pallValue: %v", err) + } + if string(b) != pallValueCompact { + t.Errorf("Marshal pallValueCompact") + diff(t, b, []byte(pallValueCompact)) + return + } +} + +var badUTF8 = []struct { + in, out string +}{ + {"hello\xffworld", `"hello\ufffdworld"`}, + {"", `""`}, + {"\xff", `"\ufffd"`}, + {"\xff\xff", `"\ufffd\ufffd"`}, + {"a\xffb", `"a\ufffdb"`}, + {"\xe6\x97\xa5\xe6\x9c\xac\xff\xaa\x9e", `"日本\ufffd\ufffd\ufffd"`}, +} + +func TestMarshalBadUTF8(t *testing.T) { + for _, tt := range badUTF8 { + b, err := Marshal(tt.in) + if string(b) != tt.out || err != nil { + t.Errorf("Marshal(%q) = %#q, %v, want %#q, nil", tt.in, b, err, tt.out) + } + } +} + +func TestMarshalNumberZeroVal(t *testing.T) { + var n Number + out, err := Marshal(n) + if err != nil { + t.Fatal(err) + } + outStr := string(out) + if outStr != "0" { + t.Fatalf("Invalid zero val for Number: %q", outStr) + } +} + +func TestMarshalEmbeds(t *testing.T) { + top := &Top{ + Level0: 1, + Embed0: Embed0{ + Level1b: 2, + Level1c: 3, + }, + Embed0a: &Embed0a{ + Level1a: 5, + Level1b: 6, + }, + Embed0b: &Embed0b{ + Level1a: 8, + Level1b: 9, + Level1c: 10, + Level1d: 11, + Level1e: 12, + }, + Loop: Loop{ + Loop1: 13, + Loop2: 14, + }, + Embed0p: Embed0p{ + Point: image.Point{X: 15, Y: 16}, + }, + Embed0q: Embed0q{ + Point: Point{Z: 17}, + }, + } + b, err := Marshal(top) + if err != nil { + t.Fatal(err) + } + want := "{\"Level0\":1,\"Level1b\":2,\"Level1c\":3,\"Level1a\":5,\"LEVEL1B\":6,\"e\":{\"Level1a\":8,\"Level1b\":9,\"Level1c\":10,\"Level1d\":11,\"x\":12},\"Loop1\":13,\"Loop2\":14,\"X\":15,\"Y\":16,\"Z\":17}" + if string(b) != want { + t.Errorf("Wrong marshal result.\n got: %q\nwant: %q", b, want) + } +} + +func TestUnmarshal(t *testing.T) { + for i, tt := range unmarshalTests { + var scan scanner + in := []byte(tt.in) + if err := checkValid(in, &scan); err != nil { + if !reflect.DeepEqual(err, tt.err) { + t.Errorf("#%d: checkValid: %#v", i, err) + continue + } + } + if tt.ptr == nil { + continue + } + // v = new(right-type) + v := reflect.New(reflect.TypeOf(tt.ptr).Elem()) + dec := NewDecoder(bytes.NewReader(in)) + if tt.useNumber { + dec.UseNumber() + } + if err := dec.Decode(v.Interface()); !reflect.DeepEqual(err, tt.err) { + t.Errorf("#%d: %v want %v", i, err, tt.err) + continue + } + if !reflect.DeepEqual(v.Elem().Interface(), tt.out) { + t.Errorf("#%d: mismatch\nhave: %#+v\nwant: %#+v", i, v.Elem().Interface(), tt.out) + data, _ := Marshal(v.Elem().Interface()) + println(string(data)) + data, _ = Marshal(tt.out) + println(string(data)) + continue + } + + // Check round trip. + if tt.err == nil { + enc, err := Marshal(v.Interface()) + if err != nil { + t.Errorf("#%d: error re-marshaling: %v", i, err) + continue + } + vv := reflect.New(reflect.TypeOf(tt.ptr).Elem()) + dec = NewDecoder(bytes.NewReader(enc)) + if tt.useNumber { + dec.UseNumber() + } + if err := dec.Decode(vv.Interface()); err != nil { + t.Errorf("#%d: error re-unmarshaling %#q: %v", i, enc, err) + continue + } + if !reflect.DeepEqual(v.Elem().Interface(), vv.Elem().Interface()) { + t.Errorf("#%d: mismatch\nhave: %#+v\nwant: %#+v", i, v.Elem().Interface(), vv.Elem().Interface()) + t.Errorf(" In: %q", strings.Map(noSpace, string(in))) + t.Errorf("Marshal: %q", strings.Map(noSpace, string(enc))) + continue + } + } + } +} + +func TestUnmarshalMarshal(t *testing.T) { + initBig() + var v interface{} + if err := Unmarshal(jsonBig, &v); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + b, err := Marshal(v) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + if !bytes.Equal(jsonBig, b) { + t.Errorf("Marshal jsonBig") + diff(t, b, jsonBig) + return + } +} + +var numberTests = []struct { + in string + i int64 + intErr string + f float64 + floatErr string +}{ + {in: "-1.23e1", intErr: "strconv.ParseInt: parsing \"-1.23e1\": invalid syntax", f: -1.23e1}, + {in: "-12", i: -12, f: -12.0}, + {in: "1e1000", intErr: "strconv.ParseInt: parsing \"1e1000\": invalid syntax", floatErr: "strconv.ParseFloat: parsing \"1e1000\": value out of range"}, +} + +// Independent of Decode, basic coverage of the accessors in Number +func TestNumberAccessors(t *testing.T) { + for _, tt := range numberTests { + n := Number(tt.in) + if s := n.String(); s != tt.in { + t.Errorf("Number(%q).String() is %q", tt.in, s) + } + if i, err := n.Int64(); err == nil && tt.intErr == "" && i != tt.i { + t.Errorf("Number(%q).Int64() is %d", tt.in, i) + } else if (err == nil && tt.intErr != "") || (err != nil && err.Error() != tt.intErr) { + t.Errorf("Number(%q).Int64() wanted error %q but got: %v", tt.in, tt.intErr, err) + } + if f, err := n.Float64(); err == nil && tt.floatErr == "" && f != tt.f { + t.Errorf("Number(%q).Float64() is %g", tt.in, f) + } else if (err == nil && tt.floatErr != "") || (err != nil && err.Error() != tt.floatErr) { + t.Errorf("Number(%q).Float64() wanted error %q but got: %v", tt.in, tt.floatErr, err) + } + } +} + +func TestLargeByteSlice(t *testing.T) { + s0 := make([]byte, 2000) + for i := range s0 { + s0[i] = byte(i) + } + b, err := Marshal(s0) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + var s1 []byte + if err := Unmarshal(b, &s1); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if !bytes.Equal(s0, s1) { + t.Errorf("Marshal large byte slice") + diff(t, s0, s1) + } +} + +type Xint struct { + X int +} + +func TestUnmarshalInterface(t *testing.T) { + var xint Xint + var i interface{} = &xint + if err := Unmarshal([]byte(`{"X":1}`), &i); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if xint.X != 1 { + t.Fatalf("Did not write to xint") + } +} + +func TestUnmarshalPtrPtr(t *testing.T) { + var xint Xint + pxint := &xint + if err := Unmarshal([]byte(`{"X":1}`), &pxint); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if xint.X != 1 { + t.Fatalf("Did not write to xint") + } +} + +func TestEscape(t *testing.T) { + const input = `"foobar"<html>` + " [\u2028 \u2029]" + const expected = `"\"foobar\"\u003chtml\u003e [\u2028 \u2029]"` + b, err := Marshal(input) + if err != nil { + t.Fatalf("Marshal error: %v", err) + } + if s := string(b); s != expected { + t.Errorf("Encoding of [%s]:\n got [%s]\nwant [%s]", input, s, expected) + } +} + +// WrongString is a struct that's misusing the ,string modifier. +type WrongString struct { + Message string `json:"result,string"` +} + +type wrongStringTest struct { + in, err string +} + +var wrongStringTests = []wrongStringTest{ + {`{"result":"x"}`, `json: invalid use of ,string struct tag, trying to unmarshal "x" into string`}, + {`{"result":"foo"}`, `json: invalid use of ,string struct tag, trying to unmarshal "foo" into string`}, + {`{"result":"123"}`, `json: invalid use of ,string struct tag, trying to unmarshal "123" into string`}, +} + +// If people misuse the ,string modifier, the error message should be +// helpful, telling the user that they're doing it wrong. +func TestErrorMessageFromMisusedString(t *testing.T) { + for n, tt := range wrongStringTests { + r := strings.NewReader(tt.in) + var s WrongString + err := NewDecoder(r).Decode(&s) + got := fmt.Sprintf("%v", err) + if got != tt.err { + t.Errorf("%d. got err = %q, want %q", n, got, tt.err) + } + } +} + +func noSpace(c rune) rune { + if isSpace(c) { + return -1 + } + return c +} + +type All struct { + Bool bool + Int int + Int8 int8 + Int16 int16 + Int32 int32 + Int64 int64 + Uint uint + Uint8 uint8 + Uint16 uint16 + Uint32 uint32 + Uint64 uint64 + Uintptr uintptr + Float32 float32 + Float64 float64 + + Foo string `json:"bar"` + Foo2 string `json:"bar2,dummyopt"` + + IntStr int64 `json:",string"` + + PBool *bool + PInt *int + PInt8 *int8 + PInt16 *int16 + PInt32 *int32 + PInt64 *int64 + PUint *uint + PUint8 *uint8 + PUint16 *uint16 + PUint32 *uint32 + PUint64 *uint64 + PUintptr *uintptr + PFloat32 *float32 + PFloat64 *float64 + + String string + PString *string + + Map map[string]Small + MapP map[string]*Small + PMap *map[string]Small + PMapP *map[string]*Small + + EmptyMap map[string]Small + NilMap map[string]Small + + Slice []Small + SliceP []*Small + PSlice *[]Small + PSliceP *[]*Small + + EmptySlice []Small + NilSlice []Small + + StringSlice []string + ByteSlice []byte + + Small Small + PSmall *Small + PPSmall **Small + + Interface interface{} + PInterface *interface{} + + unexported int +} + +type Small struct { + Tag string +} + +var allValue = All{ + Bool: true, + Int: 2, + Int8: 3, + Int16: 4, + Int32: 5, + Int64: 6, + Uint: 7, + Uint8: 8, + Uint16: 9, + Uint32: 10, + Uint64: 11, + Uintptr: 12, + Float32: 14.1, + Float64: 15.1, + Foo: "foo", + Foo2: "foo2", + IntStr: 42, + String: "16", + Map: map[string]Small{ + "17": {Tag: "tag17"}, + "18": {Tag: "tag18"}, + }, + MapP: map[string]*Small{ + "19": {Tag: "tag19"}, + "20": nil, + }, + EmptyMap: map[string]Small{}, + Slice: []Small{{Tag: "tag20"}, {Tag: "tag21"}}, + SliceP: []*Small{{Tag: "tag22"}, nil, {Tag: "tag23"}}, + EmptySlice: []Small{}, + StringSlice: []string{"str24", "str25", "str26"}, + ByteSlice: []byte{27, 28, 29}, + Small: Small{Tag: "tag30"}, + PSmall: &Small{Tag: "tag31"}, + Interface: 5.2, +} + +var pallValue = All{ + PBool: &allValue.Bool, + PInt: &allValue.Int, + PInt8: &allValue.Int8, + PInt16: &allValue.Int16, + PInt32: &allValue.Int32, + PInt64: &allValue.Int64, + PUint: &allValue.Uint, + PUint8: &allValue.Uint8, + PUint16: &allValue.Uint16, + PUint32: &allValue.Uint32, + PUint64: &allValue.Uint64, + PUintptr: &allValue.Uintptr, + PFloat32: &allValue.Float32, + PFloat64: &allValue.Float64, + PString: &allValue.String, + PMap: &allValue.Map, + PMapP: &allValue.MapP, + PSlice: &allValue.Slice, + PSliceP: &allValue.SliceP, + PPSmall: &allValue.PSmall, + PInterface: &allValue.Interface, +} + +var allValueIndent = `{ + "Bool": true, + "Int": 2, + "Int8": 3, + "Int16": 4, + "Int32": 5, + "Int64": 6, + "Uint": 7, + "Uint8": 8, + "Uint16": 9, + "Uint32": 10, + "Uint64": 11, + "Uintptr": 12, + "Float32": 14.1, + "Float64": 15.1, + "bar": "foo", + "bar2": "foo2", + "IntStr": "42", + "PBool": null, + "PInt": null, + "PInt8": null, + "PInt16": null, + "PInt32": null, + "PInt64": null, + "PUint": null, + "PUint8": null, + "PUint16": null, + "PUint32": null, + "PUint64": null, + "PUintptr": null, + "PFloat32": null, + "PFloat64": null, + "String": "16", + "PString": null, + "Map": { + "17": { + "Tag": "tag17" + }, + "18": { + "Tag": "tag18" + } + }, + "MapP": { + "19": { + "Tag": "tag19" + }, + "20": null + }, + "PMap": null, + "PMapP": null, + "EmptyMap": {}, + "NilMap": null, + "Slice": [ + { + "Tag": "tag20" + }, + { + "Tag": "tag21" + } + ], + "SliceP": [ + { + "Tag": "tag22" + }, + null, + { + "Tag": "tag23" + } + ], + "PSlice": null, + "PSliceP": null, + "EmptySlice": [], + "NilSlice": null, + "StringSlice": [ + "str24", + "str25", + "str26" + ], + "ByteSlice": "Gxwd", + "Small": { + "Tag": "tag30" + }, + "PSmall": { + "Tag": "tag31" + }, + "PPSmall": null, + "Interface": 5.2, + "PInterface": null +}` + +var allValueCompact = strings.Map(noSpace, allValueIndent) + +var pallValueIndent = `{ + "Bool": false, + "Int": 0, + "Int8": 0, + "Int16": 0, + "Int32": 0, + "Int64": 0, + "Uint": 0, + "Uint8": 0, + "Uint16": 0, + "Uint32": 0, + "Uint64": 0, + "Uintptr": 0, + "Float32": 0, + "Float64": 0, + "bar": "", + "bar2": "", + "IntStr": "0", + "PBool": true, + "PInt": 2, + "PInt8": 3, + "PInt16": 4, + "PInt32": 5, + "PInt64": 6, + "PUint": 7, + "PUint8": 8, + "PUint16": 9, + "PUint32": 10, + "PUint64": 11, + "PUintptr": 12, + "PFloat32": 14.1, + "PFloat64": 15.1, + "String": "", + "PString": "16", + "Map": null, + "MapP": null, + "PMap": { + "17": { + "Tag": "tag17" + }, + "18": { + "Tag": "tag18" + } + }, + "PMapP": { + "19": { + "Tag": "tag19" + }, + "20": null + }, + "EmptyMap": null, + "NilMap": null, + "Slice": null, + "SliceP": null, + "PSlice": [ + { + "Tag": "tag20" + }, + { + "Tag": "tag21" + } + ], + "PSliceP": [ + { + "Tag": "tag22" + }, + null, + { + "Tag": "tag23" + } + ], + "EmptySlice": null, + "NilSlice": null, + "StringSlice": null, + "ByteSlice": null, + "Small": { + "Tag": "" + }, + "PSmall": null, + "PPSmall": { + "Tag": "tag31" + }, + "Interface": null, + "PInterface": 5.2 +}` + +var pallValueCompact = strings.Map(noSpace, pallValueIndent) + +func TestRefUnmarshal(t *testing.T) { + type S struct { + // Ref is defined in encode_test.go. + R0 Ref + R1 *Ref + R2 RefText + R3 *RefText + } + want := S{ + R0: 12, + R1: new(Ref), + R2: 13, + R3: new(RefText), + } + *want.R1 = 12 + *want.R3 = 13 + + var got S + if err := Unmarshal([]byte(`{"R0":"ref","R1":"ref","R2":"ref","R3":"ref"}`), &got); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("got %+v, want %+v", got, want) + } +} + +// Test that the empty string doesn't panic decoding when ,string is specified +// Issue 3450 +func TestEmptyString(t *testing.T) { + type T2 struct { + Number1 int `json:",string"` + Number2 int `json:",string"` + } + data := `{"Number1":"1", "Number2":""}` + dec := NewDecoder(strings.NewReader(data)) + var t2 T2 + err := dec.Decode(&t2) + if err == nil { + t.Fatal("Decode: did not return error") + } + if t2.Number1 != 1 { + t.Fatal("Decode: did not set Number1") + } +} + +// Test that the returned error is non-nil when trying to unmarshal null string into int, for successive ,string option +// Issue 7046 +func TestNullString(t *testing.T) { + type T struct { + A int `json:",string"` + B int `json:",string"` + } + data := []byte(`{"A": "1", "B": null}`) + var s T + err := Unmarshal(data, &s) + if err == nil { + t.Fatalf("expected error; got %v", s) + } +} + +func intp(x int) *int { + p := new(int) + *p = x + return p +} + +func intpp(x *int) **int { + pp := new(*int) + *pp = x + return pp +} + +var interfaceSetTests = []struct { + pre interface{} + json string + post interface{} +}{ + {"foo", `"bar"`, "bar"}, + {"foo", `2`, int32(2)}, + {"foo", `true`, true}, + {"foo", `null`, nil}, + + {nil, `null`, nil}, + {new(int), `null`, nil}, + {(*int)(nil), `null`, nil}, + {new(*int), `null`, new(*int)}, + {(**int)(nil), `null`, nil}, + {intp(1), `null`, nil}, + {intpp(nil), `null`, intpp(nil)}, + {intpp(intp(1)), `null`, intpp(nil)}, +} + +func TestInterfaceSet(t *testing.T) { + for _, tt := range interfaceSetTests { + b := struct{ X interface{} }{tt.pre} + blob := `{"X":` + tt.json + `}` + if err := Unmarshal([]byte(blob), &b); err != nil { + t.Errorf("Unmarshal %#q: %v", blob, err) + continue + } + if !reflect.DeepEqual(b.X, tt.post) { + t.Errorf("Unmarshal %#q into %#v: X=%#v, want %#v", blob, tt.pre, b.X, tt.post) + } + } +} + +// JSON null values should be ignored for primitives and string values instead of resulting in an error. +// Issue 2540 +func TestUnmarshalNulls(t *testing.T) { + jsonData := []byte(`{ + "Bool" : null, + "Int" : null, + "Int8" : null, + "Int16" : null, + "Int32" : null, + "Int64" : null, + "Uint" : null, + "Uint8" : null, + "Uint16" : null, + "Uint32" : null, + "Uint64" : null, + "Float32" : null, + "Float64" : null, + "String" : null}`) + + nulls := All{ + Bool: true, + Int: 2, + Int8: 3, + Int16: 4, + Int32: 5, + Int64: 6, + Uint: 7, + Uint8: 8, + Uint16: 9, + Uint32: 10, + Uint64: 11, + Float32: 12.1, + Float64: 13.1, + String: "14"} + + err := Unmarshal(jsonData, &nulls) + if err != nil { + t.Errorf("Unmarshal of null values failed: %v", err) + } + if !nulls.Bool || nulls.Int != 2 || nulls.Int8 != 3 || nulls.Int16 != 4 || nulls.Int32 != 5 || nulls.Int64 != 6 || + nulls.Uint != 7 || nulls.Uint8 != 8 || nulls.Uint16 != 9 || nulls.Uint32 != 10 || nulls.Uint64 != 11 || + nulls.Float32 != 12.1 || nulls.Float64 != 13.1 || nulls.String != "14" { + + t.Errorf("Unmarshal of null values affected primitives") + } +} + +func TestStringKind(t *testing.T) { + type stringKind string + + var m1, m2 map[stringKind]int + m1 = map[stringKind]int{ + "foo": 42, + } + + data, err := Marshal(m1) + if err != nil { + t.Errorf("Unexpected error marshalling: %v", err) + } + + err = Unmarshal(data, &m2) + if err != nil { + t.Errorf("Unexpected error unmarshalling: %v", err) + } + + if !reflect.DeepEqual(m1, m2) { + t.Error("Items should be equal after encoding and then decoding") + } + +} + +var decodeTypeErrorTests = []struct { + dest interface{} + src string +}{ + {new(string), `{"user": "name"}`}, // issue 4628. + {new(error), `{}`}, // issue 4222 + {new(error), `[]`}, + {new(error), `""`}, + {new(error), `123`}, + {new(error), `true`}, +} + +func TestUnmarshalTypeError(t *testing.T) { + for _, item := range decodeTypeErrorTests { + err := Unmarshal([]byte(item.src), item.dest) + if _, ok := err.(*UnmarshalTypeError); !ok { + t.Errorf("expected type error for Unmarshal(%q, type %T): got %T", + item.src, item.dest, err) + } + } +} + +var unmarshalSyntaxTests = []string{ + "tru", + "fals", + "nul", + "123e", + `"hello`, + `[1,2,3`, + `{"key":1`, + `{"key":1,`, +} + +func TestUnmarshalSyntax(t *testing.T) { + var x interface{} + for _, src := range unmarshalSyntaxTests { + err := Unmarshal([]byte(src), &x) + if _, ok := err.(*SyntaxError); !ok { + t.Errorf("expected syntax error for Unmarshal(%q): got %T", src, err) + } + } +} + +// Test handling of unexported fields that should be ignored. +// Issue 4660 +type unexportedFields struct { + Name string + m map[string]interface{} + m2 map[string]interface{} +} + +func TestUnmarshalUnexported(t *testing.T) { + input := `{"Name": "Bob", "m": {"x": 123}, "m2": {"y": 456}, "abcd": {"z": 789}}` + want := &unexportedFields{Name: "Bob"} + + out := &unexportedFields{} + err := Unmarshal([]byte(input), out) + if err != nil { + t.Errorf("got error %v, expected nil", err) + } + if !reflect.DeepEqual(out, want) { + t.Errorf("got %q, want %q", out, want) + } +} + +// Time3339 is a time.Time which encodes to and from JSON +// as an RFC 3339 time in UTC. +type Time3339 time.Time + +func (t *Time3339) UnmarshalJSON(b []byte) error { + if len(b) < 2 || b[0] != '"' || b[len(b)-1] != '"' { + return fmt.Errorf("types: failed to unmarshal non-string value %q as an RFC 3339 time", b) + } + tm, err := time.Parse(time.RFC3339, string(b[1:len(b)-1])) + if err != nil { + return err + } + *t = Time3339(tm) + return nil +} + +func TestUnmarshalJSONLiteralError(t *testing.T) { + var t3 Time3339 + err := Unmarshal([]byte(`"0000-00-00T00:00:00Z"`), &t3) + if err == nil { + t.Fatalf("expected error; got time %v", time.Time(t3)) + } + if !strings.Contains(err.Error(), "range") { + t.Errorf("got err = %v; want out of range error", err) + } +} + +// Test that extra object elements in an array do not result in a +// "data changing underfoot" error. +// Issue 3717 +func TestSkipArrayObjects(t *testing.T) { + json := `[{}]` + var dest [0]interface{} + + err := Unmarshal([]byte(json), &dest) + if err != nil { + t.Errorf("got error %q, want nil", err) + } +} + +// Test semantics of pre-filled struct fields and pre-filled map fields. +// Issue 4900. +func TestPrefilled(t *testing.T) { + ptrToMap := func(m map[string]interface{}) *map[string]interface{} { return &m } + + // Values here change, cannot reuse table across runs. + var prefillTests = []struct { + in string + ptr interface{} + out interface{} + }{ + { + in: `{"X": 1, "Y": 2}`, + ptr: &XYZ{X: float32(3), Y: int16(4), Z: 1.5}, + out: &XYZ{X: int32(1), Y: int32(2), Z: 1.5}, + }, + { + in: `{"X": 1, "Y": 2}`, + ptr: ptrToMap(map[string]interface{}{"X": float32(3), "Y": int16(4), "Z": 1.5}), + out: ptrToMap(map[string]interface{}{"X": int32(1), "Y": int32(2), "Z": 1.5}), + }, + } + + for _, tt := range prefillTests { + ptrstr := fmt.Sprintf("%v", tt.ptr) + err := Unmarshal([]byte(tt.in), tt.ptr) // tt.ptr edited here + if err != nil { + t.Errorf("Unmarshal: %v", err) + } + if !reflect.DeepEqual(tt.ptr, tt.out) { + t.Errorf("Unmarshal(%#q, %s): have %v, want %v", tt.in, ptrstr, tt.ptr, tt.out) + } + } +} + +var invalidUnmarshalTests = []struct { + v interface{} + want string +}{ + {nil, "json: Unmarshal(nil)"}, + {struct{}{}, "json: Unmarshal(non-pointer struct {})"}, + {(*int)(nil), "json: Unmarshal(nil *int)"}, +} + +func TestInvalidUnmarshal(t *testing.T) { + buf := []byte(`{"a":"1"}`) + for _, tt := range invalidUnmarshalTests { + err := Unmarshal(buf, tt.v) + if err == nil { + t.Errorf("Unmarshal expecting error, got nil") + continue + } + if got := err.Error(); got != tt.want { + t.Errorf("Unmarshal = %q; want %q", got, tt.want) + } + } +} diff --git a/src/mongo/gotools/common/json/encode.go b/src/mongo/gotools/common/json/encode.go new file mode 100644 index 00000000000..ee0fd021294 --- /dev/null +++ b/src/mongo/gotools/common/json/encode.go @@ -0,0 +1,1186 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package json implements encoding and decoding of JSON objects as defined in +// RFC 4627. The mapping between JSON objects and Go values is described +// in the documentation for the Marshal and Unmarshal functions. +// +// See "JSON and Go" for an introduction to this package: +// http://golang.org/doc/articles/json_and_go.html +package json + +import ( + "bytes" + "encoding" + "encoding/base64" + "math" + "reflect" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "unicode" + "unicode/utf8" +) + +// Marshal returns the JSON encoding of v. +// +// Marshal traverses the value v recursively. +// If an encountered value implements the Marshaler interface +// and is not a nil pointer, Marshal calls its MarshalJSON method +// to produce JSON. The nil pointer exception is not strictly necessary +// but mimics a similar, necessary exception in the behavior of +// UnmarshalJSON. +// +// Otherwise, Marshal uses the following type-dependent default encodings: +// +// Boolean values encode as JSON booleans. +// +// Floating point, integer, and Number values encode as JSON numbers. +// +// String values encode as JSON strings. InvalidUTF8Error will be returned +// if an invalid UTF-8 sequence is encountered. +// The angle brackets "<" and ">" are escaped to "\u003c" and "\u003e" +// to keep some browsers from misinterpreting JSON output as HTML. +// Ampersand "&" is also escaped to "\u0026" for the same reason. +// +// Array and slice values encode as JSON arrays, except that +// []byte encodes as a base64-encoded string, and a nil slice +// encodes as the null JSON object. +// +// Struct values encode as JSON objects. Each exported struct field +// becomes a member of the object unless +// - the field's tag is "-", or +// - the field is empty and its tag specifies the "omitempty" option. +// The empty values are false, 0, any +// nil pointer or interface value, and any array, slice, map, or string of +// length zero. The object's default key string is the struct field name +// but can be specified in the struct field's tag value. The "json" key in +// the struct field's tag value is the key name, followed by an optional comma +// and options. Examples: +// +// // Field is ignored by this package. +// Field int `json:"-"` +// +// // Field appears in JSON as key "myName". +// Field int `json:"myName"` +// +// // Field appears in JSON as key "myName" and +// // the field is omitted from the object if its value is empty, +// // as defined above. +// Field int `json:"myName,omitempty"` +// +// // Field appears in JSON as key "Field" (the default), but +// // the field is skipped if empty. +// // Note the leading comma. +// Field int `json:",omitempty"` +// +// The "string" option signals that a field is stored as JSON inside a +// JSON-encoded string. It applies only to fields of string, floating point, +// or integer types. This extra level of encoding is sometimes used when +// communicating with JavaScript programs: +// +// Int64String int64 `json:",string"` +// +// The key name will be used if it's a non-empty string consisting of +// only Unicode letters, digits, dollar signs, percent signs, hyphens, +// underscores and slashes. +// +// Anonymous struct fields are usually marshaled as if their inner exported fields +// were fields in the outer struct, subject to the usual Go visibility rules amended +// as described in the next paragraph. +// An anonymous struct field with a name given in its JSON tag is treated as +// having that name, rather than being anonymous. +// +// The Go visibility rules for struct fields are amended for JSON when +// deciding which field to marshal or unmarshal. If there are +// multiple fields at the same level, and that level is the least +// nested (and would therefore be the nesting level selected by the +// usual Go rules), the following extra rules apply: +// +// 1) Of those fields, if any are JSON-tagged, only tagged fields are considered, +// even if there are multiple untagged fields that would otherwise conflict. +// 2) If there is exactly one field (tagged or not according to the first rule), that is selected. +// 3) Otherwise there are multiple fields, and all are ignored; no error occurs. +// +// Handling of anonymous struct fields is new in Go 1.1. +// Prior to Go 1.1, anonymous struct fields were ignored. To force ignoring of +// an anonymous struct field in both current and earlier versions, give the field +// a JSON tag of "-". +// +// Map values encode as JSON objects. +// The map's key type must be string; the object keys are used directly +// as map keys. +// +// Pointer values encode as the value pointed to. +// A nil pointer encodes as the null JSON object. +// +// Interface values encode as the value contained in the interface. +// A nil interface value encodes as the null JSON object. +// +// Channel, complex, and function values cannot be encoded in JSON. +// Attempting to encode such a value causes Marshal to return +// an UnsupportedTypeError. +// +// JSON cannot represent cyclic data structures and Marshal does not +// handle them. Passing cyclic structures to Marshal will result in +// an infinite recursion. +// +func Marshal(v interface{}) ([]byte, error) { + e := &encodeState{} + err := e.marshal(v) + if err != nil { + return nil, err + } + return e.Bytes(), nil +} + +// MarshalIndent is like Marshal but applies Indent to format the output. +func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) { + b, err := Marshal(v) + if err != nil { + return nil, err + } + var buf bytes.Buffer + err = Indent(&buf, b, prefix, indent) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// HTMLEscape appends to dst the JSON-encoded src with <, >, &, U+2028 and U+2029 +// characters inside string literals changed to \u003c, \u003e, \u0026, \u2028, \u2029 +// so that the JSON will be safe to embed inside HTML <script> tags. +// For historical reasons, web browsers don't honor standard HTML +// escaping within <script> tags, so an alternative JSON encoding must +// be used. +func HTMLEscape(dst *bytes.Buffer, src []byte) { + // The characters can only appear in string literals, + // so just scan the string one byte at a time. + start := 0 + for i, c := range src { + if c == '<' || c == '>' || c == '&' { + if start < i { + dst.Write(src[start:i]) + } + dst.WriteString(`\u00`) + dst.WriteByte(hex[c>>4]) + dst.WriteByte(hex[c&0xF]) + start = i + 1 + } + // Convert U+2028 and U+2029 (E2 80 A8 and E2 80 A9). + if c == 0xE2 && i+2 < len(src) && src[i+1] == 0x80 && src[i+2]&^1 == 0xA8 { + if start < i { + dst.Write(src[start:i]) + } + dst.WriteString(`\u202`) + dst.WriteByte(hex[src[i+2]&0xF]) + start = i + 3 + } + } + if start < len(src) { + dst.Write(src[start:]) + } +} + +// Marshaler is the interface implemented by objects that +// can marshal themselves into valid JSON. +type Marshaler interface { + MarshalJSON() ([]byte, error) +} + +// An UnsupportedTypeError is returned by Marshal when attempting +// to encode an unsupported value type. +type UnsupportedTypeError struct { + Type reflect.Type +} + +func (e *UnsupportedTypeError) Error() string { + return "json: unsupported type: " + e.Type.String() +} + +type UnsupportedValueError struct { + Value reflect.Value + Str string +} + +func (e *UnsupportedValueError) Error() string { + return "json: unsupported value: " + e.Str +} + +// Before Go 1.2, an InvalidUTF8Error was returned by Marshal when +// attempting to encode a string value with invalid UTF-8 sequences. +// As of Go 1.2, Marshal instead coerces the string to valid UTF-8 by +// replacing invalid bytes with the Unicode replacement rune U+FFFD. +// This error is no longer generated but is kept for backwards compatibility +// with programs that might mention it. +type InvalidUTF8Error struct { + S string // the whole string value that caused the error +} + +func (e *InvalidUTF8Error) Error() string { + return "json: invalid UTF-8 in string: " + strconv.Quote(e.S) +} + +type MarshalerError struct { + Type reflect.Type + Err error +} + +func (e *MarshalerError) Error() string { + return "json: error calling MarshalJSON for type " + e.Type.String() + ": " + e.Err.Error() +} + +var hex = "0123456789abcdef" + +// An encodeState encodes JSON into a bytes.Buffer. +type encodeState struct { + bytes.Buffer // accumulated output + scratch [64]byte +} + +var encodeStatePool sync.Pool + +func newEncodeState() *encodeState { + if v := encodeStatePool.Get(); v != nil { + e := v.(*encodeState) + e.Reset() + return e + } + return new(encodeState) +} + +func (e *encodeState) marshal(v interface{}) (err error) { + defer func() { + if r := recover(); r != nil { + if _, ok := r.(runtime.Error); ok { + panic(r) + } + if s, ok := r.(string); ok { + panic(s) + } + err = r.(error) + } + }() + e.reflectValue(reflect.ValueOf(v)) + return nil +} + +func (e *encodeState) error(err error) { + panic(err) +} + +var byteSliceType = reflect.TypeOf([]byte(nil)) + +func isEmptyValue(v reflect.Value) bool { + switch v.Kind() { + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + return v.Len() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Interface, reflect.Ptr: + return v.IsNil() + } + return false +} + +func (e *encodeState) reflectValue(v reflect.Value) { + valueEncoder(v)(e, v, false) +} + +type encoderFunc func(e *encodeState, v reflect.Value, quoted bool) + +var encoderCache struct { + sync.RWMutex + m map[reflect.Type]encoderFunc +} + +func valueEncoder(v reflect.Value) encoderFunc { + if !v.IsValid() { + return invalidValueEncoder + } + return typeEncoder(v.Type()) +} + +func typeEncoder(t reflect.Type) encoderFunc { + encoderCache.RLock() + f := encoderCache.m[t] + encoderCache.RUnlock() + if f != nil { + return f + } + + // To deal with recursive types, populate the map with an + // indirect func before we build it. This type waits on the + // real func (f) to be ready and then calls it. This indirect + // func is only used for recursive types. + encoderCache.Lock() + if encoderCache.m == nil { + encoderCache.m = make(map[reflect.Type]encoderFunc) + } + var wg sync.WaitGroup + wg.Add(1) + encoderCache.m[t] = func(e *encodeState, v reflect.Value, quoted bool) { + wg.Wait() + f(e, v, quoted) + } + encoderCache.Unlock() + + // Compute fields without lock. + // Might duplicate effort but won't hold other computations back. + f = newTypeEncoder(t, true) + wg.Done() + encoderCache.Lock() + encoderCache.m[t] = f + encoderCache.Unlock() + return f +} + +var ( + marshalerType = reflect.TypeOf(new(Marshaler)).Elem() + textMarshalerType = reflect.TypeOf(new(encoding.TextMarshaler)).Elem() +) + +// newTypeEncoder constructs an encoderFunc for a type. +// The returned encoder only checks CanAddr when allowAddr is true. +func newTypeEncoder(t reflect.Type, allowAddr bool) encoderFunc { + if t.Implements(marshalerType) { + return marshalerEncoder + } + if t.Kind() != reflect.Ptr && allowAddr { + if reflect.PtrTo(t).Implements(marshalerType) { + return newCondAddrEncoder(addrMarshalerEncoder, newTypeEncoder(t, false)) + } + } + + if t.Implements(textMarshalerType) { + return textMarshalerEncoder + } + if t.Kind() != reflect.Ptr && allowAddr { + if reflect.PtrTo(t).Implements(textMarshalerType) { + return newCondAddrEncoder(addrTextMarshalerEncoder, newTypeEncoder(t, false)) + } + } + + switch t.Kind() { + case reflect.Bool: + return boolEncoder + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return intEncoder + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return uintEncoder + case reflect.Float32: + return float32Encoder + case reflect.Float64: + return float64Encoder + case reflect.String: + return stringEncoder + case reflect.Interface: + return interfaceEncoder + case reflect.Struct: + return newStructEncoder(t) + case reflect.Map: + return newMapEncoder(t) + case reflect.Slice: + return newSliceEncoder(t) + case reflect.Array: + return newArrayEncoder(t) + case reflect.Ptr: + return newPtrEncoder(t) + default: + return unsupportedTypeEncoder + } +} + +func invalidValueEncoder(e *encodeState, v reflect.Value, quoted bool) { + e.WriteString("null") +} + +func marshalerEncoder(e *encodeState, v reflect.Value, quoted bool) { + if v.Kind() == reflect.Ptr && v.IsNil() { + e.WriteString("null") + return + } + m := v.Interface().(Marshaler) + b, err := m.MarshalJSON() + if err == nil { + // copy JSON into buffer, checking validity. + err = compact(&e.Buffer, b, true) + } + if err != nil { + e.error(&MarshalerError{v.Type(), err}) + } +} + +func addrMarshalerEncoder(e *encodeState, v reflect.Value, quoted bool) { + va := v.Addr() + if va.IsNil() { + e.WriteString("null") + return + } + m := va.Interface().(Marshaler) + b, err := m.MarshalJSON() + if err == nil { + // copy JSON into buffer, checking validity. + err = compact(&e.Buffer, b, true) + } + if err != nil { + e.error(&MarshalerError{v.Type(), err}) + } +} + +func textMarshalerEncoder(e *encodeState, v reflect.Value, quoted bool) { + if v.Kind() == reflect.Ptr && v.IsNil() { + e.WriteString("null") + return + } + m := v.Interface().(encoding.TextMarshaler) + b, err := m.MarshalText() + if err == nil { + _, err = e.stringBytes(b) + } + if err != nil { + e.error(&MarshalerError{v.Type(), err}) + } +} + +func addrTextMarshalerEncoder(e *encodeState, v reflect.Value, quoted bool) { + va := v.Addr() + if va.IsNil() { + e.WriteString("null") + return + } + m := va.Interface().(encoding.TextMarshaler) + b, err := m.MarshalText() + if err == nil { + _, err = e.stringBytes(b) + } + if err != nil { + e.error(&MarshalerError{v.Type(), err}) + } +} + +func boolEncoder(e *encodeState, v reflect.Value, quoted bool) { + if quoted { + e.WriteByte('"') + } + if v.Bool() { + e.WriteString("true") + } else { + e.WriteString("false") + } + if quoted { + e.WriteByte('"') + } +} + +func intEncoder(e *encodeState, v reflect.Value, quoted bool) { + b := strconv.AppendInt(e.scratch[:0], v.Int(), 10) + if quoted { + e.WriteByte('"') + } + e.Write(b) + if quoted { + e.WriteByte('"') + } +} + +func uintEncoder(e *encodeState, v reflect.Value, quoted bool) { + b := strconv.AppendUint(e.scratch[:0], v.Uint(), 10) + if quoted { + e.WriteByte('"') + } + e.Write(b) + if quoted { + e.WriteByte('"') + } +} + +type floatEncoder int // number of bits + +func (bits floatEncoder) encode(e *encodeState, v reflect.Value, quoted bool) { + f := v.Float() + if math.IsInf(f, 0) { + if math.IsInf(f, 1) { //positive inf + e.WriteString("+Infinity") + } else { //negative inf + e.WriteString("-Infinity") + } + return + } + if math.IsNaN(f) { + e.WriteString("NaN") + return + //e.error(&UnsupportedValueError{v, strconv.FormatFloat(f, 'g', -1, int(bits))}) + } + b := strconv.AppendFloat(e.scratch[:0], f, 'g', -1, int(bits)) + if quoted { + e.WriteByte('"') + } + e.Write(b) + if quoted { + e.WriteByte('"') + } +} + +var ( + float32Encoder = (floatEncoder(32)).encode + float64Encoder = (floatEncoder(64)).encode +) + +func stringEncoder(e *encodeState, v reflect.Value, quoted bool) { + if v.Type() == numberType { + numStr := v.String() + if numStr == "" { + numStr = "0" // Number's zero-val + } + e.WriteString(numStr) + return + } + if quoted { + sb, err := Marshal(v.String()) + if err != nil { + e.error(err) + } + e.string(string(sb)) + } else { + e.string(v.String()) + } +} + +func interfaceEncoder(e *encodeState, v reflect.Value, quoted bool) { + if v.IsNil() { + e.WriteString("null") + return + } + e.reflectValue(v.Elem()) +} + +func unsupportedTypeEncoder(e *encodeState, v reflect.Value, quoted bool) { + e.error(&UnsupportedTypeError{v.Type()}) +} + +type structEncoder struct { + fields []field + fieldEncs []encoderFunc +} + +func (se *structEncoder) encode(e *encodeState, v reflect.Value, quoted bool) { + e.WriteByte('{') + first := true + for i, f := range se.fields { + fv := fieldByIndex(v, f.index) + if !fv.IsValid() || f.omitEmpty && isEmptyValue(fv) { + continue + } + if first { + first = false + } else { + e.WriteByte(',') + } + e.string(f.name) + e.WriteByte(':') + se.fieldEncs[i](e, fv, f.quoted) + } + e.WriteByte('}') +} + +func newStructEncoder(t reflect.Type) encoderFunc { + fields := cachedTypeFields(t) + se := &structEncoder{ + fields: fields, + fieldEncs: make([]encoderFunc, len(fields)), + } + for i, f := range fields { + se.fieldEncs[i] = typeEncoder(typeByIndex(t, f.index)) + } + return se.encode +} + +type mapEncoder struct { + elemEnc encoderFunc +} + +func (me *mapEncoder) encode(e *encodeState, v reflect.Value, _ bool) { + if v.IsNil() { + e.WriteString("null") + return + } + e.WriteByte('{') + var sv stringValues + sv = v.MapKeys() + sort.Sort(sv) + for i, k := range sv { + if i > 0 { + e.WriteByte(',') + } + e.string(k.String()) + e.WriteByte(':') + me.elemEnc(e, v.MapIndex(k), false) + } + e.WriteByte('}') +} + +func newMapEncoder(t reflect.Type) encoderFunc { + if t.Key().Kind() != reflect.String { + return unsupportedTypeEncoder + } + me := &mapEncoder{typeEncoder(t.Elem())} + return me.encode +} + +func encodeByteSlice(e *encodeState, v reflect.Value, _ bool) { + if v.IsNil() { + e.WriteString("null") + return + } + s := v.Bytes() + e.WriteByte('"') + if len(s) < 1024 { + // for small buffers, using Encode directly is much faster. + dst := make([]byte, base64.StdEncoding.EncodedLen(len(s))) + base64.StdEncoding.Encode(dst, s) + e.Write(dst) + } else { + // for large buffers, avoid unnecessary extra temporary + // buffer space. + enc := base64.NewEncoder(base64.StdEncoding, e) + enc.Write(s) + enc.Close() + } + e.WriteByte('"') +} + +// sliceEncoder just wraps an arrayEncoder, checking to make sure the value isn't nil. +type sliceEncoder struct { + arrayEnc encoderFunc +} + +func (se *sliceEncoder) encode(e *encodeState, v reflect.Value, _ bool) { + if v.IsNil() { + e.WriteString("null") + return + } + se.arrayEnc(e, v, false) +} + +func newSliceEncoder(t reflect.Type) encoderFunc { + // Byte slices get special treatment; arrays don't. + if t.Elem().Kind() == reflect.Uint8 { + return encodeByteSlice + } + enc := &sliceEncoder{newArrayEncoder(t)} + return enc.encode +} + +type arrayEncoder struct { + elemEnc encoderFunc +} + +func (ae *arrayEncoder) encode(e *encodeState, v reflect.Value, _ bool) { + e.WriteByte('[') + n := v.Len() + for i := 0; i < n; i++ { + if i > 0 { + e.WriteByte(',') + } + ae.elemEnc(e, v.Index(i), false) + } + e.WriteByte(']') +} + +func newArrayEncoder(t reflect.Type) encoderFunc { + enc := &arrayEncoder{typeEncoder(t.Elem())} + return enc.encode +} + +type ptrEncoder struct { + elemEnc encoderFunc +} + +func (pe *ptrEncoder) encode(e *encodeState, v reflect.Value, _ bool) { + if v.IsNil() { + e.WriteString("null") + return + } + pe.elemEnc(e, v.Elem(), false) +} + +func newPtrEncoder(t reflect.Type) encoderFunc { + enc := &ptrEncoder{typeEncoder(t.Elem())} + return enc.encode +} + +type condAddrEncoder struct { + canAddrEnc, elseEnc encoderFunc +} + +func (ce *condAddrEncoder) encode(e *encodeState, v reflect.Value, quoted bool) { + if v.CanAddr() { + ce.canAddrEnc(e, v, quoted) + } else { + ce.elseEnc(e, v, quoted) + } +} + +// newCondAddrEncoder returns an encoder that checks whether its value +// CanAddr and delegates to canAddrEnc if so, else to elseEnc. +func newCondAddrEncoder(canAddrEnc, elseEnc encoderFunc) encoderFunc { + enc := &condAddrEncoder{canAddrEnc: canAddrEnc, elseEnc: elseEnc} + return enc.encode +} + +func isValidTag(s string) bool { + if s == "" { + return false + } + for _, c := range s { + switch { + case strings.ContainsRune("!#$%&()*+-./:<=>?@[]^_{|}~ ", c): + // Backslash and quote chars are reserved, but + // otherwise any punctuation chars are allowed + // in a tag name. + default: + if !unicode.IsLetter(c) && !unicode.IsDigit(c) { + return false + } + } + } + return true +} + +func fieldByIndex(v reflect.Value, index []int) reflect.Value { + for _, i := range index { + if v.Kind() == reflect.Ptr { + if v.IsNil() { + return reflect.Value{} + } + v = v.Elem() + } + v = v.Field(i) + } + return v +} + +func typeByIndex(t reflect.Type, index []int) reflect.Type { + for _, i := range index { + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + t = t.Field(i).Type + } + return t +} + +// stringValues is a slice of reflect.Value holding *reflect.StringValue. +// It implements the methods to sort by string. +type stringValues []reflect.Value + +func (sv stringValues) Len() int { return len(sv) } +func (sv stringValues) Swap(i, j int) { sv[i], sv[j] = sv[j], sv[i] } +func (sv stringValues) Less(i, j int) bool { return sv.get(i) < sv.get(j) } +func (sv stringValues) get(i int) string { return sv[i].String() } + +// NOTE: keep in sync with stringBytes below. +func (e *encodeState) string(s string) (int, error) { + len0 := e.Len() + e.WriteByte('"') + start := 0 + for i := 0; i < len(s); { + if b := s[i]; b < utf8.RuneSelf { + if 0x20 <= b && b != '\\' && b != '"' && b != '<' && b != '>' && b != '&' { + i++ + continue + } + if start < i { + e.WriteString(s[start:i]) + } + switch b { + case '\\', '"': + e.WriteByte('\\') + e.WriteByte(b) + case '\n': + e.WriteByte('\\') + e.WriteByte('n') + case '\r': + e.WriteByte('\\') + e.WriteByte('r') + default: + // This encodes bytes < 0x20 except for \n and \r, + // as well as <, > and &. The latter are escaped because they + // can lead to security holes when user-controlled strings + // are rendered into JSON and served to some browsers. + e.WriteString(`\u00`) + e.WriteByte(hex[b>>4]) + e.WriteByte(hex[b&0xF]) + } + i++ + start = i + continue + } + c, size := utf8.DecodeRuneInString(s[i:]) + if c == utf8.RuneError && size == 1 { + if start < i { + e.WriteString(s[start:i]) + } + e.WriteString(`\ufffd`) + i += size + start = i + continue + } + // U+2028 is LINE SEPARATOR. + // U+2029 is PARAGRAPH SEPARATOR. + // They are both technically valid characters in JSON strings, + // but don't work in JSONP, which has to be evaluated as JavaScript, + // and can lead to security holes there. It is valid JSON to + // escape them, so we do so unconditionally. + // See http://timelessrepo.com/json-isnt-a-javascript-subset for discussion. + if c == '\u2028' || c == '\u2029' { + if start < i { + e.WriteString(s[start:i]) + } + e.WriteString(`\u202`) + e.WriteByte(hex[c&0xF]) + i += size + start = i + continue + } + i += size + } + if start < len(s) { + e.WriteString(s[start:]) + } + e.WriteByte('"') + return e.Len() - len0, nil +} + +// NOTE: keep in sync with string above. +func (e *encodeState) stringBytes(s []byte) (int, error) { + len0 := e.Len() + e.WriteByte('"') + start := 0 + for i := 0; i < len(s); { + if b := s[i]; b < utf8.RuneSelf { + if 0x20 <= b && b != '\\' && b != '"' && b != '<' && b != '>' && b != '&' { + i++ + continue + } + if start < i { + e.Write(s[start:i]) + } + switch b { + case '\\', '"': + e.WriteByte('\\') + e.WriteByte(b) + case '\n': + e.WriteByte('\\') + e.WriteByte('n') + case '\r': + e.WriteByte('\\') + e.WriteByte('r') + default: + // This encodes bytes < 0x20 except for \n and \r, + // as well as < and >. The latter are escaped because they + // can lead to security holes when user-controlled strings + // are rendered into JSON and served to some browsers. + e.WriteString(`\u00`) + e.WriteByte(hex[b>>4]) + e.WriteByte(hex[b&0xF]) + } + i++ + start = i + continue + } + c, size := utf8.DecodeRune(s[i:]) + if c == utf8.RuneError && size == 1 { + if start < i { + e.Write(s[start:i]) + } + e.WriteString(`\ufffd`) + i += size + start = i + continue + } + // U+2028 is LINE SEPARATOR. + // U+2029 is PARAGRAPH SEPARATOR. + // They are both technically valid characters in JSON strings, + // but don't work in JSONP, which has to be evaluated as JavaScript, + // and can lead to security holes there. It is valid JSON to + // escape them, so we do so unconditionally. + // See http://timelessrepo.com/json-isnt-a-javascript-subset for discussion. + if c == '\u2028' || c == '\u2029' { + if start < i { + e.Write(s[start:i]) + } + e.WriteString(`\u202`) + e.WriteByte(hex[c&0xF]) + i += size + start = i + continue + } + i += size + } + if start < len(s) { + e.Write(s[start:]) + } + e.WriteByte('"') + return e.Len() - len0, nil +} + +// A field represents a single field found in a struct. +type field struct { + name string + nameBytes []byte // []byte(name) + equalFold func(s, t []byte) bool // bytes.EqualFold or equivalent + + tag bool + index []int + typ reflect.Type + omitEmpty bool + quoted bool +} + +func fillField(f field) field { + f.nameBytes = []byte(f.name) + f.equalFold = foldFunc(f.nameBytes) + return f +} + +// byName sorts field by name, breaking ties with depth, +// then breaking ties with "name came from json tag", then +// breaking ties with index sequence. +type byName []field + +func (x byName) Len() int { return len(x) } + +func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + +func (x byName) Less(i, j int) bool { + if x[i].name != x[j].name { + return x[i].name < x[j].name + } + if len(x[i].index) != len(x[j].index) { + return len(x[i].index) < len(x[j].index) + } + if x[i].tag != x[j].tag { + return x[i].tag + } + return byIndex(x).Less(i, j) +} + +// byIndex sorts field by index sequence. +type byIndex []field + +func (x byIndex) Len() int { return len(x) } + +func (x byIndex) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + +func (x byIndex) Less(i, j int) bool { + for k, xik := range x[i].index { + if k >= len(x[j].index) { + return false + } + if xik != x[j].index[k] { + return xik < x[j].index[k] + } + } + return len(x[i].index) < len(x[j].index) +} + +// typeFields returns a list of fields that JSON should recognize for the given type. +// The algorithm is breadth-first search over the set of structs to include - the top struct +// and then any reachable anonymous structs. +func typeFields(t reflect.Type) []field { + // Anonymous fields to explore at the current level and the next. + current := []field{} + next := []field{{typ: t}} + + // Count of queued names for current level and the next. + count := map[reflect.Type]int{} + nextCount := map[reflect.Type]int{} + + // Types already visited at an earlier level. + visited := map[reflect.Type]bool{} + + // Fields found. + var fields []field + + for len(next) > 0 { + current, next = next, current[:0] + count, nextCount = nextCount, map[reflect.Type]int{} + + for _, f := range current { + if visited[f.typ] { + continue + } + visited[f.typ] = true + + // Scan f.typ for fields to include. + for i := 0; i < f.typ.NumField(); i++ { + sf := f.typ.Field(i) + if sf.PkgPath != "" { // unexported + continue + } + tag := sf.Tag.Get("json") + if tag == "-" { + continue + } + name, opts := parseTag(tag) + if !isValidTag(name) { + name = "" + } + index := make([]int, len(f.index)+1) + copy(index, f.index) + index[len(f.index)] = i + + ft := sf.Type + if ft.Name() == "" && ft.Kind() == reflect.Ptr { + // Follow pointer. + ft = ft.Elem() + } + + // Record found field and index sequence. + if name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { + tagged := name != "" + if name == "" { + name = sf.Name + } + fields = append(fields, fillField(field{ + name: name, + tag: tagged, + index: index, + typ: ft, + omitEmpty: opts.Contains("omitempty"), + quoted: opts.Contains("string"), + })) + if count[f.typ] > 1 { + // If there were multiple instances, add a second, + // so that the annihilation code will see a duplicate. + // It only cares about the distinction between 1 or 2, + // so don't bother generating any more copies. + fields = append(fields, fields[len(fields)-1]) + } + continue + } + + // Record new anonymous struct to explore in next round. + nextCount[ft]++ + if nextCount[ft] == 1 { + next = append(next, fillField(field{name: ft.Name(), index: index, typ: ft})) + } + } + } + } + + sort.Sort(byName(fields)) + + // Delete all fields that are hidden by the Go rules for embedded fields, + // except that fields with JSON tags are promoted. + + // The fields are sorted in primary order of name, secondary order + // of field index length. Loop over names; for each name, delete + // hidden fields by choosing the one dominant field that survives. + out := fields[:0] + for advance, i := 0, 0; i < len(fields); i += advance { + // One iteration per name. + // Find the sequence of fields with the name of this first field. + fi := fields[i] + name := fi.name + for advance = 1; i+advance < len(fields); advance++ { + fj := fields[i+advance] + if fj.name != name { + break + } + } + if advance == 1 { // Only one field with this name + out = append(out, fi) + continue + } + dominant, ok := dominantField(fields[i : i+advance]) + if ok { + out = append(out, dominant) + } + } + + fields = out + sort.Sort(byIndex(fields)) + + return fields +} + +// dominantField looks through the fields, all of which are known to +// have the same name, to find the single field that dominates the +// others using Go's embedding rules, modified by the presence of +// JSON tags. If there are multiple top-level fields, the boolean +// will be false: This condition is an error in Go and we skip all +// the fields. +func dominantField(fields []field) (field, bool) { + // The fields are sorted in increasing index-length order. The winner + // must therefore be one with the shortest index length. Drop all + // longer entries, which is easy: just truncate the slice. + length := len(fields[0].index) + tagged := -1 // Index of first tagged field. + for i, f := range fields { + if len(f.index) > length { + fields = fields[:i] + break + } + if f.tag { + if tagged >= 0 { + // Multiple tagged fields at the same level: conflict. + // Return no field. + return field{}, false + } + tagged = i + } + } + if tagged >= 0 { + return fields[tagged], true + } + // All remaining fields have the same length. If there's more than one, + // we have a conflict (two fields named "X" at the same level) and we + // return no field. + if len(fields) > 1 { + return field{}, false + } + return fields[0], true +} + +var fieldCache struct { + sync.RWMutex + m map[reflect.Type][]field +} + +// cachedTypeFields is like typeFields but uses a cache to avoid repeated work. +func cachedTypeFields(t reflect.Type) []field { + fieldCache.RLock() + f := fieldCache.m[t] + fieldCache.RUnlock() + if f != nil { + return f + } + + // Compute fields without lock. + // Might duplicate effort but won't hold other computations back. + f = typeFields(t) + if f == nil { + f = []field{} + } + + fieldCache.Lock() + if fieldCache.m == nil { + fieldCache.m = map[reflect.Type][]field{} + } + fieldCache.m[t] = f + fieldCache.Unlock() + return f +} diff --git a/src/mongo/gotools/common/json/encode_test.go b/src/mongo/gotools/common/json/encode_test.go new file mode 100644 index 00000000000..8226a3df71a --- /dev/null +++ b/src/mongo/gotools/common/json/encode_test.go @@ -0,0 +1,453 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import ( + "bytes" + "math" + "reflect" + "testing" + "unicode" +) + +type Optionals struct { + Sr string `json:"sr"` + So string `json:"so,omitempty"` + Sw string `json:"-"` + + Ir int `json:"omitempty"` // actually named omitempty, not an option + Io int `json:"io,omitempty"` + + Slr []string `json:"slr,random"` + Slo []string `json:"slo,omitempty"` + + Mr map[string]interface{} `json:"mr"` + Mo map[string]interface{} `json:",omitempty"` + + Fr float64 `json:"fr"` + Fo float64 `json:"fo,omitempty"` + + Br bool `json:"br"` + Bo bool `json:"bo,omitempty"` + + Ur uint `json:"ur"` + Uo uint `json:"uo,omitempty"` + + Str struct{} `json:"str"` + Sto struct{} `json:"sto,omitempty"` +} + +var optionalsExpected = `{ + "sr": "", + "omitempty": 0, + "slr": null, + "mr": {}, + "fr": 0, + "br": false, + "ur": 0, + "str": {}, + "sto": {} +}` + +func TestOmitEmpty(t *testing.T) { + var o Optionals + o.Sw = "something" + o.Mr = map[string]interface{}{} + o.Mo = map[string]interface{}{} + + got, err := MarshalIndent(&o, "", " ") + if err != nil { + t.Fatal(err) + } + if got := string(got); got != optionalsExpected { + t.Errorf(" got: %s\nwant: %s\n", got, optionalsExpected) + } +} + +type StringTag struct { + BoolStr bool `json:",string"` + IntStr int64 `json:",string"` + StrStr string `json:",string"` +} + +var stringTagExpected = `{ + "BoolStr": "true", + "IntStr": "42", + "StrStr": "\"xzbit\"" +}` + +func TestStringTag(t *testing.T) { + var s StringTag + s.BoolStr = true + s.IntStr = 42 + s.StrStr = "xzbit" + got, err := MarshalIndent(&s, "", " ") + if err != nil { + t.Fatal(err) + } + if got := string(got); got != stringTagExpected { + t.Fatalf(" got: %s\nwant: %s\n", got, stringTagExpected) + } + + // Verify that it round-trips. + var s2 StringTag + err = NewDecoder(bytes.NewReader(got)).Decode(&s2) + if err != nil { + t.Fatalf("Decode: %v", err) + } + if !reflect.DeepEqual(s, s2) { + t.Fatalf("decode didn't match.\nsource: %#v\nEncoded as:\n%s\ndecode: %#v", s, string(got), s2) + } +} + +// byte slices are special even if they're renamed types. +type renamedByte byte +type renamedByteSlice []byte +type renamedRenamedByteSlice []renamedByte + +func TestEncodeRenamedByteSlice(t *testing.T) { + s := renamedByteSlice("abc") + result, err := Marshal(s) + if err != nil { + t.Fatal(err) + } + expect := `"YWJj"` + if string(result) != expect { + t.Errorf(" got %s want %s", result, expect) + } + r := renamedRenamedByteSlice("abc") + result, err = Marshal(r) + if err != nil { + t.Fatal(err) + } + if string(result) != expect { + t.Errorf(" got %s want %s", result, expect) + } +} + +func TestFloatSpecialValues(t *testing.T) { + _, err := Marshal(math.NaN()) + if err != nil { + t.Errorf("Got error for NaN: %v", err) + } + + _, err = Marshal(math.Inf(-1)) + if err != nil { + t.Errorf("Got error for -Inf: %v", err) + } + + _, err = Marshal(math.Inf(1)) + if err != nil { + t.Errorf("Got error for +Inf: %v", err) + } +} + +// Ref has Marshaler and Unmarshaler methods with pointer receiver. +type Ref int + +func (*Ref) MarshalJSON() ([]byte, error) { + return []byte(`"ref"`), nil +} + +func (r *Ref) UnmarshalJSON([]byte) error { + *r = 12 + return nil +} + +// Val has Marshaler methods with value receiver. +type Val int + +func (Val) MarshalJSON() ([]byte, error) { + return []byte(`"val"`), nil +} + +// RefText has Marshaler and Unmarshaler methods with pointer receiver. +type RefText int + +func (*RefText) MarshalText() ([]byte, error) { + return []byte(`"ref"`), nil +} + +func (r *RefText) UnmarshalText([]byte) error { + *r = 13 + return nil +} + +// ValText has Marshaler methods with value receiver. +type ValText int + +func (ValText) MarshalText() ([]byte, error) { + return []byte(`"val"`), nil +} + +func TestRefValMarshal(t *testing.T) { + var s = struct { + R0 Ref + R1 *Ref + R2 RefText + R3 *RefText + V0 Val + V1 *Val + V2 ValText + V3 *ValText + }{ + R0: 12, + R1: new(Ref), + R2: 14, + R3: new(RefText), + V0: 13, + V1: new(Val), + V2: 15, + V3: new(ValText), + } + const want = `{"R0":"ref","R1":"ref","R2":"\"ref\"","R3":"\"ref\"","V0":"val","V1":"val","V2":"\"val\"","V3":"\"val\""}` + b, err := Marshal(&s) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + if got := string(b); got != want { + t.Errorf("got %q, want %q", got, want) + } +} + +// C1 implements Marshaler and returns unescaped JSON. +type C1 int + +func (C1) MarshalJSON() ([]byte, error) { + return []byte(`"<&>"`), nil +} + +// CText implements Marshaler and returns unescaped text. +type CText int + +func (CText) MarshalText() ([]byte, error) { + return []byte(`"<&>"`), nil +} + +func TestMarshalerEscaping(t *testing.T) { + var c C1 + want := `"\u003c\u0026\u003e"` + b, err := Marshal(c) + if err != nil { + t.Fatalf("Marshal(c1): %v", err) + } + if got := string(b); got != want { + t.Errorf("Marshal(c1) = %#q, want %#q", got, want) + } + + var ct CText + want = `"\"\u003c\u0026\u003e\""` + b, err = Marshal(ct) + if err != nil { + t.Fatalf("Marshal(ct): %v", err) + } + if got := string(b); got != want { + t.Errorf("Marshal(ct) = %#q, want %#q", got, want) + } +} + +type IntType int + +type MyStruct struct { + IntType +} + +func TestAnonymousNonstruct(t *testing.T) { + var i IntType = 11 + a := MyStruct{i} + const want = `{"IntType":11}` + + b, err := Marshal(a) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + if got := string(b); got != want { + t.Errorf("got %q, want %q", got, want) + } +} + +type BugA struct { + S string +} + +type BugB struct { + BugA + S string +} + +type BugC struct { + S string +} + +// Legal Go: We never use the repeated embedded field (S). +type BugX struct { + A int + BugA + BugB +} + +// Issue 5245. +func TestEmbeddedBug(t *testing.T) { + v := BugB{ + BugA{"A"}, + "B", + } + b, err := Marshal(v) + if err != nil { + t.Fatal("Marshal:", err) + } + want := `{"S":"B"}` + got := string(b) + if got != want { + t.Fatalf("Marshal: got %s want %s", got, want) + } + // Now check that the duplicate field, S, does not appear. + x := BugX{ + A: 23, + } + b, err = Marshal(x) + if err != nil { + t.Fatal("Marshal:", err) + } + want = `{"A":23}` + got = string(b) + if got != want { + t.Fatalf("Marshal: got %s want %s", got, want) + } +} + +type BugD struct { // Same as BugA after tagging. + XXX string `json:"S"` +} + +// BugD's tagged S field should dominate BugA's. +type BugY struct { + BugA + BugD +} + +// Test that a field with a tag dominates untagged fields. +func TestTaggedFieldDominates(t *testing.T) { + v := BugY{ + BugA{"BugA"}, + BugD{"BugD"}, + } + b, err := Marshal(v) + if err != nil { + t.Fatal("Marshal:", err) + } + want := `{"S":"BugD"}` + got := string(b) + if got != want { + t.Fatalf("Marshal: got %s want %s", got, want) + } +} + +// There are no tags here, so S should not appear. +type BugZ struct { + BugA + BugC + BugY // Contains a tagged S field through BugD; should not dominate. +} + +func TestDuplicatedFieldDisappears(t *testing.T) { + v := BugZ{ + BugA{"BugA"}, + BugC{"BugC"}, + BugY{ + BugA{"nested BugA"}, + BugD{"nested BugD"}, + }, + } + b, err := Marshal(v) + if err != nil { + t.Fatal("Marshal:", err) + } + want := `{}` + got := string(b) + if got != want { + t.Fatalf("Marshal: got %s want %s", got, want) + } +} + +func TestStringBytes(t *testing.T) { + // Test that encodeState.stringBytes and encodeState.string use the same encoding. + es := &encodeState{} + var r []rune + for i := '\u0000'; i <= unicode.MaxRune; i++ { + r = append(r, i) + } + s := string(r) + "\xff\xff\xffhello" // some invalid UTF-8 too + _, err := es.string(s) + if err != nil { + t.Fatal(err) + } + + esBytes := &encodeState{} + _, err = esBytes.stringBytes([]byte(s)) + if err != nil { + t.Fatal(err) + } + + enc := es.Buffer.String() + encBytes := esBytes.Buffer.String() + if enc != encBytes { + i := 0 + for i < len(enc) && i < len(encBytes) && enc[i] == encBytes[i] { + i++ + } + enc = enc[i:] + encBytes = encBytes[i:] + i = 0 + for i < len(enc) && i < len(encBytes) && enc[len(enc)-i-1] == encBytes[len(encBytes)-i-1] { + i++ + } + enc = enc[:len(enc)-i] + encBytes = encBytes[:len(encBytes)-i] + + if len(enc) > 20 { + enc = enc[:20] + "..." + } + if len(encBytes) > 20 { + encBytes = encBytes[:20] + "..." + } + + t.Errorf("encodings differ at %#q vs %#q", enc, encBytes) + } +} + +func TestIssue6458(t *testing.T) { + type Foo struct { + M RawMessage + } + x := Foo{RawMessage(`"foo"`)} + + b, err := Marshal(&x) + if err != nil { + t.Fatal(err) + } + if want := `{"M":"foo"}`; string(b) != want { + t.Errorf("Marshal(&x) = %#q; want %#q", b, want) + } + + b, err = Marshal(x) + if err != nil { + t.Fatal(err) + } + + if want := `{"M":"ImZvbyI="}`; string(b) != want { + t.Errorf("Marshal(x) = %#q; want %#q", b, want) + } +} + +func TestHTMLEscape(t *testing.T) { + var b, want bytes.Buffer + m := `{"M":"<html>foo &` + "\xe2\x80\xa8 \xe2\x80\xa9" + `</html>"}` + want.Write([]byte(`{"M":"\u003chtml\u003efoo \u0026\u2028 \u2029\u003c/html\u003e"}`)) + HTMLEscape(&b, []byte(m)) + if !bytes.Equal(b.Bytes(), want.Bytes()) { + t.Errorf("HTMLEscape(&b, []byte(m)) = %s; want %s", b.Bytes(), want.Bytes()) + } +} diff --git a/src/mongo/gotools/common/json/example_test.go b/src/mongo/gotools/common/json/example_test.go new file mode 100644 index 00000000000..ca4e5ae68d4 --- /dev/null +++ b/src/mongo/gotools/common/json/example_test.go @@ -0,0 +1,161 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json_test + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log" + "os" + "strings" +) + +func ExampleMarshal() { + type ColorGroup struct { + ID int + Name string + Colors []string + } + group := ColorGroup{ + ID: 1, + Name: "Reds", + Colors: []string{"Crimson", "Red", "Ruby", "Maroon"}, + } + b, err := json.Marshal(group) + if err != nil { + fmt.Println("error:", err) + } + os.Stdout.Write(b) + // Output: + // {"ID":1,"Name":"Reds","Colors":["Crimson","Red","Ruby","Maroon"]} +} + +func ExampleUnmarshal() { + var jsonBlob = []byte(`[ + {"Name": "Platypus", "Order": "Monotremata"}, + {"Name": "Quoll", "Order": "Dasyuromorphia"} + ]`) + type Animal struct { + Name string + Order string + } + var animals []Animal + err := json.Unmarshal(jsonBlob, &animals) + if err != nil { + fmt.Println("error:", err) + } + fmt.Printf("%+v", animals) + // Output: + // [{Name:Platypus Order:Monotremata} {Name:Quoll Order:Dasyuromorphia}] +} + +// This example uses a Decoder to decode a stream of distinct JSON values. +func ExampleDecoder() { + const jsonStream = ` + {"Name": "Ed", "Text": "Knock knock."} + {"Name": "Sam", "Text": "Who's there?"} + {"Name": "Ed", "Text": "Go fmt."} + {"Name": "Sam", "Text": "Go fmt who?"} + {"Name": "Ed", "Text": "Go fmt yourself!"} + ` + type Message struct { + Name, Text string + } + dec := json.NewDecoder(strings.NewReader(jsonStream)) + for { + var m Message + if err := dec.Decode(&m); err == io.EOF { + break + } else if err != nil { + log.Fatal(err) + } + fmt.Printf("%s: %s\n", m.Name, m.Text) + } + // Output: + // Ed: Knock knock. + // Sam: Who's there? + // Ed: Go fmt. + // Sam: Go fmt who? + // Ed: Go fmt yourself! +} + +// This example uses RawMessage to delay parsing part of a JSON message. +func ExampleRawMessage() { + type Color struct { + Space string + Point json.RawMessage // delay parsing until we know the color space + } + type RGB struct { + R uint8 + G uint8 + B uint8 + } + type YCbCr struct { + Y uint8 + Cb int8 + Cr int8 + } + + var j = []byte(`[ + {"Space": "YCbCr", "Point": {"Y": 255, "Cb": 0, "Cr": -10}}, + {"Space": "RGB", "Point": {"R": 98, "G": 218, "B": 255}} + ]`) + var colors []Color + err := json.Unmarshal(j, &colors) + if err != nil { + log.Fatalln("error:", err) + } + + for _, c := range colors { + var dst interface{} + switch c.Space { + case "RGB": + dst = new(RGB) + case "YCbCr": + dst = new(YCbCr) + } + err := json.Unmarshal(c.Point, dst) + if err != nil { + log.Fatalln("error:", err) + } + fmt.Println(c.Space, dst) + } + // Output: + // YCbCr &{255 0 -10} + // RGB &{98 218 255} +} + +func ExampleIndent() { + type Road struct { + Name string + Number int + } + roads := []Road{ + {"Diamond Fork", 29}, + {"Sheep Creek", 51}, + } + + b, err := json.Marshal(roads) + if err != nil { + log.Fatal(err) + } + + var out bytes.Buffer + json.Indent(&out, b, "=", "\t") + out.WriteTo(os.Stdout) + // Output: + // [ + // = { + // = "Name": "Diamond Fork", + // = "Number": 29 + // = }, + // = { + // = "Name": "Sheep Creek", + // = "Number": 51 + // = } + // =] +} diff --git a/src/mongo/gotools/common/json/float_test.go b/src/mongo/gotools/common/json/float_test.go new file mode 100644 index 00000000000..32b9a6315d6 --- /dev/null +++ b/src/mongo/gotools/common/json/float_test.go @@ -0,0 +1,93 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestNumberFloatValue(t *testing.T) { + + Convey("When unmarshaling JSON with float values", t, func() { + + Convey("converts to a JSON NumberFloat value", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "5.5" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(float64) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, NumberFloat(5.5)) + + }) + }) + + Convey("When unmarshaling and marshaling NumberFloat values", t, func() { + key := "key" + + Convey("maintains decimal point with trailing zero", func() { + var jsonMap map[string]interface{} + + value := "5.0" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(float64) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, NumberFloat(5.0)) + + numFloat := NumberFloat(jsonValue) + byteValue, err := numFloat.MarshalJSON() + So(err, ShouldBeNil) + So(string(byteValue), ShouldEqual, "5.0") + + }) + + Convey("maintains precision with large decimals", func() { + var jsonMap map[string]interface{} + + value := "5.52342123" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(float64) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, NumberFloat(5.52342123)) + + numFloat := NumberFloat(jsonValue) + byteValue, err := numFloat.MarshalJSON() + So(err, ShouldBeNil) + So(string(byteValue), ShouldEqual, "5.52342123") + + }) + + Convey("maintains exponent values", func() { + var jsonMap map[string]interface{} + + value := "5e+32" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(float64) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, NumberFloat(5e32)) + + numFloat := NumberFloat(jsonValue) + byteValue, err := numFloat.MarshalJSON() + So(err, ShouldBeNil) + So(string(byteValue), ShouldEqual, "5e+32") + + }) + }) +} diff --git a/src/mongo/gotools/common/json/fold.go b/src/mongo/gotools/common/json/fold.go new file mode 100644 index 00000000000..d6f77c93e57 --- /dev/null +++ b/src/mongo/gotools/common/json/fold.go @@ -0,0 +1,143 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import ( + "bytes" + "unicode/utf8" +) + +const ( + caseMask = ^byte(0x20) // Mask to ignore case in ASCII. + kelvin = '\u212a' + smallLongEss = '\u017f' +) + +// foldFunc returns one of four different case folding equivalence +// functions, from most general (and slow) to fastest: +// +// 1) bytes.EqualFold, if the key s contains any non-ASCII UTF-8 +// 2) equalFoldRight, if s contains special folding ASCII ('k', 'K', 's', 'S') +// 3) asciiEqualFold, no special, but includes non-letters (including _) +// 4) simpleLetterEqualFold, no specials, no non-letters. +// +// The letters S and K are special because they map to 3 runes, not just 2: +// * S maps to s and to U+017F 'ſ' Latin small letter long s +// * k maps to K and to U+212A 'K' Kelvin sign +// See http://play.golang.org/p/tTxjOc0OGo +// +// The returned function is specialized for matching against s and +// should only be given s. It's not curried for performance reasons. +func foldFunc(s []byte) func(s, t []byte) bool { + nonLetter := false + special := false // special letter + for _, b := range s { + if b >= utf8.RuneSelf { + return bytes.EqualFold + } + upper := b & caseMask + if upper < 'A' || upper > 'Z' { + nonLetter = true + } else if upper == 'K' || upper == 'S' { + // See above for why these letters are special. + special = true + } + } + if special { + return equalFoldRight + } + if nonLetter { + return asciiEqualFold + } + return simpleLetterEqualFold +} + +// equalFoldRight is a specialization of bytes.EqualFold when s is +// known to be all ASCII (including punctuation), but contains an 's', +// 'S', 'k', or 'K', requiring a Unicode fold on the bytes in t. +// See comments on foldFunc. +func equalFoldRight(s, t []byte) bool { + for _, sb := range s { + if len(t) == 0 { + return false + } + tb := t[0] + if tb < utf8.RuneSelf { + if sb != tb { + sbUpper := sb & caseMask + if 'A' <= sbUpper && sbUpper <= 'Z' { + if sbUpper != tb&caseMask { + return false + } + } else { + return false + } + } + t = t[1:] + continue + } + // sb is ASCII and t is not. t must be either kelvin + // sign or long s; sb must be s, S, k, or K. + tr, size := utf8.DecodeRune(t) + switch sb { + case 's', 'S': + if tr != smallLongEss { + return false + } + case 'k', 'K': + if tr != kelvin { + return false + } + default: + return false + } + t = t[size:] + + } + if len(t) > 0 { + return false + } + return true +} + +// asciiEqualFold is a specialization of bytes.EqualFold for use when +// s is all ASCII (but may contain non-letters) and contains no +// special-folding letters. +// See comments on foldFunc. +func asciiEqualFold(s, t []byte) bool { + if len(s) != len(t) { + return false + } + for i, sb := range s { + tb := t[i] + if sb == tb { + continue + } + if ('a' <= sb && sb <= 'z') || ('A' <= sb && sb <= 'Z') { + if sb&caseMask != tb&caseMask { + return false + } + } else { + return false + } + } + return true +} + +// simpleLetterEqualFold is a specialization of bytes.EqualFold for +// use when s is all ASCII letters (no underscores, etc) and also +// doesn't contain 'k', 'K', 's', or 'S'. +// See comments on foldFunc. +func simpleLetterEqualFold(s, t []byte) bool { + if len(s) != len(t) { + return false + } + for i, b := range s { + if b&caseMask != t[i]&caseMask { + return false + } + } + return true +} diff --git a/src/mongo/gotools/common/json/fold_test.go b/src/mongo/gotools/common/json/fold_test.go new file mode 100644 index 00000000000..9fb94646a85 --- /dev/null +++ b/src/mongo/gotools/common/json/fold_test.go @@ -0,0 +1,116 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import ( + "bytes" + "strings" + "testing" + "unicode/utf8" +) + +var foldTests = []struct { + fn func(s, t []byte) bool + s, t string + want bool +}{ + {equalFoldRight, "", "", true}, + {equalFoldRight, "a", "a", true}, + {equalFoldRight, "", "a", false}, + {equalFoldRight, "a", "", false}, + {equalFoldRight, "a", "A", true}, + {equalFoldRight, "AB", "ab", true}, + {equalFoldRight, "AB", "ac", false}, + {equalFoldRight, "sbkKc", "ſbKKc", true}, + {equalFoldRight, "SbKkc", "ſbKKc", true}, + {equalFoldRight, "SbKkc", "ſbKK", false}, + {equalFoldRight, "e", "é", false}, + {equalFoldRight, "s", "S", true}, + + {simpleLetterEqualFold, "", "", true}, + {simpleLetterEqualFold, "abc", "abc", true}, + {simpleLetterEqualFold, "abc", "ABC", true}, + {simpleLetterEqualFold, "abc", "ABCD", false}, + {simpleLetterEqualFold, "abc", "xxx", false}, + + {asciiEqualFold, "a_B", "A_b", true}, + {asciiEqualFold, "aa@", "aa`", false}, // verify 0x40 and 0x60 aren't case-equivalent +} + +func TestFold(t *testing.T) { + for i, tt := range foldTests { + if got := tt.fn([]byte(tt.s), []byte(tt.t)); got != tt.want { + t.Errorf("%d. %q, %q = %v; want %v", i, tt.s, tt.t, got, tt.want) + } + truth := strings.EqualFold(tt.s, tt.t) + if truth != tt.want { + t.Errorf("strings.EqualFold doesn't agree with case %d", i) + } + } +} + +func TestFoldAgainstUnicode(t *testing.T) { + const bufSize = 5 + buf1 := make([]byte, 0, bufSize) + buf2 := make([]byte, 0, bufSize) + var runes []rune + for i := 0x20; i <= 0x7f; i++ { + runes = append(runes, rune(i)) + } + runes = append(runes, kelvin, smallLongEss) + + funcs := []struct { + name string + fold func(s, t []byte) bool + letter bool // must be ASCII letter + simple bool // must be simple ASCII letter (not 'S' or 'K') + }{ + { + name: "equalFoldRight", + fold: equalFoldRight, + }, + { + name: "asciiEqualFold", + fold: asciiEqualFold, + simple: true, + }, + { + name: "simpleLetterEqualFold", + fold: simpleLetterEqualFold, + simple: true, + letter: true, + }, + } + + for _, ff := range funcs { + for _, r := range runes { + if r >= utf8.RuneSelf { + continue + } + if ff.letter && !isASCIILetter(byte(r)) { + continue + } + if ff.simple && (r == 's' || r == 'S' || r == 'k' || r == 'K') { + continue + } + for _, r2 := range runes { + buf1 := append(buf1[:0], 'x') + buf2 := append(buf2[:0], 'x') + buf1 = buf1[:1+utf8.EncodeRune(buf1[1:bufSize], r)] + buf2 = buf2[:1+utf8.EncodeRune(buf2[1:bufSize], r2)] + buf1 = append(buf1, 'x') + buf2 = append(buf2, 'x') + want := bytes.EqualFold(buf1, buf2) + if got := ff.fold(buf1, buf2); got != want { + t.Errorf("%s(%q, %q) = %v; want %v", ff.name, buf1, buf2, got, want) + } + } + } + } +} + +func isASCIILetter(b byte) bool { + return ('A' <= b && b <= 'Z') || ('a' <= b && b <= 'z') +} diff --git a/src/mongo/gotools/common/json/frac_test.go b/src/mongo/gotools/common/json/frac_test.go new file mode 100644 index 00000000000..33d5058cad9 --- /dev/null +++ b/src/mongo/gotools/common/json/frac_test.go @@ -0,0 +1,98 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestFractionalNumber(t *testing.T) { + + Convey("When unmarshalling JSON with fractional numeric values "+ + "without a leading zero", t, func() { + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + key := "key" + value := ".123" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(float64) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldAlmostEqual, 0.123) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + value1, value2, value3 := ".123", ".456", ".789" + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value1, key2, value2, key3, value3) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue1, ok := jsonMap[key1].(float64) + So(ok, ShouldBeTrue) + So(jsonValue1, ShouldAlmostEqual, 0.123) + + jsonValue2, ok := jsonMap[key2].(float64) + So(ok, ShouldBeTrue) + So(jsonValue2, ShouldAlmostEqual, 0.456) + + jsonValue3, ok := jsonMap[key3].(float64) + So(ok, ShouldBeTrue) + So(jsonValue3, ShouldAlmostEqual, 0.789) + }) + + Convey("works in an array", func() { + var jsonMap map[string]interface{} + + key := "key" + value := ".42" + data := fmt.Sprintf(`{"%v":[%v,%v,%v]}`, + key, value, value, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonArray, ok := jsonMap[key].([]interface{}) + So(ok, ShouldBeTrue) + + for _, _jsonValue := range jsonArray { + jsonValue, ok := _jsonValue.(float64) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldAlmostEqual, 0.42) + } + }) + + Convey("can have a sign ('+' or '-')", func() { + var jsonMap map[string]interface{} + + key := "key" + value := ".106" + data := fmt.Sprintf(`{"%v":+%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(float64) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldAlmostEqual, 0.106) + + data = fmt.Sprintf(`{"%v":-%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok = jsonMap[key].(float64) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldAlmostEqual, -0.106) + }) + }) +} diff --git a/src/mongo/gotools/common/json/helpers.go b/src/mongo/gotools/common/json/helpers.go new file mode 100644 index 00000000000..09b7e3ca5b9 --- /dev/null +++ b/src/mongo/gotools/common/json/helpers.go @@ -0,0 +1,76 @@ +package json + +import "fmt" + +// Returns true if the byte array represents the null literal, +// and false otherwise. Assumes that `nu` is sufficient to distinguish +// between these cases. +func isNull(s []byte) bool { + return len(s) > 1 && s[0] == 'n' && s[1] == 'u' +} + +// Returns true if the byte array represents some kind of number literal, +// e.g. +123, -0.456, NaN, or Infinity, and false otherwise. Assumes that +// the first character is sufficient to distinguish between these cases +// with the exception of `N` where the second letter must be checked. +func isNumber(s []byte) bool { + if len(s) == 0 { + return false + } + if len(s) > 1 && (s[0] == 'N' && s[1] == 'a') || (s[0] == 'I' && s[1] == 'n') { // NaN + return true + } + return s[0] == '+' || s[0] == '-' || s[0] == '.' || (s[0] >= '0' && s[0] <= '9') +} + +// Returns true if the string represents the start of a hexadecimal +// literal, e.g. 0X123, -0x456, +0x789. +func isHexPrefix(s string) bool { + if len(s) > 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X') { + return true + } + return (s[0] == '+' || s[0] == '-') && isHexPrefix(s[1:]) +} + +// Returns the accept state (transition function) if x is empty. +// Otherwise returns a function that upon matching the first element +// of x will generate another function to match the second, etc. +// (or accept if no remaining elements). +func generateState(name string, x []byte, accept func(*scanner, int) int) func(*scanner, int) int { + if len(x) == 0 { + return accept + } + + return func(s *scanner, c int) int { + if c == int(x[0]) { + s.step = generateState(name, x[1:], accept) + return scanContinue + } + return s.error(c, fmt.Sprintf("in literal %v (expecting '%v')", name, string(x[0]))) + } +} + +// stateOptionalConstructor is the state where there is the possibility of entering an empty constructor. +func stateOptionalConstructor(s *scanner, c int) int { + if c <= ' ' && isSpace(rune(c)) { + return scanContinue + } + if c == '(' { + s.step = stateInParen + return scanContinue + } + return stateEndValue(s, c) +} + +// stateInParen is the state when inside a `(` waiting for a `)` +func stateInParen(s *scanner, c int) int { + if c <= ' ' && isSpace(rune(c)) { + return scanContinue + } + if c == ')' { + s.step = stateEndValue + return scanContinue + } + return s.error(c, "expecting ')' as next character") + +} diff --git a/src/mongo/gotools/common/json/hex.go b/src/mongo/gotools/common/json/hex.go new file mode 100644 index 00000000000..625dcab6493 --- /dev/null +++ b/src/mongo/gotools/common/json/hex.go @@ -0,0 +1,13 @@ +package json + +// Transition function for recognizing hexadecimal numbers. +// Adapted from encoding/json/scanner.go. + +// stateHex is the state after reading `0x` or `0X`. +func stateHex(s *scanner, c int) int { + if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { + s.step = stateHex + return scanContinue + } + return stateEndValue(s, c) +} diff --git a/src/mongo/gotools/common/json/hex_test.go b/src/mongo/gotools/common/json/hex_test.go new file mode 100644 index 00000000000..3067d07d34f --- /dev/null +++ b/src/mongo/gotools/common/json/hex_test.go @@ -0,0 +1,117 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestHexadecimalNumber(t *testing.T) { + value := "0x123" + intValue := 0x123 + + Convey("When unmarshalling JSON with hexadecimal numeric values", t, func() { + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + key := "key" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + jsonValue, ok := jsonMap[key].(int32) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, intValue) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + value1, value2, value3 := "0x100", "0x101", "0x102" + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value1, key2, value2, key3, value3) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue1, ok := jsonMap[key1].(int32) + So(ok, ShouldBeTrue) + So(jsonValue1, ShouldEqual, 0x100) + + jsonValue2, ok := jsonMap[key2].(int32) + So(ok, ShouldBeTrue) + So(jsonValue2, ShouldEqual, 0x101) + + jsonValue3, ok := jsonMap[key3].(int32) + So(ok, ShouldBeTrue) + So(jsonValue3, ShouldEqual, 0x102) + }) + + Convey("works in an array", func() { + var jsonMap map[string]interface{} + + key := "key" + data := fmt.Sprintf(`{"%v":[%v,%v,%v]}`, + key, value, value, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonArray, ok := jsonMap[key].([]interface{}) + So(ok, ShouldBeTrue) + + for _, _jsonValue := range jsonArray { + jsonValue, ok := _jsonValue.(int32) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, intValue) + } + }) + + Convey("can have a sign ('+' or '-')", func() { + var jsonMap map[string]interface{} + + key := "key" + data := fmt.Sprintf(`{"%v":+%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(int32) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, intValue) + + data = fmt.Sprintf(`{"%v":-%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok = jsonMap[key].(int32) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, -intValue) + }) + + Convey("can use '0x' or '0X' prefix", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "123" + data := fmt.Sprintf(`{"%v":0x%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(int32) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, intValue) + + data = fmt.Sprintf(`{"%v":0X%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok = jsonMap[key].(int32) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, intValue) + }) + }) +} diff --git a/src/mongo/gotools/common/json/indent.go b/src/mongo/gotools/common/json/indent.go new file mode 100644 index 00000000000..e1bacafd6b8 --- /dev/null +++ b/src/mongo/gotools/common/json/indent.go @@ -0,0 +1,137 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import "bytes" + +// Compact appends to dst the JSON-encoded src with +// insignificant space characters elided. +func Compact(dst *bytes.Buffer, src []byte) error { + return compact(dst, src, false) +} + +func compact(dst *bytes.Buffer, src []byte, escape bool) error { + origLen := dst.Len() + var scan scanner + scan.reset() + start := 0 + for i, c := range src { + if escape && (c == '<' || c == '>' || c == '&') { + if start < i { + dst.Write(src[start:i]) + } + dst.WriteString(`\u00`) + dst.WriteByte(hex[c>>4]) + dst.WriteByte(hex[c&0xF]) + start = i + 1 + } + // Convert U+2028 and U+2029 (E2 80 A8 and E2 80 A9). + if c == 0xE2 && i+2 < len(src) && src[i+1] == 0x80 && src[i+2]&^1 == 0xA8 { + if start < i { + dst.Write(src[start:i]) + } + dst.WriteString(`\u202`) + dst.WriteByte(hex[src[i+2]&0xF]) + start = i + 3 + } + v := scan.step(&scan, int(c)) + if v >= scanSkipSpace { + if v == scanError { + break + } + if start < i { + dst.Write(src[start:i]) + } + start = i + 1 + } + } + if scan.eof() == scanError { + dst.Truncate(origLen) + return scan.err + } + if start < len(src) { + dst.Write(src[start:]) + } + return nil +} + +func newline(dst *bytes.Buffer, prefix, indent string, depth int) { + dst.WriteByte('\n') + dst.WriteString(prefix) + for i := 0; i < depth; i++ { + dst.WriteString(indent) + } +} + +// Indent appends to dst an indented form of the JSON-encoded src. +// Each element in a JSON object or array begins on a new, +// indented line beginning with prefix followed by one or more +// copies of indent according to the indentation nesting. +// The data appended to dst does not begin with the prefix nor +// any indentation, and has no trailing newline, to make it +// easier to embed inside other formatted JSON data. +func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { + origLen := dst.Len() + var scan scanner + scan.reset() + needIndent := false + depth := 0 + for _, c := range src { + scan.bytes++ + v := scan.step(&scan, int(c)) + if v == scanSkipSpace { + continue + } + if v == scanError { + break + } + if needIndent && v != scanEndObject && v != scanEndArray { + needIndent = false + depth++ + newline(dst, prefix, indent, depth) + } + + // Emit semantically uninteresting bytes + // (in particular, punctuation in strings) unmodified. + if v == scanContinue { + dst.WriteByte(c) + continue + } + + // Add spacing around real punctuation. + switch c { + case '{', '[': + // delay indent so that empty object and array are formatted as {} and []. + needIndent = true + dst.WriteByte(c) + + case ',': + dst.WriteByte(c) + newline(dst, prefix, indent, depth) + + case ':': + dst.WriteByte(c) + dst.WriteByte(' ') + + case '}', ']': + if needIndent { + // suppress indent in empty object/array + needIndent = false + } else { + depth-- + newline(dst, prefix, indent, depth) + } + dst.WriteByte(c) + + default: + dst.WriteByte(c) + } + } + if scan.eof() == scanError { + dst.Truncate(origLen) + return scan.err + } + return nil +} diff --git a/src/mongo/gotools/common/json/infinity.go b/src/mongo/gotools/common/json/infinity.go new file mode 100644 index 00000000000..d5504ce0a44 --- /dev/null +++ b/src/mongo/gotools/common/json/infinity.go @@ -0,0 +1,13 @@ +package json + +// Transition functions for recognizing Infinity. +// Adapted from encoding/json/scanner.go. + +// stateI is the state after reading `In`. +func stateIn(s *scanner, c int) int { + if c == 'f' { + s.step = generateState("Infinity", []byte("inity"), stateEndValue) + return scanContinue + } + return s.error(c, "in literal Infinity (expecting 'f')") +} diff --git a/src/mongo/gotools/common/json/infinity_test.go b/src/mongo/gotools/common/json/infinity_test.go new file mode 100644 index 00000000000..d02eb0fcbf5 --- /dev/null +++ b/src/mongo/gotools/common/json/infinity_test.go @@ -0,0 +1,98 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "math" + "testing" +) + +func TestInfinityValue(t *testing.T) { + + Convey("When unmarshalling JSON with Infinity values", t, func() { + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "Infinity" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(float64) + So(ok, ShouldBeTrue) + So(math.IsInf(jsonValue, 1), ShouldBeTrue) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + value := "Infinity" + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value, key2, value, key3, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue1, ok := jsonMap[key1].(float64) + So(ok, ShouldBeTrue) + So(math.IsInf(jsonValue1, 1), ShouldBeTrue) + + jsonValue2, ok := jsonMap[key2].(float64) + So(ok, ShouldBeTrue) + So(math.IsInf(jsonValue2, 1), ShouldBeTrue) + + jsonValue3, ok := jsonMap[key3].(float64) + So(ok, ShouldBeTrue) + So(math.IsInf(jsonValue3, 1), ShouldBeTrue) + }) + + Convey("works in an array", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "Infinity" + data := fmt.Sprintf(`{"%v":[%v,%v,%v]}`, + key, value, value, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonArray, ok := jsonMap[key].([]interface{}) + So(ok, ShouldBeTrue) + + for _, _jsonValue := range jsonArray { + jsonValue, ok := _jsonValue.(float64) + So(ok, ShouldBeTrue) + So(math.IsInf(jsonValue, 1), ShouldBeTrue) + } + }) + + Convey("can have a sign ('+' or '-')", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "Infinity" + data := fmt.Sprintf(`{"%v":+%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(float64) + So(ok, ShouldBeTrue) + So(math.IsInf(jsonValue, 1), ShouldBeTrue) + + data = fmt.Sprintf(`{"%v":-%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok = jsonMap[key].(float64) + So(ok, ShouldBeTrue) + So(math.IsInf(jsonValue, -1), ShouldBeTrue) + }) + }) +} diff --git a/src/mongo/gotools/common/json/iso_date.go b/src/mongo/gotools/common/json/iso_date.go new file mode 100644 index 00000000000..1144257edbf --- /dev/null +++ b/src/mongo/gotools/common/json/iso_date.go @@ -0,0 +1,45 @@ +package json + +import ( + "fmt" + "reflect" +) + +// Transition functions for recognizing ISODate. +// Adapted from encoding/json/scanner.go. + +// stateIS is the state after reading `IS`. +func stateIS(s *scanner, c int) int { + if c == 'O' { + s.step = stateISO + return scanContinue + } + return s.error(c, "in literal ISODate (expecting 'O')") +} + +// stateISO is the state after reading `ISO`. +func stateISO(s *scanner, c int) int { + if c == 'D' { + s.step = stateD + return scanContinue + } + return s.error(c, "in literal ISODate (expecting 'D')") +} + +// Decodes a ISODate literal stored in the underlying byte data into v. +func (d *decodeState) storeISODate(v reflect.Value) { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginCtor { + d.error(fmt.Errorf("expected beginning of constructor")) + } + args, err := d.ctor("ISODate", []reflect.Type{isoDateType}) + if err != nil { + d.error(err) + } + switch kind := v.Kind(); kind { + case reflect.Interface: + v.Set(args[0]) + default: + d.error(fmt.Errorf("cannot store %v value into %v type", isoDateType, kind)) + } +} diff --git a/src/mongo/gotools/common/json/iso_date_test.go b/src/mongo/gotools/common/json/iso_date_test.go new file mode 100644 index 00000000000..2a34435d3a1 --- /dev/null +++ b/src/mongo/gotools/common/json/iso_date_test.go @@ -0,0 +1,124 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestISODateValue(t *testing.T) { + + Convey("When unmarshalling JSON with ISODate values", t, func() { + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "ISODate(\"2006-01-02T15:04-0700\")" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(ISODate) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, ISODate("2006-01-02T15:04-0700")) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + value1, value2, value3 := "ISODate(\"2006-01-02T15:04Z0700\")", "ISODate(\"2013-01-02T15:04Z0700\")", "ISODate(\"2014-02-02T15:04Z0700\")" + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value1, key2, value2, key3, value3) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue1, ok := jsonMap[key1].(ISODate) + So(ok, ShouldBeTrue) + So(jsonValue1, ShouldEqual, ISODate("2006-01-02T15:04Z0700")) + + jsonValue2, ok := jsonMap[key2].(ISODate) + So(ok, ShouldBeTrue) + So(jsonValue2, ShouldEqual, ISODate("2013-01-02T15:04Z0700")) + + jsonValue3, ok := jsonMap[key3].(ISODate) + So(ok, ShouldBeTrue) + So(jsonValue3, ShouldEqual, ISODate("2014-02-02T15:04Z0700")) + }) + + Convey("works in an array", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "ISODate(\"2006-01-02T15:04-0700\")" + data := fmt.Sprintf(`{"%v":[%v,%v,%v]}`, + key, value, value, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonArray, ok := jsonMap[key].([]interface{}) + So(ok, ShouldBeTrue) + + for _, _jsonValue := range jsonArray { + jsonValue, ok := _jsonValue.(ISODate) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, ISODate("2006-01-02T15:04-0700")) + } + }) + + Convey("will take valid format 2006-01-02T15:04:05.000-0700", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "ISODate(\"2006-01-02T15:04:05.000-0700\")" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(ISODate) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, ISODate("2006-01-02T15:04:05.000-0700")) + }) + + + Convey("will take valid format 2006-01-02T15:04:05", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "ISODate(\"2014-01-02T15:04:05Z\")" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(ISODate) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, ISODate("2014-01-02T15:04:05Z")) + }) + + + Convey("will take valid format 2006-01-02T15:04-0700", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "ISODate(\"2006-01-02T15:04-0700\")" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(ISODate) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, ISODate("2006-01-02T15:04-0700")) + }) + + + + }) +} + diff --git a/src/mongo/gotools/common/json/json_format.go b/src/mongo/gotools/common/json/json_format.go new file mode 100644 index 00000000000..f9243a4dcab --- /dev/null +++ b/src/mongo/gotools/common/json/json_format.go @@ -0,0 +1,163 @@ +package json + +import ( + "bytes" + "fmt" + "math" + "strconv" + "strings" + "time" +) + +const JSON_DATE_FORMAT = "2006-01-02T15:04:05.000Z" + +func (b BinData) MarshalJSON() ([]byte, error) { + data := fmt.Sprintf(`{ "$binary": "%v", "$type": "%0x" }`, + b.Base64, []byte{b.Type}) + return []byte(data), nil +} + +func (d128 Decimal128) MarshalJSON() ([]byte, error) { + s := d128.Decimal128.String() + return []byte(fmt.Sprintf(`{ "$numberDecimal" : "%s" }`, s)), nil +} + +func (js JavaScript) MarshalJSON() ([]byte, error) { + data := []byte(fmt.Sprintf(`{ "$code": %q`, js.Code)) + + scopeChunk := []byte{} + var err error + if js.Scope != nil { + scopeChunk, err = Marshal(js.Scope) + if err != nil { + return nil, err + } + scopeChunk = []byte(fmt.Sprintf(`, "$scope": %v `, string(scopeChunk))) + } + scopeChunk = append(scopeChunk, '}') + + data = append(data, scopeChunk...) + return data, nil +} + +func (d Date) MarshalJSON() ([]byte, error) { + var data string + n := int64(d) + if d.isFormatable() { + t := time.Unix(n/1e3, n%1e3*1e6) + data = fmt.Sprintf(`{ "$date": "%v" }`, t.UTC().Format(JSON_DATE_FORMAT)) + } else { + data = fmt.Sprintf(`{ "$date": { "$numberLong" : "%v" }}`, n) + } + + return []byte(data), nil +} + +func (d DBRef) MarshalJSON() ([]byte, error) { + // Convert the $id field to JSON + idChunk, err := Marshal(d.Id) + if err != nil { + return nil, err + } + + // Need to form JSON like { "$ref": "REF", "$id": ID, "$db": "DB" } + // so piece chunks together since can only get $id field as bytes. + refChunk := []byte(fmt.Sprintf(`{ "$ref": "%v", "$id": `, d.Collection)) + + dbChunk := []byte{} + if d.Database != "" { + dbChunk = []byte(fmt.Sprintf(`, "$db": "%v" `, d.Database)) + } + dbChunk = append(dbChunk, '}') + + data := make([]byte, len(refChunk)+len(idChunk)+len(dbChunk)) + copy(data, refChunk) + copy(data[len(refChunk):], idChunk) + copy(data[len(refChunk)+len(idChunk):], dbChunk) + + return data, nil +} + +func (d DBPointer) MarshalJSON() ([]byte, error) { + buffer := bytes.Buffer{} + // Convert the $id field to JSON + idChunk, err := Marshal(d.Id) + if err != nil { + return nil, err + } + buffer.Write([]byte(fmt.Sprintf(`{ "$ref": "%v", "$id": { "$oid" : `, d.Namespace))) + buffer.Write(idChunk) + buffer.Write([]byte("}}")) + return buffer.Bytes(), nil +} + +func (_ MinKey) MarshalJSON() ([]byte, error) { + data := `{ "$minKey": 1 }` + return []byte(data), nil +} + +func (_ MaxKey) MarshalJSON() ([]byte, error) { + data := `{ "$maxKey": 1 }` + return []byte(data), nil +} + +func (n NumberInt) MarshalJSON() ([]byte, error) { + return []byte(strconv.FormatInt(int64(n), 10)), nil +} + +func (n NumberLong) MarshalJSON() ([]byte, error) { + data := fmt.Sprintf(`{ "$numberLong": "%v" }`, int64(n)) + return []byte(data), nil +} + +func (n NumberFloat) MarshalJSON() ([]byte, error) { + + // check floats for infinity and return +Infinity or -Infinity if so + if math.IsInf(float64(n), 1) { + return []byte("+Infinity"), nil + } + if math.IsInf(float64(n), -1) { + return []byte("-Infinity"), nil + } + + floatString := strconv.FormatFloat(float64(n), 'g', -1, 64) + + // determine if the float has a decimal point and if not + // add one to maintain consistency when importing. + if _, d := math.Modf(float64(n)); d == 0 { + // check for 'e' to determine if the float's formatted in scientific notation + if strings.IndexByte(floatString, 'e') == -1 { + return []byte(floatString + ".0"), nil + } + + } + return []byte(floatString), nil +} + +// Assumes that o represents a valid ObjectId +// (composed of 24 hexadecimal characters). +func (o ObjectId) MarshalJSON() ([]byte, error) { + data := fmt.Sprintf(`{ "$oid": "%v" }`, string(o)) + return []byte(data), nil +} + +func (r RegExp) MarshalJSON() ([]byte, error) { + pattern, err := Marshal(r.Pattern) + if err != nil { + return nil, err + } + data := fmt.Sprintf(`{ "$regex": %v, "$options": "%v" }`, + string(pattern), r.Options) + return []byte(data), nil +} + +func (t Timestamp) MarshalJSON() ([]byte, error) { + data := fmt.Sprintf(`{ "$timestamp": { "t": %v, "i": %v } }`, + t.Seconds, t.Increment) + return []byte(data), nil +} + +func (_ Undefined) MarshalJSON() ([]byte, error) { + data := `{ "$undefined": true }` + return []byte(data), nil +} diff --git a/src/mongo/gotools/common/json/maxkey.go b/src/mongo/gotools/common/json/maxkey.go new file mode 100644 index 00000000000..6451fd9ae63 --- /dev/null +++ b/src/mongo/gotools/common/json/maxkey.go @@ -0,0 +1,13 @@ +package json + +// Transition functions for recognizing MaxKey. +// Adapted from encoding/json/scanner.go. + +// stateUpperMa is the state after reading `Ma`. +func stateUpperMa(s *scanner, c int) int { + if c == 'x' { + s.step = generateState("MaxKey", []byte("Key"), stateOptionalConstructor) + return scanContinue + } + return s.error(c, "in literal MaxKey (expecting 'x')") +} diff --git a/src/mongo/gotools/common/json/maxkey_test.go b/src/mongo/gotools/common/json/maxkey_test.go new file mode 100644 index 00000000000..4e6557b72b9 --- /dev/null +++ b/src/mongo/gotools/common/json/maxkey_test.go @@ -0,0 +1,180 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestMaxKeyValue(t *testing.T) { + + key := "key" + + Convey("Unmarshalling JSON with MaxKey values", t, func() { + value := "MaxKey" + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + data := fmt.Sprintf(`{"%v":%v}`, key, value) + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(MaxKey) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, MaxKey{}) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value, key2, value, key3, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue1, ok := jsonMap[key1].(MaxKey) + So(ok, ShouldBeTrue) + So(jsonValue1, ShouldResemble, MaxKey{}) + + jsonValue2, ok := jsonMap[key2].(MaxKey) + So(ok, ShouldBeTrue) + So(jsonValue2, ShouldResemble, MaxKey{}) + + jsonValue3, ok := jsonMap[key3].(MaxKey) + So(ok, ShouldBeTrue) + So(jsonValue3, ShouldResemble, MaxKey{}) + }) + + Convey("works in an array", func() { + var jsonMap map[string]interface{} + + data := fmt.Sprintf(`{"%v":[%v,%v,%v]}`, + key, value, value, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonArray, ok := jsonMap[key].([]interface{}) + So(ok, ShouldBeTrue) + + for _, _jsonValue := range jsonArray { + jsonValue, ok := _jsonValue.(MaxKey) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, MaxKey{}) + } + }) + + Convey("cannot have a sign ('+' or '-')", func() { + var jsonMap map[string]interface{} + + data := fmt.Sprintf(`{"%v":+%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + + data = fmt.Sprintf(`{"%v":-%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + }) + }) + + Convey("Unmarshalling JSON with MaxKey() values", t, func() { + value := "MaxKey()" + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(MaxKey) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, MaxKey{}) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value, key2, value, key3, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue1, ok := jsonMap[key1].(MaxKey) + So(ok, ShouldBeTrue) + So(jsonValue1, ShouldResemble, MaxKey{}) + + jsonValue2, ok := jsonMap[key2].(MaxKey) + So(ok, ShouldBeTrue) + So(jsonValue2, ShouldResemble, MaxKey{}) + + jsonValue3, ok := jsonMap[key3].(MaxKey) + So(ok, ShouldBeTrue) + So(jsonValue3, ShouldResemble, MaxKey{}) + }) + + Convey("works in an array", func() { + var jsonMap map[string]interface{} + + data := fmt.Sprintf(`{"%v":[%v,%v,%v]}`, + key, value, value, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonArray, ok := jsonMap[key].([]interface{}) + So(ok, ShouldBeTrue) + + for _, _jsonValue := range jsonArray { + jsonValue, ok := _jsonValue.(MaxKey) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, MaxKey{}) + } + }) + + Convey("cannot have a sign ('+' or '-')", func() { + var jsonMap map[string]interface{} + + data := fmt.Sprintf(`{"%v":+%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + + data = fmt.Sprintf(`{"%v":-%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + }) + + Convey("can have whitespace inside or around()", func() { + var jsonMap map[string]interface{} + + value = "MaxKey ( )" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(MaxKey) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, MaxKey{}) + }) + + Convey("cannot have any value other than whitespace inside ()", func() { + var jsonMap map[string]interface{} + value = "MaxKey(5)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + }) + }) +} diff --git a/src/mongo/gotools/common/json/minkey.go b/src/mongo/gotools/common/json/minkey.go new file mode 100644 index 00000000000..e87897d68dc --- /dev/null +++ b/src/mongo/gotools/common/json/minkey.go @@ -0,0 +1,13 @@ +package json + +// Transition functions for recognizing MinKey. +// Adapted from encoding/json/scanner.go. + +// stateUpperMi is the state after reading `Mi`. +func stateUpperMi(s *scanner, c int) int { + if c == 'n' { + s.step = generateState("MinKey", []byte("Key"), stateOptionalConstructor) + return scanContinue + } + return s.error(c, "in literal MinKey (expecting 'n')") +} diff --git a/src/mongo/gotools/common/json/minkey_test.go b/src/mongo/gotools/common/json/minkey_test.go new file mode 100644 index 00000000000..ccbf86aff64 --- /dev/null +++ b/src/mongo/gotools/common/json/minkey_test.go @@ -0,0 +1,184 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestMinKeyValue(t *testing.T) { + key := "key" + Convey("Unmarshalling JSON with MinKey values", t, func() { + + value := "MinKey" + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(MinKey) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, MinKey{}) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value, key2, value, key3, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue1, ok := jsonMap[key1].(MinKey) + So(ok, ShouldBeTrue) + So(jsonValue1, ShouldResemble, MinKey{}) + + jsonValue2, ok := jsonMap[key2].(MinKey) + So(ok, ShouldBeTrue) + So(jsonValue2, ShouldResemble, MinKey{}) + + jsonValue3, ok := jsonMap[key3].(MinKey) + So(ok, ShouldBeTrue) + So(jsonValue3, ShouldResemble, MinKey{}) + }) + + Convey("works in an array", func() { + var jsonMap map[string]interface{} + + data := fmt.Sprintf(`{"%v":[%v,%v,%v]}`, + key, value, value, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonArray, ok := jsonMap[key].([]interface{}) + So(ok, ShouldBeTrue) + + for _, _jsonValue := range jsonArray { + jsonValue, ok := _jsonValue.(MinKey) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, MinKey{}) + } + }) + + Convey("cannot have a sign ('+' or '-')", func() { + var jsonMap map[string]interface{} + + data := fmt.Sprintf(`{"%v":+%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + + data = fmt.Sprintf(`{"%v":-%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + }) + }) + + Convey("Unmarshalling JSON with MinKey() values", t, func() { + + value := "MinKey()" + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(MinKey) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, MinKey{}) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value, key2, value, key3, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue1, ok := jsonMap[key1].(MinKey) + So(ok, ShouldBeTrue) + So(jsonValue1, ShouldResemble, MinKey{}) + + jsonValue2, ok := jsonMap[key2].(MinKey) + So(ok, ShouldBeTrue) + So(jsonValue2, ShouldResemble, MinKey{}) + + jsonValue3, ok := jsonMap[key3].(MinKey) + So(ok, ShouldBeTrue) + So(jsonValue3, ShouldResemble, MinKey{}) + }) + + Convey("works in an array", func() { + var jsonMap map[string]interface{} + + data := fmt.Sprintf(`{"%v":[%v,%v,%v]}`, + key, value, value, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonArray, ok := jsonMap[key].([]interface{}) + So(ok, ShouldBeTrue) + + for _, _jsonValue := range jsonArray { + jsonValue, ok := _jsonValue.(MinKey) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, MinKey{}) + } + }) + + Convey("cannot have a sign ('+' or '-')", func() { + var jsonMap map[string]interface{} + + value := "MinKey()" + data := fmt.Sprintf(`{"%v":+%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + + data = fmt.Sprintf(`{"%v":-%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + }) + + Convey("can have whitespace inside or around()", func() { + var jsonMap map[string]interface{} + + value = "MinKey ( )" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(MinKey) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, MinKey{}) + }) + + Convey("cannot have any value other than whitespace inside ()", func() { + var jsonMap map[string]interface{} + + value = "MinKey(5)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + }) + }) + +} diff --git a/src/mongo/gotools/common/json/mongo_extjson.go b/src/mongo/gotools/common/json/mongo_extjson.go new file mode 100644 index 00000000000..606b1368b3c --- /dev/null +++ b/src/mongo/gotools/common/json/mongo_extjson.go @@ -0,0 +1,392 @@ +package json + +import ( + "fmt" + "gopkg.in/mgo.v2/bson" + "reflect" +) + +// Represents base-64 encoded binary data +type BinData struct { + Type byte + Base64 string +} + +// Represents the number of milliseconds since the Unix epoch. +type Date int64 + +type ISODate string + +type ObjectId string + +// Represents a reference to another document. +type DBRef struct { + Collection string + Id interface{} + Database string // optional +} + +// Refers to a document in some namespace by wrapping a string containing the namespace +// and the objectId in which the _id of the document is contained +type DBPointer struct { + Namespace string + Id bson.ObjectId +} + +// Represents the literal MinKey. +type MinKey struct{} + +// Represents the literal MaxKey. +type MaxKey struct{} + +// Represents a signed 32-bit integer. +type NumberInt int32 + +// Represents a signed 64-bit integer. +type NumberLong int64 + +// Represents a signed 64-bit float. +type NumberFloat float64 + +type Decimal128 struct { + bson.Decimal128 +} + +// Represents a regular expression. +type RegExp struct { + Pattern string + Options string +} + +// Represents a timestamp value. +type Timestamp struct { + Seconds uint32 + Increment uint32 +} + +type JavaScript struct { + Code string + Scope interface{} +} + +type Float float64 + +// Represents the literal undefined. +type Undefined struct{} + +var ( + // primitive types + byteType = reflect.TypeOf(byte(0)) + stringType = reflect.TypeOf(string("")) + uint32Type = reflect.TypeOf(uint32(0)) + + // object types + binDataType = reflect.TypeOf(BinData{}) + dateType = reflect.TypeOf(Date(0)) + isoDateType = reflect.TypeOf(ISODate("")) + dbRefType = reflect.TypeOf(DBRef{}) + dbPointerType = reflect.TypeOf(DBPointer{}) + maxKeyType = reflect.TypeOf(MaxKey{}) + minKeyType = reflect.TypeOf(MinKey{}) + numberIntType = reflect.TypeOf(NumberInt(0)) + numberLongType = reflect.TypeOf(NumberLong(0)) + numberFloatType = reflect.TypeOf(NumberFloat(0)) + objectIdType = reflect.TypeOf(ObjectId("")) + regexpType = reflect.TypeOf(RegExp{}) + timestampType = reflect.TypeOf(Timestamp{}) + undefinedType = reflect.TypeOf(Undefined{}) + orderedBSONType = reflect.TypeOf(bson.D{}) + interfaceType = reflect.TypeOf((*interface{})(nil)) +) + +func (d Date) isFormatable() bool { + return int64(d) < int64(32535215999000) +} + +func stateBeginExtendedValue(s *scanner, c int) int { + switch c { + case 'u': // beginning of undefined + s.step = stateU + case 'B': // beginning of BinData or Boolean + s.step = stateB + case 'D': // beginning of Date + s.step = stateD + case 'I': // beginning of Infinity or ISODate + s.step = stateI + case 'M': // beginning of MinKey or MaxKey + s.step = stateM + case 'N': // beginning of NaN or NumberXX + s.step = stateUpperN + case 'O': // beginning of ObjectId + s.step = stateO + case 'R': // beginning of RegExp + s.step = stateR + case 'T': // beginning of Timestamp + s.step = stateUpperT + case '/': // beginning of /foo/i + s.step = stateInRegexpPattern + default: + return s.error(c, "looking for beginning of value") + } + + return scanBeginLiteral +} + +// stateB is the state after reading `B`. +func stateB(s *scanner, c int) int { + if c == 'i' { + s.step = stateBi + return scanContinue + } + if c == 'o' { + s.step = stateBo + return scanContinue + } + return s.error(c, "in literal BinData or Boolean (expecting 'i' or 'o')") +} + +// stateUpperN is the state after reading `N`. +func stateUpperN(s *scanner, c int) int { + if c == 'a' { + s.step = stateUpperNa + return scanContinue + } + if c == 'u' { + s.step = stateUpperNu + return scanContinue + } + return s.error(c, "in literal NaN or Number (expecting 'a' or 'u')") +} + +// stateM is the state after reading `M`. +func stateM(s *scanner, c int) int { + if c == 'a' { + s.step = stateUpperMa + return scanContinue + } + if c == 'i' { + s.step = stateUpperMi + return scanContinue + } + return s.error(c, "in literal MaxKey or MinKey (expecting 'a' or 'i')") +} + +// stateD is the state after reading `D`. +func stateD(s *scanner, c int) int { + switch c { + case 'a': + s.step = stateDa + case 'B': + s.step = stateDB + case 'b': + s.step = stateDb + default: + return s.error(c, "in literal Date or DBRef (expecting 'a' or 'B')") + } + return scanContinue +} + +// stateDB is the state after reading `DB`. +func stateDB(s *scanner, c int) int { + if c == 'R' { + s.step = stateDBR + return scanContinue + } + if c == 'P' { + s.step = stateDBP + return scanContinue + } + return s.error(c, "in state DB (expecting 'R or P')") +} + +// stateI is the state after reading `I`. +func stateI(s *scanner, c int) int { + switch c { + case 'n': + s.step = stateIn + case 'S': + s.step = stateIS + default: + return s.error(c, "in literal Infinity or ISO (expecting 'n' or 'S')") + } + return scanContinue +} + +// Decodes a literal stored in item into v. +func (d *decodeState) storeExtendedLiteral(item []byte, v reflect.Value, fromQuoted bool) bool { + switch c := item[0]; c { + case 'n': + d.storeNewLiteral(v, fromQuoted) + + case 'u': // undefined + switch kind := v.Kind(); kind { + case reflect.Interface: + v.Set(reflect.ValueOf(Undefined{})) + default: + d.error(fmt.Errorf("cannot store %v value into %v type", undefinedType, kind)) + } + + case 'B': // BinData or Boolean + switch item[1] { + case 'i': // BinData + d.storeBinData(v) + case 'o': // Boolean + d.storeBoolean(v) + } + case 'D': // Date, DBRef, DBPointer, Dbpointer,or Dbref + switch item[1] { + case 'a': // Date + d.storeDate(v) + case 'b': // Dbref + d.storeDBRef(v) + case 'B': // DBRef or DBPointer + switch item[2] { + case 'R': //DBRef + d.storeDBRef(v) + case 'P': //DBPointer + d.storeDBPointer(v) + } + } + case 'I': + switch item[1] { + case 'S': // ISODate + d.storeISODate(v) + } + + case 'M': // MinKey or MaxKey + switch item[1] { + case 'i': // MinKey + switch kind := v.Kind(); kind { + case reflect.Interface: + v.Set(reflect.ValueOf(MinKey{})) + default: + d.error(fmt.Errorf("cannot store %v value into %v type", minKeyType, kind)) + } + case 'a': // MaxKey + switch kind := v.Kind(); kind { + case reflect.Interface: + v.Set(reflect.ValueOf(MaxKey{})) + default: + d.error(fmt.Errorf("cannot store %v value into %v type", maxKeyType, kind)) + } + } + + case 'O': // ObjectId + d.storeObjectId(v) + + case 'N': // NumberInt or NumberLong + switch item[6] { + case 'I': // NumberInt + d.storeNumberInt(v) + case 'L': // NumberLong + d.storeNumberLong(v) + } + + case 'R': // RegExp constructor + d.storeRegexp(v) + + case 'T': // Timestamp + d.storeTimestamp(v) + + case '/': // regular expression literal + op := d.scanWhile(scanSkipSpace) + if op != scanRegexpPattern { + d.error(fmt.Errorf("expected beginning of regular expression pattern")) + } + + pattern, options, err := d.regexp() + if err != nil { + d.error(err) + } + switch kind := v.Kind(); kind { + case reflect.Interface: + v.Set(reflect.ValueOf(RegExp{pattern, options})) + default: + d.error(fmt.Errorf("cannot store %v value into %v type", regexpType, kind)) + } + + default: + return false + } + + return true +} + +// Returns a literal from the underlying byte data. +func (d *decodeState) getExtendedLiteral(item []byte) (interface{}, bool) { + switch c := item[0]; c { + case 'n': + return d.getNewLiteral(), true + + case 'u': // undefined + return Undefined{}, true + + case 'B': // BinData or Boolean + switch item[1] { + case 'i': // BinData + return d.getBinData(), true + case 'o': // Boolean + return d.getBoolean(), true + } + + case 'D': // Date, DBRef, or Dbref + switch item[1] { + case 'a': // Date + return d.getDate(), true + case 'b': // Dbref + return d.getDBRef(), true + case 'B': // DBRef or DBPoiner + switch item[2] { + case 'R': // DBRef + return d.getDBRef(), true + case 'P': // DBPointer + return d.getDBPointer(), true + } + } + + case 'M': // MinKey or MaxKey + switch item[1] { + case 'i': // MinKey + return MinKey{}, true + case 'a': // MaxKey + return MaxKey{}, true + } + + case 'O': // ObjectId + return d.getObjectId(), true + + case 'N': // NumberInt or NumberLong + switch item[6] { + case 'I': // NumberInt + return d.getNumberInt(), true + case 'L': // NumberLong + return d.getNumberLong(), true + } + + case 'R': // RegExp constructor + return d.getRegexp(), true + + case 'T': // Timestamp + return d.getTimestamp(), true + + case 'I': // ISO Date + switch item[1] { + case 'S': // ISODate + return d.getDate(), true + } + + case '/': // regular expression literal + op := d.scanWhile(scanSkipSpace) + if op != scanRegexpPattern { + d.error(fmt.Errorf("expected beginning of regular expression pattern")) + } + + pattern, options, err := d.regexp() + if err != nil { + d.error(err) + } + return RegExp{pattern, options}, true + } + + return nil, false +} diff --git a/src/mongo/gotools/common/json/nan.go b/src/mongo/gotools/common/json/nan.go new file mode 100644 index 00000000000..23219906ce3 --- /dev/null +++ b/src/mongo/gotools/common/json/nan.go @@ -0,0 +1,13 @@ +package json + +// Transition functions for recognizing NaN. +// Adapted from encoding/json/scanner.go. + +// stateUpperNa is the state after reading `Na`. +func stateUpperNa(s *scanner, c int) int { + if c == 'N' { + s.step = stateEndValue + return scanContinue + } + return s.error(c, "in literal NaN (expecting 'N')") +} diff --git a/src/mongo/gotools/common/json/nan_test.go b/src/mongo/gotools/common/json/nan_test.go new file mode 100644 index 00000000000..6653d8c6d5d --- /dev/null +++ b/src/mongo/gotools/common/json/nan_test.go @@ -0,0 +1,90 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "math" + "testing" +) + +func TestNaNValue(t *testing.T) { + + Convey("When unmarshalling JSON with NaN values", t, func() { + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "NaN" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(float64) + So(ok, ShouldBeTrue) + So(math.IsNaN(jsonValue), ShouldBeTrue) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + value := "NaN" + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value, key2, value, key3, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue1, ok := jsonMap[key1].(float64) + So(ok, ShouldBeTrue) + So(math.IsNaN(jsonValue1), ShouldBeTrue) + + jsonValue2, ok := jsonMap[key2].(float64) + So(ok, ShouldBeTrue) + So(math.IsNaN(jsonValue2), ShouldBeTrue) + + jsonValue3, ok := jsonMap[key3].(float64) + So(ok, ShouldBeTrue) + So(math.IsNaN(jsonValue3), ShouldBeTrue) + }) + + Convey("works in an array", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "NaN" + data := fmt.Sprintf(`{"%v":[%v,%v,%v]}`, + key, value, value, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonArray, ok := jsonMap[key].([]interface{}) + So(ok, ShouldBeTrue) + + for _, _jsonValue := range jsonArray { + jsonValue, ok := _jsonValue.(float64) + So(ok, ShouldBeTrue) + So(math.IsNaN(jsonValue), ShouldBeTrue) + } + }) + + Convey("cannot have a sign ('+' or '-')", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "NaN" + data := fmt.Sprintf(`{"%v":+%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + + data = fmt.Sprintf(`{"%v":-%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + }) + }) +} diff --git a/src/mongo/gotools/common/json/new.go b/src/mongo/gotools/common/json/new.go new file mode 100644 index 00000000000..71c9bc4b7ef --- /dev/null +++ b/src/mongo/gotools/common/json/new.go @@ -0,0 +1,92 @@ +package json + +import ( + "fmt" + "reflect" +) + +// Transition functions for recognizing new. +// Adapted from encoding/json/scanner.go. + +// stateNe is the state after reading `ne`. +func stateNe(s *scanner, c int) int { + if c == 'w' { + s.step = stateNew + return scanContinue + } + return s.error(c, "in literal new (expecting 'w')") +} + +// stateNew is the state after reading `new`. +// Ensures that there is a space after the new keyword. +func stateNew(s *scanner, c int) int { + if c <= ' ' && isSpace(rune(c)) { + s.step = stateBeginObjectValue + return scanContinue + } + return s.error(c, "expected space after new keyword") +} + +// stateBeginObjectValue is the state after reading `new`. +func stateBeginObjectValue(s *scanner, c int) int { + if c <= ' ' && isSpace(rune(c)) { + return scanSkipSpace + } + switch c { + case 'B': // beginning of BinData or Boolean + s.step = stateB + case 'D': // beginning of Date + s.step = stateD + case 'N': // beginning of NumberInt or NumberLong + s.step = stateNumberUpperN + case 'O': // beginning of ObjectId + s.step = stateO + case 'R': // beginning of RegExp + s.step = stateR + case 'T': // beginning of Timestamp + s.step = stateUpperT + default: + return s.error(c, "looking for beginning of value") + } + + return scanBeginLiteral +} + +// stateNumberUpperN is the state after reading `N`. +func stateNumberUpperN(s *scanner, c int) int { + if c == 'u' { + s.step = stateUpperNu + return scanContinue + } + return s.error(c, "in literal NumberInt or NumberLong (expecting 'u')") +} + +// Decodes a literal stored in the underlying byte data into v. +func (d *decodeState) storeNewLiteral(v reflect.Value, fromQuoted bool) { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginLiteral { + d.error(fmt.Errorf("expected beginning of constructor")) + } + + // Read constructor identifier + start := d.off - 1 + op = d.scanWhile(scanContinue) + if op != scanBeginCtor { + d.error(fmt.Errorf("expected beginning of constructor")) + } + + // Back up so d.ctor can have the byte we just read. + d.off-- + d.scan.undo(op) + + d.literalStore(d.data[start:d.off-1], v, fromQuoted) +} + +// Returns a literal from the underlying byte data. +func (d *decodeState) getNewLiteral() interface{} { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginLiteral { + d.error(fmt.Errorf("expected beginning of constructor")) + } + return d.literalInterface() +} diff --git a/src/mongo/gotools/common/json/new_test.go b/src/mongo/gotools/common/json/new_test.go new file mode 100644 index 00000000000..3d91ed9dcfb --- /dev/null +++ b/src/mongo/gotools/common/json/new_test.go @@ -0,0 +1,197 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestNewKeyword(t *testing.T) { + + Convey("When unmarshalling JSON using the new keyword", t, func() { + + Convey("can be used with BinData constructor", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `new BinData(1, "xyz")` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(BinData) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, BinData{1, "xyz"}) + }) + + Convey("can be used with Boolean constructor", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `new Boolean(1)` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, true) + + key = "key" + value = `new Boolean(0)` + data = fmt.Sprintf(`{"%v":%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok = jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, false) + }) + + Convey("can be used with Date constructor", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "new Date(123)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(Date) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, Date(123)) + }) + + Convey("can be used with DBRef constructor", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `new BinData(1, "xyz")` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(BinData) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, BinData{1, "xyz"}) + }) + + Convey("can be used with NumberInt constructor", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "new NumberInt(123)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(NumberInt) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, NumberInt(123)) + }) + + Convey("can be used with NumberLong constructor", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "new NumberLong(123)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(NumberLong) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, NumberLong(123)) + }) + + Convey("can be used with ObjectId constructor", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `new ObjectId("123")` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(ObjectId) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, ObjectId("123")) + }) + + Convey("can be used with RegExp constructor", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `new RegExp("foo", "i")` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(RegExp) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, RegExp{"foo", "i"}) + }) + + Convey("can be used with Timestamp constructor", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "new Timestamp(123, 321)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(Timestamp) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, Timestamp{123, 321}) + }) + + Convey("cannot be used with literals", func() { + var jsonMap map[string]interface{} + + key := "key" + literals := []string{"null", "true", "false", "undefined", + "NaN", "Infinity", "MinKey", "MaxKey"} + + for _, value := range literals { + data := fmt.Sprintf(`{"%v":new %v}`, key, value) + Convey(value, func() { + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + }) + } + }) + + Convey("must be followed by a space", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "newDate(123)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + }) + + Convey("cannot be chained togther (`new new ...`)", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "new new Date(123)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + }) + }) +} diff --git a/src/mongo/gotools/common/json/number.go b/src/mongo/gotools/common/json/number.go new file mode 100644 index 00000000000..3222b5e6ef6 --- /dev/null +++ b/src/mongo/gotools/common/json/number.go @@ -0,0 +1,136 @@ +package json + +import ( + "fmt" + "reflect" +) + +// Transition functions for recognizing NumberInt and NumberLong. +// Adapted from encoding/json/scanner.go. + +// stateUpperNu is the state after reading `Nu`. +func stateUpperNu(s *scanner, c int) int { + if c == 'm' { + s.step = generateState("Number", []byte("ber"), stateUpperNumber) + return scanContinue + } + return s.error(c, "in literal Number (expecting 'm')") +} + +// stateUpperNumber is the state after reading `Number`. +func stateUpperNumber(s *scanner, c int) int { + if c == 'I' { + s.step = generateState("NumberInt", []byte("nt"), stateConstructor) + return scanContinue + } + if c == 'L' { + s.step = generateState("NumberLong", []byte("ong"), stateConstructor) + return scanContinue + } + return s.error(c, "in literal NumberInt or NumberLong (expecting 'I' or 'L')") +} + +// Decodes a NumberInt literal stored in the underlying byte data into v. +func (d *decodeState) storeNumberInt(v reflect.Value) { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginCtor { + d.error(fmt.Errorf("expected beginning of constructor")) + } + + args, err := d.ctor("NumberInt", []reflect.Type{numberIntType}) + if err != nil { + d.error(err) + } + switch kind := v.Kind(); kind { + case reflect.Interface: + v.Set(args[0]) + default: + d.error(fmt.Errorf("cannot store %v value into %v type", numberIntType, kind)) + } +} + +// Returns a NumberInt literal from the underlying byte data. +func (d *decodeState) getNumberInt() interface{} { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginCtor { + d.error(fmt.Errorf("expected beginning of constructor")) + } + + // Prevent d.convertNumber() from parsing the argument as a float64. + useNumber := d.useNumber + d.useNumber = true + + args := d.ctorInterface() + if err := ctorNumArgsMismatch("NumberInt", 1, len(args)); err != nil { + d.error(err) + } + var number Number + switch v := args[0].(type) { + case Number: + number = v + case string: + number = Number(v) + default: + d.error(fmt.Errorf("expected int32 for first argument of NumberInt constructor, got %T (value was %v)", v, v)) + } + + d.useNumber = useNumber + arg0, err := number.Int32() + if err != nil { + d.error(fmt.Errorf("expected int32 for first argument of NumberInt constructor, got %T (value was %v)", number, number)) + } + return NumberInt(arg0) +} + +// Decodes a NumberLong literal stored in the underlying byte data into v. +func (d *decodeState) storeNumberLong(v reflect.Value) { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginCtor { + d.error(fmt.Errorf("expected beginning of constructor")) + } + + args, err := d.ctor("NumberLong", []reflect.Type{numberLongType}) + if err != nil { + d.error(err) + } + switch kind := v.Kind(); kind { + case reflect.Interface: + v.Set(args[0]) + default: + d.error(fmt.Errorf("cannot store %v value into %v type", numberLongType, kind)) + } +} + +// Returns a NumberLong literal from the underlying byte data. +func (d *decodeState) getNumberLong() interface{} { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginCtor { + d.error(fmt.Errorf("expected beginning of constructor")) + } + + // Prevent d.convertNumber() from parsing the argument as a float64. + useNumber := d.useNumber + d.useNumber = true + + args := d.ctorInterface() + if err := ctorNumArgsMismatch("NumberLong", 1, len(args)); err != nil { + d.error(err) + } + var number Number + switch v := args[0].(type) { + case Number: + number = v + case string: + number = Number(v) + + default: + d.error(fmt.Errorf("expected int64 for first argument of NumberLong constructor, got %T (value was %v)", v, v)) + } + + d.useNumber = useNumber + arg0, err := number.Int64() + if err != nil { + d.error(fmt.Errorf("expected int64 for first argument of NumberLong constructor, got %T (value was %v)", number, number)) + } + return NumberLong(arg0) +} diff --git a/src/mongo/gotools/common/json/number_test.go b/src/mongo/gotools/common/json/number_test.go new file mode 100644 index 00000000000..88ce8de9a34 --- /dev/null +++ b/src/mongo/gotools/common/json/number_test.go @@ -0,0 +1,191 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestNumberIntValue(t *testing.T) { + + Convey("When unmarshalling JSON with NumberInt values", t, func() { + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "NumberInt(123)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(NumberInt) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, NumberInt(123)) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + value1, value2, value3 := "NumberInt(123)", "NumberInt(456)", "NumberInt(789)" + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value1, key2, value2, key3, value3) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue1, ok := jsonMap[key1].(NumberInt) + So(ok, ShouldBeTrue) + So(jsonValue1, ShouldEqual, NumberInt(123)) + + jsonValue2, ok := jsonMap[key2].(NumberInt) + So(ok, ShouldBeTrue) + So(jsonValue2, ShouldEqual, NumberInt(456)) + + jsonValue3, ok := jsonMap[key3].(NumberInt) + So(ok, ShouldBeTrue) + So(jsonValue3, ShouldEqual, NumberInt(789)) + }) + + Convey("works in an array", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "NumberInt(42)" + data := fmt.Sprintf(`{"%v":[%v,%v,%v]}`, + key, value, value, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonArray, ok := jsonMap[key].([]interface{}) + So(ok, ShouldBeTrue) + + for _, _jsonValue := range jsonArray { + jsonValue, ok := _jsonValue.(NumberInt) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, NumberInt(42)) + } + }) + + Convey("can use string as argument", func() { + key := "key" + value := `NumberInt("123")` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + jsonValue, err := UnmarshalBsonD([]byte(data)) + + So(jsonValue[0].Value, ShouldEqual, NumberInt(123)) + So(err, ShouldBeNil) + }) + + Convey("can specify argument in hexadecimal", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "NumberInt(0x5f)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(NumberInt) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, NumberInt(0x5f)) + }) + }) +} + +func TestNumberLongValue(t *testing.T) { + + Convey("When unmarshalling JSON with NumberLong values", t, func() { + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "NumberLong(123)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(NumberLong) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, NumberLong(123)) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + value1, value2, value3 := "NumberLong(123)", "NumberLong(456)", "NumberLong(789)" + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value1, key2, value2, key3, value3) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue1, ok := jsonMap[key1].(NumberLong) + So(ok, ShouldBeTrue) + So(jsonValue1, ShouldEqual, NumberLong(123)) + + jsonValue2, ok := jsonMap[key2].(NumberLong) + So(ok, ShouldBeTrue) + So(jsonValue2, ShouldEqual, NumberLong(456)) + + jsonValue3, ok := jsonMap[key3].(NumberLong) + So(ok, ShouldBeTrue) + So(jsonValue3, ShouldEqual, NumberLong(789)) + }) + + Convey("works in an array", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "NumberLong(42)" + data := fmt.Sprintf(`{"%v":[%v,%v,%v]}`, + key, value, value, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonArray, ok := jsonMap[key].([]interface{}) + So(ok, ShouldBeTrue) + + for _, _jsonValue := range jsonArray { + jsonValue, ok := _jsonValue.(NumberLong) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, NumberLong(42)) + } + }) + + Convey("can use string as argument", func() { + key := "key" + value := `NumberLong("123")` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + jsonValue, err := UnmarshalBsonD([]byte(data)) + + So(jsonValue[0].Value, ShouldEqual, NumberLong(123)) + So(err, ShouldBeNil) + }) + + Convey("can specify argument in hexadecimal", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "NumberLong(0x5f)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(NumberLong) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, NumberLong(0x5f)) + }) + }) +} diff --git a/src/mongo/gotools/common/json/objectid.go b/src/mongo/gotools/common/json/objectid.go new file mode 100644 index 00000000000..c38f4df5225 --- /dev/null +++ b/src/mongo/gotools/common/json/objectid.go @@ -0,0 +1,55 @@ +package json + +import ( + "fmt" + "reflect" +) + +// Transition functions for recognizing ObjectId. +// Adapted from encoding/json/scanner.go. + +// stateO is the state after reading `O`. +func stateO(s *scanner, c int) int { + if c == 'b' { + s.step = generateState("ObjectId", []byte("jectId"), stateConstructor) + return scanContinue + } + return s.error(c, "in literal ObjectId (expecting 'b')") +} + +// Decodes an ObjectId literal stored in the underlying byte data into v. +func (d *decodeState) storeObjectId(v reflect.Value) { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginCtor { + d.error(fmt.Errorf("expected beginning of constructor")) + } + + args, err := d.ctor("ObjectId", []reflect.Type{objectIdType}) + if err != nil { + d.error(err) + } + switch kind := v.Kind(); kind { + case reflect.Interface: + v.Set(args[0]) + default: + d.error(fmt.Errorf("cannot store %v value into %v type", objectIdType, kind)) + } +} + +// Returns an ObjectId literal from the underlying byte data. +func (d *decodeState) getObjectId() interface{} { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginCtor { + d.error(fmt.Errorf("expected beginning of constructor")) + } + + args := d.ctorInterface() + if err := ctorNumArgsMismatch("ObjectId", 1, len(args)); err != nil { + d.error(err) + } + arg0, ok := args[0].(string) + if !ok { + d.error(fmt.Errorf("expected string for first argument of ObjectId constructor")) + } + return ObjectId(arg0) +} diff --git a/src/mongo/gotools/common/json/objectid_test.go b/src/mongo/gotools/common/json/objectid_test.go new file mode 100644 index 00000000000..2bb25ba7d7c --- /dev/null +++ b/src/mongo/gotools/common/json/objectid_test.go @@ -0,0 +1,84 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestObjectIdValue(t *testing.T) { + + Convey("When unmarshalling JSON with ObjectId values", t, func() { + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `ObjectId("123")` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(ObjectId) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, ObjectId("123")) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + value1, value2, value3 := `ObjectId("123")`, `ObjectId("456")`, `ObjectId("789")` + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value1, key2, value2, key3, value3) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue1, ok := jsonMap[key1].(ObjectId) + So(ok, ShouldBeTrue) + So(jsonValue1, ShouldEqual, ObjectId("123")) + + jsonValue2, ok := jsonMap[key2].(ObjectId) + So(ok, ShouldBeTrue) + So(jsonValue2, ShouldEqual, ObjectId("456")) + + jsonValue3, ok := jsonMap[key3].(ObjectId) + So(ok, ShouldBeTrue) + So(jsonValue3, ShouldEqual, ObjectId("789")) + }) + + Convey("works in an array", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `ObjectId("000")` + data := fmt.Sprintf(`{"%v":[%v,%v,%v]}`, + key, value, value, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonArray, ok := jsonMap[key].([]interface{}) + So(ok, ShouldBeTrue) + + for _, _jsonValue := range jsonArray { + jsonValue, ok := _jsonValue.(ObjectId) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, ObjectId("000")) + } + }) + + Convey("cannot use number as argument", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `ObjectId(123)` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + }) + }) +} diff --git a/src/mongo/gotools/common/json/regexp.go b/src/mongo/gotools/common/json/regexp.go new file mode 100644 index 00000000000..a9d06fe1d48 --- /dev/null +++ b/src/mongo/gotools/common/json/regexp.go @@ -0,0 +1,275 @@ +package json + +import ( + "fmt" + "reflect" + "unicode" + "unicode/utf16" + "unicode/utf8" +) + +// Transition functions for recognizing RegExp. +// Adapted from encoding/json/scanner.go. + +// stateR is the state after reading `R`. +func stateR(s *scanner, c int) int { + if c == 'e' { + s.step = generateState("RegExp", []byte("gExp"), stateConstructor) + return scanContinue + } + return s.error(c, "in literal RegExp (expecting 'e')") +} + +// stateInRegexpPattern is the state after reading `/`. +func stateInRegexpPattern(s *scanner, c int) int { + if c == '/' { + s.step = stateInRegexpOptions + return scanRegexpOptions + } + if c == '\\' { + s.step = stateInRegexpPatternEsc + return scanRegexpPattern + } + if c < 0x20 { + return s.error(c, "in regular expression literal") + } + return scanRegexpPattern +} + +// stateInRegexpPatternEsc is the state after reading `'\` during a regex pattern. +func stateInRegexpPatternEsc(s *scanner, c int) int { + switch c { + case 'b', 'f', 'n', 'r', 't', '\\', '/', '\'': + s.step = stateInRegexpPattern + return scanRegexpPattern + } + if c == 'u' { + s.step = stateInRegexpPatternEscU + return scanRegexpPattern + } + return s.error(c, "in string escape code") +} + +// stateInRegexpPatternEscU is the state after reading `'\u` during a regex pattern. +func stateInRegexpPatternEscU(s *scanner, c int) int { + if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { + s.step = stateInRegexpPatternEscU1 + return scanRegexpPattern + } + // numbers + return s.error(c, "in \\u hexadecimal character escape") +} + +// stateInRegexpPatternEscU1 is the state after reading `'\u1` during a regex pattern. +func stateInRegexpPatternEscU1(s *scanner, c int) int { + if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { + s.step = stateInRegexpPatternEscU12 + return scanRegexpPattern + } + // numbers + return s.error(c, "in \\u hexadecimal character escape") +} + +// stateInRegexpPatternEscU12 is the state after reading `'\u12` during a regex pattern. +func stateInRegexpPatternEscU12(s *scanner, c int) int { + if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { + s.step = stateInRegexpPatternEscU123 + return scanRegexpPattern + } + // numbers + return s.error(c, "in \\u hexadecimal character escape") +} + +// stateInRegexpPatternEscU123 is the state after reading `'\u123` during a regex pattern. +func stateInRegexpPatternEscU123(s *scanner, c int) int { + if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { + s.step = stateInRegexpPattern + return scanRegexpPattern + } + // numbers + return s.error(c, "in \\u hexadecimal character escape") +} + +// stateInRegexpOptions is the state after reading `/foo/`. +func stateInRegexpOptions(s *scanner, c int) int { + switch c { + case 'g', 'i', 'm', 's': + return scanRegexpOptions + } + return stateEndValue(s, c) +} + +// Decodes a RegExp literal stored in the underlying byte data into v. +func (d *decodeState) storeRegexp(v reflect.Value) { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginCtor { + d.error(fmt.Errorf("expected beginning of constructor")) + } + + args, err := d.ctor("RegExp", []reflect.Type{stringType, stringType}) + if err != nil { + d.error(err) + } + switch kind := v.Kind(); kind { + case reflect.Interface: + arg0 := args[0].String() + arg1 := args[1].String() + v.Set(reflect.ValueOf(RegExp{arg0, arg1})) + default: + d.error(fmt.Errorf("cannot store %v value into %v type", regexpType, kind)) + } +} + +// Returns a RegExp literal from the underlying byte data. +func (d *decodeState) getRegexp() interface{} { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginCtor { + d.error(fmt.Errorf("expected beginning of constructor")) + } + + args := d.ctorInterface() + if err := ctorNumArgsMismatch("RegExp", 2, len(args)); err != nil { + d.error(err) + } + arg0, ok := args[0].(string) + if !ok { + d.error(fmt.Errorf("expected string for first argument of RegExp constructor")) + } + arg1, ok := args[1].(string) + if !ok { + d.error(fmt.Errorf("expected string for second argument of RegExp constructor")) + } + return RegExp{arg0, arg1} +} + +// Decoder function that breaks a regular expression literal into its pattern and options. +// Adapted from encoding/json/decode.go. + +// regexp consumes a regular expression from d.data[d.off-1:]. +// the two bytes of the regexp ("/a") have been read already. +func (d *decodeState) regexp() (string, string, error) { + start := d.off - 1 + + // Look ahead for /. + op := d.scanWhile(scanRegexpPattern) + if op != scanRegexpOptions { + return "", "", fmt.Errorf("expected beginning of regular expression options") + } + pattern := d.data[start : d.off-1] + + start = d.off + op = d.scanWhile(scanRegexpOptions) + + // Back up so caller can have the byte we just read. + d.off-- + d.scan.undo(op) + + options := d.data[start:d.off] + + // Check for unusual characters. If there are none, + // then no copying is needed, so return string of the + // original bytes. + r := 0 + for r < len(pattern) { + c := pattern[r] + if c == '\\' || c == '/' || c < ' ' { + break + } + if c < utf8.RuneSelf { + r++ + continue + } + rr, size := utf8.DecodeRune(pattern[r:]) + if rr == utf8.RuneError && size == 1 { + break + } + r += size + } + if r == len(pattern) { + return string(pattern), string(options), nil + } + + b := make([]byte, len(pattern)+2*utf8.UTFMax) + w := copy(b, pattern[0:r]) + for r < len(pattern) { + // Out of room? Can only happen if pattern is full of + // malformed UTF-8 and we're replacing each + // byte with RuneError. + if w >= len(b)-2*utf8.UTFMax { + nb := make([]byte, (len(b)+utf8.UTFMax)*2) + copy(nb, b[0:w]) + b = nb + } + switch c := pattern[r]; { + case c == '\\': + r++ + if r >= len(pattern) { + return "", "", errPhase + } + switch pattern[r] { + default: + return "", "", fmt.Errorf("invalid escape character") + case '"', '\\', '/', '\'': + b[w] = pattern[r] + r++ + w++ + case 'b': + b[w] = '\b' + r++ + w++ + case 'f': + b[w] = '\f' + r++ + w++ + case 'n': + b[w] = '\n' + r++ + w++ + case 'r': + b[w] = '\r' + r++ + w++ + case 't': + b[w] = '\t' + r++ + w++ + case 'u': + r-- + rr := getu4(pattern[r:]) + if rr < 0 { + return "", "", fmt.Errorf("non-hexadecimal character found") + } + r += 6 + if utf16.IsSurrogate(rr) { + rr1 := getu4(pattern[r:]) + if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar { + // A valid pair; consume. + r += 6 + w += utf8.EncodeRune(b[w:], dec) + break + } + // Invalid surrogate; fall back to replacement rune. + rr = unicode.ReplacementChar + } + w += utf8.EncodeRune(b[w:], rr) + } + + // Forward slash, control characters are invalid. + case c == '/', c < ' ': + d.error(fmt.Errorf("regular expression pattern cannot contain unescaped '/'")) + + // ASCII + case c < utf8.RuneSelf: + b[w] = c + r++ + w++ + + // Coerce to well-formed UTF-8. + default: + rr, size := utf8.DecodeRune(pattern[r:]) + r += size + w += utf8.EncodeRune(b[w:], rr) + } + } + return string(b[0:w]), string(options), nil +} diff --git a/src/mongo/gotools/common/json/regexp_test.go b/src/mongo/gotools/common/json/regexp_test.go new file mode 100644 index 00000000000..a1f611b6c05 --- /dev/null +++ b/src/mongo/gotools/common/json/regexp_test.go @@ -0,0 +1,243 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestRegExpValue(t *testing.T) { + + Convey("When unmarshalling JSON with RegExp values", t, func() { + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `RegExp("foo", "i")` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(RegExp) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, RegExp{"foo", "i"}) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + value1, value2, value3 := `RegExp("foo", "i")`, + `RegExp("bar", "i")`, `RegExp("baz", "i")` + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value1, key2, value2, key3, value3) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue1, ok := jsonMap[key1].(RegExp) + So(ok, ShouldBeTrue) + So(jsonValue1, ShouldResemble, RegExp{"foo", "i"}) + + jsonValue2, ok := jsonMap[key2].(RegExp) + So(ok, ShouldBeTrue) + So(jsonValue2, ShouldResemble, RegExp{"bar", "i"}) + + jsonValue3, ok := jsonMap[key3].(RegExp) + So(ok, ShouldBeTrue) + So(jsonValue3, ShouldResemble, RegExp{"baz", "i"}) + }) + + Convey("works in an array", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `RegExp("xyz", "i")` + data := fmt.Sprintf(`{"%v":[%v,%v,%v]}`, + key, value, value, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonArray, ok := jsonMap[key].([]interface{}) + So(ok, ShouldBeTrue) + + for _, _jsonValue := range jsonArray { + jsonValue, ok := _jsonValue.(RegExp) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, RegExp{"xyz", "i"}) + } + }) + + Convey("can use options 'g', 'i', 'm', and 's'", func() { + var jsonMap map[string]interface{} + + key := "key" + options := []string{"g", "i", "m", "s"} + + for _, option := range options { + data := fmt.Sprintf(`{"%v":RegExp("xyz", "%v")}`, key, option) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(RegExp) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, RegExp{"xyz", option}) + } + }) + + Convey("can use multiple options", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `RegExp("foo", "gims")` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(RegExp) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, RegExp{"foo", "gims"}) + }) + }) +} + +func TestRegexpLiteral(t *testing.T) { + + Convey("When unmarshalling JSON with regular expression literals", t, func() { + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "/foo/i" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(RegExp) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, RegExp{"foo", "i"}) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + value1, value2, value3 := "/foo/i", "/bar/i", "/baz/i" + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value1, key2, value2, key3, value3) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue1, ok := jsonMap[key1].(RegExp) + So(ok, ShouldBeTrue) + So(jsonValue1, ShouldResemble, RegExp{"foo", "i"}) + + jsonValue2, ok := jsonMap[key2].(RegExp) + So(ok, ShouldBeTrue) + So(jsonValue2, ShouldResemble, RegExp{"bar", "i"}) + + jsonValue3, ok := jsonMap[key3].(RegExp) + So(ok, ShouldBeTrue) + So(jsonValue3, ShouldResemble, RegExp{"baz", "i"}) + }) + + Convey("works in an array", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "/xyz/i" + data := fmt.Sprintf(`{"%v":[%v,%v,%v]}`, + key, value, value, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonArray, ok := jsonMap[key].([]interface{}) + So(ok, ShouldBeTrue) + + for _, _jsonValue := range jsonArray { + jsonValue, ok := _jsonValue.(RegExp) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, RegExp{"xyz", "i"}) + } + }) + + Convey("can use options 'g', 'i', 'm', and 's'", func() { + var jsonMap map[string]interface{} + + key := "key" + options := []string{"g", "i", "m", "s"} + + for _, option := range options { + data := fmt.Sprintf(`{"%v":/xyz/%v}`, key, option) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(RegExp) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, RegExp{"xyz", option}) + } + }) + + Convey("can use multiple options", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "/foo/gims" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(RegExp) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, RegExp{"foo", "gims"}) + }) + + Convey("can contain unescaped quotes (`'` and `\"`)", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `/f'o"o/i` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(RegExp) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, RegExp{`f'o"o`, "i"}) + }) + + Convey("cannot contain unescaped forward slashes ('/')", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "/f/o/o/i" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + }) + + Convey("cannot contain invalid escape sequences", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `/f\o\o/` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + }) + }) +} diff --git a/src/mongo/gotools/common/json/scanner.go b/src/mongo/gotools/common/json/scanner.go new file mode 100644 index 00000000000..4317432d761 --- /dev/null +++ b/src/mongo/gotools/common/json/scanner.go @@ -0,0 +1,669 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +// JSON value parser state machine. +// Just about at the limit of what is reasonable to write by hand. +// Some parts are a bit tedious, but overall it nicely factors out the +// otherwise common code from the multiple scanning functions +// in this package (Compact, Indent, checkValid, nextValue, etc). +// +// This file starts with two simple examples using the scanner +// before diving into the scanner itself. + +import "strconv" + +// checkValid verifies that data is valid JSON-encoded data. +// scan is passed in for use by checkValid to avoid an allocation. +func checkValid(data []byte, scan *scanner) error { + scan.reset() + for _, c := range data { + scan.bytes++ + if scan.step(scan, int(c)) == scanError { + return scan.err + } + } + if scan.eof() == scanError { + return scan.err + } + return nil +} + +// nextValue splits data after the next whole JSON value, +// returning that value and the bytes that follow it as separate slices. +// scan is passed in for use by nextValue to avoid an allocation. +func nextValue(data []byte, scan *scanner) (value, rest []byte, err error) { + scan.reset() + for i, c := range data { + v := scan.step(scan, int(c)) + if v >= scanEnd { + switch v { + case scanError: + return nil, nil, scan.err + case scanEnd: + return data[0:i], data[i:], nil + } + } + } + if scan.eof() == scanError { + return nil, nil, scan.err + } + return data, nil, nil +} + +// A SyntaxError is a description of a JSON syntax error. +type SyntaxError struct { + msg string // description of error + Offset int64 // error occurred after reading Offset bytes +} + +func (e *SyntaxError) Error() string { return e.msg } + +// A scanner is a JSON scanning state machine. +// Callers call scan.reset() and then pass bytes in one at a time +// by calling scan.step(&scan, c) for each byte. +// The return value, referred to as an opcode, tells the +// caller about significant parsing events like beginning +// and ending literals, objects, and arrays, so that the +// caller can follow along if it wishes. +// The return value scanEnd indicates that a single top-level +// JSON value has been completed, *before* the byte that +// just got passed in. (The indication must be delayed in order +// to recognize the end of numbers: is 123 a whole value or +// the beginning of 12345e+6?). +type scanner struct { + // The step is a func to be called to execute the next transition. + // Also tried using an integer constant and a single func + // with a switch, but using the func directly was 10% faster + // on a 64-bit Mac Mini, and it's nicer to read. + step func(*scanner, int) int + + // Reached end of top-level value. + endTop bool + + // Stack of what we're in the middle of - array values, object keys, object values. + parseState []int + + // Error that happened, if any. + err error + + // 1-byte redo (see undo method) + redo bool + redoCode int + redoState func(*scanner, int) int + + // total bytes consumed, updated by decoder.Decode + bytes int64 +} + +// These values are returned by the state transition functions +// assigned to scanner.state and the method scanner.eof. +// They give details about the current state of the scan that +// callers might be interested to know about. +// It is okay to ignore the return value of any particular +// call to scanner.state: if one call returns scanError, +// every subsequent call will return scanError too. +const ( + // Continue. + scanContinue = iota // uninteresting byte + scanBeginLiteral // end implied by next result != scanContinue + scanBeginObject // begin object + scanObjectKey // just finished object key (string) + scanObjectValue // just finished non-last object value + scanEndObject // end object (implies scanObjectValue if possible) + scanBeginArray // begin array + scanArrayValue // just finished array value + scanEndArray // end array (implies scanArrayValue if possible) + scanBeginCtor // begin constructor + scanCtorArg // just finished constructor argument + scanEndCtor // end constructor (implies scanCtorArg if possible) + scanRegexpPattern // inside regular expression pattern + scanRegexpOptions // inside regular expression options + scanSkipSpace // space byte; can skip; known to be last "continue" result + + // Stop. + scanEnd // top-level value ended *before* this byte; known to be first "stop" result + scanError // hit an error, scanner.err. +) + +// These values are stored in the parseState stack. +// They give the current state of a composite value +// being scanned. If the parser is inside a nested value +// the parseState describes the nested state, outermost at entry 0. +const ( + parseObjectKey = iota // parsing object key (before colon) + parseObjectValue // parsing object value (after colon) + parseArrayValue // parsing array value + parseCtorArg // parsing constructor argument +) + +// reset prepares the scanner for use. +// It must be called before calling s.step. +func (s *scanner) reset() { + s.step = stateBeginValue + s.parseState = s.parseState[0:0] + s.err = nil + s.redo = false + s.endTop = false +} + +// eof tells the scanner that the end of input has been reached. +// It returns a scan status just as s.step does. +func (s *scanner) eof() int { + if s.err != nil { + return scanError + } + if s.endTop { + return scanEnd + } + s.step(s, ' ') + if s.endTop { + return scanEnd + } + if s.err == nil { + s.err = &SyntaxError{"unexpected end of JSON input", s.bytes} + } + return scanError +} + +// pushParseState pushes a new parse state p onto the parse stack. +func (s *scanner) pushParseState(p int) { + s.parseState = append(s.parseState, p) +} + +// popParseState pops a parse state (already obtained) off the stack +// and updates s.step accordingly. +func (s *scanner) popParseState() { + n := len(s.parseState) - 1 + s.parseState = s.parseState[0:n] + s.redo = false + if n == 0 { + s.step = stateEndTop + s.endTop = true + } else { + s.step = stateEndValue + } +} + +func isSpace(c rune) bool { + return c == ' ' || c == '\t' || c == '\r' || c == '\n' +} + +// stateBeginValueOrEmpty is the state after reading `[`. +func stateBeginValueOrEmpty(s *scanner, c int) int { + if c <= ' ' && isSpace(rune(c)) { + return scanSkipSpace + } + if c == ']' { + return stateEndValue(s, c) + } + return stateBeginValue(s, c) +} + +// stateBeginValue is the state at the beginning of the input. +func stateBeginValue(s *scanner, c int) int { + if c <= ' ' && isSpace(rune(c)) { + return scanSkipSpace + } + switch c { + case '{': + s.step = stateBeginStringOrEmpty + s.pushParseState(parseObjectKey) + return scanBeginObject + case '[': + s.step = stateBeginValueOrEmpty + s.pushParseState(parseArrayValue) + return scanBeginArray + case '"': + s.step = stateInString + return scanBeginLiteral + case '\'': + s.step = stateInSingleQuotedString + return scanBeginLiteral + case '+', '-': + s.step = stateSign + return scanBeginLiteral + case '0': // beginning of 0.123 + s.step = state0 + return scanBeginLiteral + case '.': // beginning of .123 + s.step = stateDot + return scanBeginLiteral + case 't': // beginning of true + s.step = stateT + return scanBeginLiteral + case 'f': // beginning of false + s.step = stateF + return scanBeginLiteral + case 'n': // beginning of null + s.step = stateN + return scanBeginLiteral + } + if '1' <= c && c <= '9' { // beginning of 1234.5 + s.step = state1 + return scanBeginLiteral + } + return stateBeginExtendedValue(s, c) +} + +// stateBeginStringOrEmpty is the state after reading `{`. +func stateBeginStringOrEmpty(s *scanner, c int) int { + if c <= ' ' && isSpace(rune(c)) { + return scanSkipSpace + } + if c == '}' { + n := len(s.parseState) + s.parseState[n-1] = parseObjectValue + return stateEndValue(s, c) + } + return stateBeginString(s, c) +} + +// stateBeginString is the state after reading `{"key": value,`. +func stateBeginString(s *scanner, c int) int { + if c <= ' ' && isSpace(rune(c)) { + return scanSkipSpace + } + if c == '"' { + s.step = stateInString + return scanBeginLiteral + } + if c == '\'' { + s.step = stateInSingleQuotedString + return scanBeginLiteral + } + if isBeginUnquotedString(c) { + s.step = stateInUnquotedString + return scanBeginLiteral + } + return s.error(c, "looking for beginning of object key string") +} + +// stateEndValue is the state after completing a value, +// such as after reading `{}` or `true` or `["x"`. +func stateEndValue(s *scanner, c int) int { + n := len(s.parseState) + if n == 0 { + // Completed top-level before the current byte. + s.step = stateEndTop + s.endTop = true + return stateEndTop(s, c) + } + if c <= ' ' && isSpace(rune(c)) { + s.step = stateEndValue + return scanSkipSpace + } + ps := s.parseState[n-1] + switch ps { + case parseObjectKey: + if c == ':' { + s.parseState[n-1] = parseObjectValue + s.step = stateBeginValue + return scanObjectKey + } + return s.error(c, "after object key") + case parseObjectValue: + if c == ',' { + s.parseState[n-1] = parseObjectKey + s.step = stateBeginString + return scanObjectValue + } + if c == '}' { + s.popParseState() + return scanEndObject + } + return s.error(c, "after object key:value pair") + case parseArrayValue: + if c == ',' { + s.step = stateBeginValue + return scanArrayValue + } + if c == ']' { + s.popParseState() + return scanEndArray + } + return s.error(c, "after array element") + case parseCtorArg: + if c == ',' { + s.step = stateBeginValue + return scanCtorArg + } + if c == ')' { + s.popParseState() + return scanEndCtor + } + return s.error(c, "after constructor argument") + } + return s.error(c, "") +} + +// stateEndTop is the state after finishing the top-level value, +// such as after reading `{}` or `[1,2,3]`. +// Only space characters should be seen now. +func stateEndTop(s *scanner, c int) int { + if c != ' ' && c != '\t' && c != '\r' && c != '\n' { + // Complain about non-space byte on next call. + s.error(c, "after top-level value") + } + return scanEnd +} + +// stateInString is the state after reading `"`. +func stateInString(s *scanner, c int) int { + if c == '"' { + s.step = stateEndValue + return scanContinue + } + if c == '\\' { + s.step = stateInStringEsc + return scanContinue + } + if c < 0x20 { + return s.error(c, "in string literal") + } + return scanContinue +} + +// stateInStringEsc is the state after reading `"\` during a quoted string. +func stateInStringEsc(s *scanner, c int) int { + switch c { + case 'b', 'f', 'n', 'r', 't', '\\', '/', '"': + s.step = stateInString + return scanContinue + } + if c == 'u' { + s.step = stateInStringEscU + return scanContinue + } + return s.error(c, "in string escape code") +} + +// stateInStringEscU is the state after reading `"\u` during a quoted string. +func stateInStringEscU(s *scanner, c int) int { + if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { + s.step = stateInStringEscU1 + return scanContinue + } + // numbers + return s.error(c, "in \\u hexadecimal character escape") +} + +// stateInStringEscU1 is the state after reading `"\u1` during a quoted string. +func stateInStringEscU1(s *scanner, c int) int { + if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { + s.step = stateInStringEscU12 + return scanContinue + } + // numbers + return s.error(c, "in \\u hexadecimal character escape") +} + +// stateInStringEscU12 is the state after reading `"\u12` during a quoted string. +func stateInStringEscU12(s *scanner, c int) int { + if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { + s.step = stateInStringEscU123 + return scanContinue + } + // numbers + return s.error(c, "in \\u hexadecimal character escape") +} + +// stateInStringEscU123 is the state after reading `"\u123` during a quoted string. +func stateInStringEscU123(s *scanner, c int) int { + if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { + s.step = stateInString + return scanContinue + } + // numbers + return s.error(c, "in \\u hexadecimal character escape") +} + +// stateSign is the state after reading `+` or `-` during a number. +func stateSign(s *scanner, c int) int { + if c == '0' { + s.step = state0 + return scanContinue + } + if c == '.' { + s.step = stateDot + return scanContinue + } + if '1' <= c && c <= '9' { + s.step = state1 + return scanContinue + } + if c == 'I' { + s.step = stateI + return scanContinue + } + return s.error(c, "in numeric literal") +} + +// state1 is the state after reading a non-zero integer during a number, +// such as after reading `1` or `100` but not `0`. +func state1(s *scanner, c int) int { + if '0' <= c && c <= '9' { + s.step = state1 + return scanContinue + } + return state0(s, c) +} + +// state0 is the state after reading `0` during a number. +func state0(s *scanner, c int) int { + if c == '.' { + s.step = stateDot + return scanContinue + } + if c == 'e' || c == 'E' { + s.step = stateE + return scanContinue + } + if c == 'x' || c == 'X' { + s.step = stateHex + return scanContinue + } + return stateEndValue(s, c) +} + +// stateDot is the state after reading the integer and decimal point in a number, +// such as after reading `1.`. +func stateDot(s *scanner, c int) int { + if '0' <= c && c <= '9' { + s.step = stateDot0 + return scanContinue + } + return s.error(c, "after decimal point in numeric literal") +} + +// stateDot0 is the state after reading the integer, decimal point, and subsequent +// digits of a number, such as after reading `3.14`. +func stateDot0(s *scanner, c int) int { + if '0' <= c && c <= '9' { + s.step = stateDot0 + return scanContinue + } + if c == 'e' || c == 'E' { + s.step = stateE + return scanContinue + } + return stateEndValue(s, c) +} + +// stateE is the state after reading the mantissa and e in a number, +// such as after reading `314e` or `0.314e`. +func stateE(s *scanner, c int) int { + if c == '+' { + s.step = stateESign + return scanContinue + } + if c == '-' { + s.step = stateESign + return scanContinue + } + return stateESign(s, c) +} + +// stateESign is the state after reading the mantissa, e, and sign in a number, +// such as after reading `314e-` or `0.314e+`. +func stateESign(s *scanner, c int) int { + if '0' <= c && c <= '9' { + s.step = stateE0 + return scanContinue + } + return s.error(c, "in exponent of numeric literal") +} + +// stateE0 is the state after reading the mantissa, e, optional sign, +// and at least one digit of the exponent in a number, +// such as after reading `314e-2` or `0.314e+1` or `3.14e0`. +func stateE0(s *scanner, c int) int { + if '0' <= c && c <= '9' { + s.step = stateE0 + return scanContinue + } + return stateEndValue(s, c) +} + +// stateT is the state after reading `t`. +func stateT(s *scanner, c int) int { + if c == 'r' { + s.step = stateTr + return scanContinue + } + return s.error(c, "in literal true (expecting 'r')") +} + +// stateTr is the state after reading `tr`. +func stateTr(s *scanner, c int) int { + if c == 'u' { + s.step = stateTru + return scanContinue + } + return s.error(c, "in literal true (expecting 'u')") +} + +// stateTru is the state after reading `tru`. +func stateTru(s *scanner, c int) int { + if c == 'e' { + s.step = stateEndValue + return scanContinue + } + return s.error(c, "in literal true (expecting 'e')") +} + +// stateF is the state after reading `f`. +func stateF(s *scanner, c int) int { + if c == 'a' { + s.step = stateFa + return scanContinue + } + return s.error(c, "in literal false (expecting 'a')") +} + +// stateFa is the state after reading `fa`. +func stateFa(s *scanner, c int) int { + if c == 'l' { + s.step = stateFal + return scanContinue + } + return s.error(c, "in literal false (expecting 'l')") +} + +// stateFal is the state after reading `fal`. +func stateFal(s *scanner, c int) int { + if c == 's' { + s.step = stateFals + return scanContinue + } + return s.error(c, "in literal false (expecting 's')") +} + +// stateFals is the state after reading `fals`. +func stateFals(s *scanner, c int) int { + if c == 'e' { + s.step = stateEndValue + return scanContinue + } + return s.error(c, "in literal false (expecting 'e')") +} + +// stateN is the state after reading `n`. +func stateN(s *scanner, c int) int { + if c == 'e' { + s.step = stateNe + return scanContinue + } + if c == 'u' { + s.step = stateNu + return scanContinue + } + return s.error(c, "in literal new or null (expecting 'e' or 'u')") +} + +// stateNu is the state after reading `nu`. +func stateNu(s *scanner, c int) int { + if c == 'l' { + s.step = stateNul + return scanContinue + } + return s.error(c, "in literal null (expecting 'l')") +} + +// stateNul is the state after reading `nul`. +func stateNul(s *scanner, c int) int { + if c == 'l' { + s.step = stateEndValue + return scanContinue + } + return s.error(c, "in literal null (expecting 'l')") +} + +// stateError is the state after reaching a syntax error, +// such as after reading `[1}` or `5.1.2`. +func stateError(s *scanner, c int) int { + return scanError +} + +// error records an error and switches to the error state. +func (s *scanner) error(c int, context string) int { + s.step = stateError + s.err = &SyntaxError{"invalid character " + quoteChar(c) + " " + context, s.bytes} + return scanError +} + +// quoteChar formats c as a quoted character literal +func quoteChar(c int) string { + // special cases - different from quoted strings + if c == '\'' { + return `'\''` + } + if c == '"' { + return `'"'` + } + + // use quoted string with different quotation marks + s := strconv.Quote(string(c)) + return "'" + s[1:len(s)-1] + "'" +} + +// undo causes the scanner to return scanCode from the next state transition. +// This gives callers a simple 1-byte undo mechanism. +func (s *scanner) undo(scanCode int) { + if s.redo { + panic("json: invalid use of scanner") + } + s.redoCode = scanCode + s.redoState = s.step + s.step = stateRedo + s.redo = true +} + +// stateRedo helps implement the scanner's 1-byte undo. +func stateRedo(s *scanner, c int) int { + s.redo = false + s.step = s.redoState + return s.redoCode +} diff --git a/src/mongo/gotools/common/json/scanner_test.go b/src/mongo/gotools/common/json/scanner_test.go new file mode 100644 index 00000000000..93dc25a5f1b --- /dev/null +++ b/src/mongo/gotools/common/json/scanner_test.go @@ -0,0 +1,315 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import ( + "bytes" + "math" + "math/rand" + "reflect" + "testing" +) + +// Tests of simple examples. + +type example struct { + compact string + indent string +} + +var examples = []example{ + {`1`, `1`}, + {`{}`, `{}`}, + {`[]`, `[]`}, + {`{"":2}`, "{\n\t\"\": 2\n}"}, + {`[3]`, "[\n\t3\n]"}, + {`[1,2,3]`, "[\n\t1,\n\t2,\n\t3\n]"}, + {`{"x":1}`, "{\n\t\"x\": 1\n}"}, + {ex1, ex1i}, +} + +var ex1 = `[true,false,null,"x",1,1.5,0,-5e+2]` + +var ex1i = `[ + true, + false, + null, + "x", + 1, + 1.5, + 0, + -5e+2 +]` + +func TestCompact(t *testing.T) { + var buf bytes.Buffer + for _, tt := range examples { + buf.Reset() + if err := Compact(&buf, []byte(tt.compact)); err != nil { + t.Errorf("Compact(%#q): %v", tt.compact, err) + } else if s := buf.String(); s != tt.compact { + t.Errorf("Compact(%#q) = %#q, want original", tt.compact, s) + } + + buf.Reset() + if err := Compact(&buf, []byte(tt.indent)); err != nil { + t.Errorf("Compact(%#q): %v", tt.indent, err) + continue + } else if s := buf.String(); s != tt.compact { + t.Errorf("Compact(%#q) = %#q, want %#q", tt.indent, s, tt.compact) + } + } +} + +func TestCompactSeparators(t *testing.T) { + // U+2028 and U+2029 should be escaped inside strings. + // They should not appear outside strings. + tests := []struct { + in, compact string + }{ + {"{\"\u2028\": 1}", `{"\u2028":1}`}, + {"{\"\u2029\" :2}", `{"\u2029":2}`}, + } + for _, tt := range tests { + var buf bytes.Buffer + if err := Compact(&buf, []byte(tt.in)); err != nil { + t.Errorf("Compact(%q): %v", tt.in, err) + } else if s := buf.String(); s != tt.compact { + t.Errorf("Compact(%q) = %q, want %q", tt.in, s, tt.compact) + } + } +} + +func TestIndent(t *testing.T) { + var buf bytes.Buffer + for _, tt := range examples { + buf.Reset() + if err := Indent(&buf, []byte(tt.indent), "", "\t"); err != nil { + t.Errorf("Indent(%#q): %v", tt.indent, err) + } else if s := buf.String(); s != tt.indent { + t.Errorf("Indent(%#q) = %#q, want original", tt.indent, s) + } + + buf.Reset() + if err := Indent(&buf, []byte(tt.compact), "", "\t"); err != nil { + t.Errorf("Indent(%#q): %v", tt.compact, err) + continue + } else if s := buf.String(); s != tt.indent { + t.Errorf("Indent(%#q) = %#q, want %#q", tt.compact, s, tt.indent) + } + } +} + +// Tests of a large random structure. + +func TestCompactBig(t *testing.T) { + initBig() + var buf bytes.Buffer + if err := Compact(&buf, jsonBig); err != nil { + t.Fatalf("Compact: %v", err) + } + b := buf.Bytes() + if !bytes.Equal(b, jsonBig) { + t.Error("Compact(jsonBig) != jsonBig") + diff(t, b, jsonBig) + return + } +} + +func TestIndentBig(t *testing.T) { + initBig() + var buf bytes.Buffer + if err := Indent(&buf, jsonBig, "", "\t"); err != nil { + t.Fatalf("Indent1: %v", err) + } + b := buf.Bytes() + if len(b) == len(jsonBig) { + // jsonBig is compact (no unnecessary spaces); + // indenting should make it bigger + t.Fatalf("Indent(jsonBig) did not get bigger") + } + + // should be idempotent + var buf1 bytes.Buffer + if err := Indent(&buf1, b, "", "\t"); err != nil { + t.Fatalf("Indent2: %v", err) + } + b1 := buf1.Bytes() + if !bytes.Equal(b1, b) { + t.Error("Indent(Indent(jsonBig)) != Indent(jsonBig)") + diff(t, b1, b) + return + } + + // should get back to original + buf1.Reset() + if err := Compact(&buf1, b); err != nil { + t.Fatalf("Compact: %v", err) + } + b1 = buf1.Bytes() + if !bytes.Equal(b1, jsonBig) { + t.Error("Compact(Indent(jsonBig)) != jsonBig") + diff(t, b1, jsonBig) + return + } +} + +type indentErrorTest struct { + in string + err error +} + +var indentErrorTests = []indentErrorTest{ + {`{"X": "foo", "Y"}`, &SyntaxError{"invalid character '}' after object key", 17}}, + {`{"X": "foo" "Y": "bar"}`, &SyntaxError{"invalid character '\"' after object key:value pair", 13}}, +} + +func TestIndentErrors(t *testing.T) { + for i, tt := range indentErrorTests { + var slice []uint8 + buf := bytes.NewBuffer(slice) + if err := Indent(buf, []uint8(tt.in), "", ""); err != nil { + if !reflect.DeepEqual(err, tt.err) { + t.Errorf("#%d: Indent: %#v", i, err) + continue + } + } + } +} + +func TestNextValueBig(t *testing.T) { + initBig() + var scan scanner + item, rest, err := nextValue(jsonBig, &scan) + if err != nil { + t.Fatalf("nextValue: %s", err) + } + if len(item) != len(jsonBig) || &item[0] != &jsonBig[0] { + t.Errorf("invalid item: %d %d", len(item), len(jsonBig)) + } + if len(rest) != 0 { + t.Errorf("invalid rest: %d", len(rest)) + } + + item, rest, err = nextValue(append(jsonBig, "HELLO WORLD"...), &scan) + if err != nil { + t.Fatalf("nextValue extra: %s", err) + } + if len(item) != len(jsonBig) { + t.Errorf("invalid item: %d %d", len(item), len(jsonBig)) + } + if string(rest) != "HELLO WORLD" { + t.Errorf("invalid rest: %d", len(rest)) + } +} + +var benchScan scanner + +func BenchmarkSkipValue(b *testing.B) { + initBig() + for i := 0; i < b.N; i++ { + nextValue(jsonBig, &benchScan) + } + b.SetBytes(int64(len(jsonBig))) +} + +func diff(t *testing.T, a, b []byte) { + for i := 0; ; i++ { + if i >= len(a) || i >= len(b) || a[i] != b[i] { + j := i - 10 + if j < 0 { + j = 0 + } + t.Errorf("diverge at %d: «%s» vs «%s»", i, trim(a[j:]), trim(b[j:])) + return + } + } +} + +func trim(b []byte) []byte { + if len(b) > 20 { + return b[0:20] + } + return b +} + +// Generate a random JSON object. + +var jsonBig []byte + +func initBig() { + n := 10000 + if testing.Short() { + n = 100 + } + b, err := Marshal(genValue(n)) + if err != nil { + panic(err) + } + jsonBig = b +} + +func genValue(n int) interface{} { + if n > 1 { + switch rand.Intn(2) { + case 0: + return genArray(n) + case 1: + return genMap(n) + } + } + switch rand.Intn(3) { + case 0: + return rand.Intn(2) == 0 + case 1: + return rand.NormFloat64() + case 2: + return genString(30) + } + panic("unreachable") +} + +func genString(stddev float64) string { + n := int(math.Abs(rand.NormFloat64()*stddev + stddev/2)) + c := make([]rune, n) + for i := range c { + f := math.Abs(rand.NormFloat64()*64 + 32) + if f > 0x10ffff { + f = 0x10ffff + } + c[i] = rune(f) + } + return string(c) +} + +func genArray(n int) []interface{} { + f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2))) + if f > n { + f = n + } + if f < 1 { + f = 1 + } + x := make([]interface{}, f) + for i := range x { + x[i] = genValue(((i+1)*n)/f - (i*n)/f) + } + return x +} + +func genMap(n int) map[string]interface{} { + f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2))) + if f > n { + f = n + } + if n > 0 && f == 0 { + f = 1 + } + x := make(map[string]interface{}) + for i := 0; i < f; i++ { + x[genString(10)] = genValue(((i+1)*n)/f - (i*n)/f) + } + return x +} diff --git a/src/mongo/gotools/common/json/single_quoted.go b/src/mongo/gotools/common/json/single_quoted.go new file mode 100644 index 00000000000..ca465ee04f2 --- /dev/null +++ b/src/mongo/gotools/common/json/single_quoted.go @@ -0,0 +1,74 @@ +package json + +// Transition functions for recognizing single-quoted strings. +// Adapted from encoding/json/scanner.go. + +// stateInSingleQuotedString is the state after reading `'`. +func stateInSingleQuotedString(s *scanner, c int) int { + if c == '\'' { + s.step = stateEndValue + return scanContinue + } + if c == '\\' { + s.step = stateInSingleQuotedStringEsc + return scanContinue + } + if c < 0x20 { + return s.error(c, "in string literal") + } + return scanContinue +} + +// stateInSingleQuotedStringEsc is the state after reading `'\` during a quoted string. +func stateInSingleQuotedStringEsc(s *scanner, c int) int { + switch c { + case 'b', 'f', 'n', 'r', 't', '\\', '/', '\'': + s.step = stateInSingleQuotedString + return scanContinue + } + if c == 'u' { + s.step = stateInSingleQuotedStringEscU + return scanContinue + } + return s.error(c, "in string escape code") +} + +// stateInSingleQuotedStringEscU is the state after reading `'\u` during a quoted string. +func stateInSingleQuotedStringEscU(s *scanner, c int) int { + if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { + s.step = stateInSingleQuotedStringEscU1 + return scanContinue + } + // numbers + return s.error(c, "in \\u hexadecimal character escape") +} + +// stateInSingleQuotedStringEscU1 is the state after reading `'\u1` during a quoted string. +func stateInSingleQuotedStringEscU1(s *scanner, c int) int { + if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { + s.step = stateInSingleQuotedStringEscU12 + return scanContinue + } + // numbers + return s.error(c, "in \\u hexadecimal character escape") +} + +// stateInSingleQuotedStringEscU12 is the state after reading `'\u12` during a quoted string. +func stateInSingleQuotedStringEscU12(s *scanner, c int) int { + if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { + s.step = stateInSingleQuotedStringEscU123 + return scanContinue + } + // numbers + return s.error(c, "in \\u hexadecimal character escape") +} + +// stateInSingleQuotedStringEscU123 is the state after reading `'\u123` during a quoted string. +func stateInSingleQuotedStringEscU123(s *scanner, c int) int { + if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' { + s.step = stateInSingleQuotedString + return scanContinue + } + // numbers + return s.error(c, "in \\u hexadecimal character escape") +} diff --git a/src/mongo/gotools/common/json/single_quoted_test.go b/src/mongo/gotools/common/json/single_quoted_test.go new file mode 100644 index 00000000000..6b9849109c4 --- /dev/null +++ b/src/mongo/gotools/common/json/single_quoted_test.go @@ -0,0 +1,156 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestSingleQuotedKeys(t *testing.T) { + + Convey("When unmarshalling JSON with single quotes around its keys", t, func() { + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "value" + data := fmt.Sprintf(`{'%v':"%v"}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + So(jsonMap[key], ShouldEqual, value) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + value1, value2, value3 := "value1", "value2", "value3" + data := fmt.Sprintf(`{'%v':"%v",'%v':"%v",'%v':"%v"}`, + key1, value1, key2, value2, key3, value3) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + So(jsonMap[key1], ShouldEqual, value1) + So(jsonMap[key2], ShouldEqual, value2) + So(jsonMap[key3], ShouldEqual, value3) + }) + }) +} + +func TestSingleQuotedValues(t *testing.T) { + + Convey("When unmarshalling JSON with single quotes around its values", t, func() { + + Convey("works for a single value", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "value" + data := fmt.Sprintf(`{"%v":'%v'}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + So(jsonMap[key], ShouldEqual, value) + }) + + Convey("works for multiple values", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + value1, value2, value3 := "value1", "value2", "value3" + data := fmt.Sprintf(`{"%v":'%v',"%v":'%v',"%v":'%v'}`, + key1, value1, key2, value2, key3, value3) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + So(jsonMap[key1], ShouldEqual, value1) + So(jsonMap[key2], ShouldEqual, value2) + So(jsonMap[key3], ShouldEqual, value3) + }) + + Convey("can be used within BinData constructor", func() { + var jsonMap map[string]interface{} + + key := "bindata" + value := "BinData(1, 'xyz')" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(BinData) + So(ok, ShouldBeTrue) + So(jsonValue.Type, ShouldEqual, 1) + So(jsonValue.Base64, ShouldEqual, "xyz") + }) + + Convey("can be used within Boolean constructor", func() { + var jsonMap map[string]interface{} + + key := "boolean" + value := "Boolean('xyz')" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(bool) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, true) + }) + + Convey("can be used within DBRef constructor", func() { + var jsonMap map[string]interface{} + + key := "dbref" + value := "DBRef('examples', 'xyz')" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(DBRef) + So(ok, ShouldBeTrue) + So(jsonValue.Collection, ShouldEqual, "examples") + So(jsonValue.Id, ShouldEqual, "xyz") + So(jsonValue.Database, ShouldBeEmpty) + }) + + Convey("can be used within ObjectId constructor", func() { + var jsonMap map[string]interface{} + + key := "_id" + value := "ObjectId('xyz')" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(ObjectId) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldEqual, ObjectId("xyz")) + }) + + Convey("can be used within RegExp constructor", func() { + var jsonMap map[string]interface{} + + key := "regex" + value := "RegExp('xyz', 'i')" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(RegExp) + So(ok, ShouldBeTrue) + So(jsonValue.Pattern, ShouldEqual, "xyz") + So(jsonValue.Options, ShouldEqual, "i") + }) + }) +} diff --git a/src/mongo/gotools/common/json/stream.go b/src/mongo/gotools/common/json/stream.go new file mode 100644 index 00000000000..58443f211ec --- /dev/null +++ b/src/mongo/gotools/common/json/stream.go @@ -0,0 +1,243 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import ( + "bytes" + "errors" + "io" +) + +// A Decoder reads and decodes JSON objects from an input stream. +type Decoder struct { + R io.Reader + Buf []byte + d decodeState + scan scanner + err error +} + +// NewDecoder returns a new decoder that reads from r. +// +// The decoder introduces its own buffering and may +// read data from r beyond the JSON values requested. +func NewDecoder(r io.Reader) *Decoder { + return &Decoder{R: r} +} + +// UseNumber causes the Decoder to unmarshal a number into an interface{} as a +// Number instead of as a float64. +func (dec *Decoder) UseNumber() { dec.d.useNumber = true } + +// Decode reads the next JSON-encoded value from its +// input and stores it in the value pointed to by v. +// +// See the documentation for Unmarshal for details about +// the conversion of JSON into a Go value. +func (dec *Decoder) DecodeMap() (map[string]interface{}, error) { + if dec.err != nil { + return nil, dec.err + } + + n, err := dec.readValue() + if err != nil { + return nil, err + } + + // Don't save err from unmarshal into dec.err: + // the connection is still usable since we read a complete JSON + // object from it before the error happened. + dec.d.init(dec.Buf[0:n]) + out, err := dec.d.unmarshalMap() + + // Slide rest of data down. + rest := copy(dec.Buf, dec.Buf[n:]) + dec.Buf = dec.Buf[0:rest] + + return out, err +} + +func (dec *Decoder) ScanObject() ([]byte, error) { + if dec.err != nil { + return nil, dec.err + } + + n, err := dec.readValue() + if err != nil { + return nil, err + } + + outbuf := make([]byte, n) + copy(outbuf, dec.Buf[0:n]) + // Slide rest of data down. + rest := copy(dec.Buf, dec.Buf[n:]) + dec.Buf = dec.Buf[0:rest] + return outbuf, nil + +} + +func (dec *Decoder) Decode(v interface{}) error { + if dec.err != nil { + return dec.err + } + + n, err := dec.readValue() + if err != nil { + return err + } + + // Don't save err from unmarshal into dec.err: + // the connection is still usable since we read a complete JSON + // object from it before the error happened. + dec.d.init(dec.Buf[0:n]) + err = dec.d.unmarshal(v) + + // Slide rest of data down. + rest := copy(dec.Buf, dec.Buf[n:]) + dec.Buf = dec.Buf[0:rest] + + return err +} + +// Buffered returns a reader of the data remaining in the Decoder's +// buffer. The reader is valid until the next call to Decode. +func (dec *Decoder) Buffered() io.Reader { + return bytes.NewReader(dec.Buf) +} + +// readValue reads a JSON value into dec.Buf. +// It returns the length of the encoding. +func (dec *Decoder) readValue() (int, error) { + dec.scan.reset() + + scanp := 0 + var err error +Input: + for { + // Look in the buffer for a new value. + for i, c := range dec.Buf[scanp:] { + dec.scan.bytes++ + v := dec.scan.step(&dec.scan, int(c)) + if v == scanEnd { + scanp += i + break Input + } + // scanEnd is delayed one byte. + // We might block trying to get that byte from src, + // so instead invent a space byte. + if (v == scanEndObject || v == scanEndArray) && dec.scan.step(&dec.scan, ' ') == scanEnd { + scanp += i + 1 + break Input + } + if v == scanError { + dec.err = dec.scan.err + return 0, dec.scan.err + } + } + scanp = len(dec.Buf) + + // Did the last read have an error? + // Delayed until now to allow buffer scan. + if err != nil { + if err == io.EOF { + if dec.scan.step(&dec.scan, ' ') == scanEnd { + break Input + } + if nonSpace(dec.Buf) { + err = io.ErrUnexpectedEOF + } + } + dec.err = err + return 0, err + } + + // Make room to read more into the buffer. + const minRead = 512 + if cap(dec.Buf)-len(dec.Buf) < minRead { + newBuf := make([]byte, len(dec.Buf), 2*cap(dec.Buf)+minRead) + copy(newBuf, dec.Buf) + dec.Buf = newBuf + } + + // Read. Delay error for next iteration (after scan). + var n int + n, err = dec.R.Read(dec.Buf[len(dec.Buf):cap(dec.Buf)]) + dec.Buf = dec.Buf[0 : len(dec.Buf)+n] + } + return scanp, nil +} + +func nonSpace(b []byte) bool { + for _, c := range b { + if !isSpace(rune(c)) { + return true + } + } + return false +} + +// An Encoder writes JSON objects to an output stream. +type Encoder struct { + w io.Writer + e encodeState + err error +} + +// NewEncoder returns a new encoder that writes to w. +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{w: w} +} + +// Encode writes the JSON encoding of v to the stream, +// followed by a newline character. +// +// See the documentation for Marshal for details about the +// conversion of Go values to JSON. +func (enc *Encoder) Encode(v interface{}) error { + if enc.err != nil { + return enc.err + } + e := newEncodeState() + err := e.marshal(v) + if err != nil { + return err + } + + // Terminate each value with a newline. + // This makes the output look a little nicer + // when debugging, and some kind of space + // is required if the encoded value was a number, + // so that the reader knows there aren't more + // digits coming. + e.WriteByte('\n') + + if _, err = enc.w.Write(e.Bytes()); err != nil { + enc.err = err + } + encodeStatePool.Put(e) + return err +} + +// RawMessage is a raw encoded JSON object. +// It implements Marshaler and Unmarshaler and can +// be used to delay JSON decoding or precompute a JSON encoding. +type RawMessage []byte + +// MarshalJSON returns *m as the JSON encoding of m. +func (m *RawMessage) MarshalJSON() ([]byte, error) { + return *m, nil +} + +// UnmarshalJSON sets *m to a copy of data. +func (m *RawMessage) UnmarshalJSON(data []byte) error { + if m == nil { + return errors.New("json.RawMessage: UnmarshalJSON on nil pointer") + } + *m = append((*m)[0:0], data...) + return nil +} + +var _ Marshaler = (*RawMessage)(nil) +var _ Unmarshaler = (*RawMessage)(nil) diff --git a/src/mongo/gotools/common/json/stream_test.go b/src/mongo/gotools/common/json/stream_test.go new file mode 100644 index 00000000000..b562e87690d --- /dev/null +++ b/src/mongo/gotools/common/json/stream_test.go @@ -0,0 +1,206 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import ( + "bytes" + "io/ioutil" + "net" + "reflect" + "strings" + "testing" +) + +// Test values for the stream test. +// One of each JSON kind. +var streamTest = []interface{}{ + 0.1, + "hello", + nil, + true, + false, + []interface{}{"a", "b", "c"}, + map[string]interface{}{"K": "Kelvin", "ß": "long s"}, + 3.14, // another value to make sure something can follow map +} + +var streamEncoded = `0.1 +"hello" +null +true +false +["a","b","c"] +{"ß":"long s","K":"Kelvin"} +3.14 +` + +func TestEncoder(t *testing.T) { + for i := 0; i <= len(streamTest); i++ { + var buf bytes.Buffer + enc := NewEncoder(&buf) + for j, v := range streamTest[0:i] { + if err := enc.Encode(v); err != nil { + t.Fatalf("encode #%d: %v", j, err) + } + } + if have, want := buf.String(), nlines(streamEncoded, i); have != want { + t.Errorf("encoding %d items: mismatch", i) + diff(t, []byte(have), []byte(want)) + break + } + } +} + +func TestDecoder(t *testing.T) { + for i := 0; i <= len(streamTest); i++ { + // Use stream without newlines as input, + // just to stress the decoder even more. + // Our test input does not include back-to-back numbers. + // Otherwise stripping the newlines would + // merge two adjacent JSON values. + var buf bytes.Buffer + for _, c := range nlines(streamEncoded, i) { + if c != '\n' { + buf.WriteRune(c) + } + } + out := make([]interface{}, i) + dec := NewDecoder(&buf) + for j := range out { + if err := dec.Decode(&out[j]); err != nil { + t.Fatalf("decode #%d/%d: %v", j, i, err) + } + } + if !reflect.DeepEqual(out, streamTest[0:i]) { + t.Errorf("decoding %d items: mismatch", i) + for j := range out { + if !reflect.DeepEqual(out[j], streamTest[j]) { + t.Errorf("#%d: have %v want %v", j, out[j], streamTest[j]) + } + } + break + } + } +} + +func TestDecoderBuffered(t *testing.T) { + r := strings.NewReader(`{"Name": "Gopher"} extra `) + var m struct { + Name string + } + d := NewDecoder(r) + err := d.Decode(&m) + if err != nil { + t.Fatal(err) + } + if m.Name != "Gopher" { + t.Errorf("Name = %q; want Gopher", m.Name) + } + rest, err := ioutil.ReadAll(d.Buffered()) + if err != nil { + t.Fatal(err) + } + if g, w := string(rest), " extra "; g != w { + t.Errorf("Remaining = %q; want %q", g, w) + } +} + +func nlines(s string, n int) string { + if n <= 0 { + return "" + } + for i, c := range s { + if c == '\n' { + if n--; n == 0 { + return s[0 : i+1] + } + } + } + return s +} + +func TestRawMessage(t *testing.T) { + // TODO(rsc): Should not need the * in *RawMessage + var data struct { + X float64 + Id *RawMessage + Y float32 + } + const raw = `["\u0056",null]` + const msg = `{"X":0.1,"Id":["\u0056",null],"Y":0.2}` + err := Unmarshal([]byte(msg), &data) + if err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if string([]byte(*data.Id)) != raw { + t.Fatalf("Raw mismatch: have %#q want %#q", []byte(*data.Id), raw) + } + b, err := Marshal(&data) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + if string(b) != msg { + t.Fatalf("Marshal: have %#q want %#q", b, msg) + } +} + +func TestNullRawMessage(t *testing.T) { + // TODO(rsc): Should not need the * in *RawMessage + var data struct { + X float64 + Id *RawMessage + Y float32 + } + data.Id = new(RawMessage) + const msg = `{"X":0.1,"Id":null,"Y":0.2}` + err := Unmarshal([]byte(msg), &data) + if err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if data.Id != nil { + t.Fatalf("Raw mismatch: have non-nil, want nil") + } + b, err := Marshal(&data) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + if string(b) != msg { + t.Fatalf("Marshal: have %#q want %#q", b, msg) + } +} + +var blockingTests = []string{ + `{"x": 1}`, + `[1, 2, 3]`, +} + +func TestBlocking(t *testing.T) { + for _, enc := range blockingTests { + r, w := net.Pipe() + go w.Write([]byte(enc)) + var val interface{} + + // If Decode reads beyond what w.Write writes above, + // it will block, and the test will deadlock. + if err := NewDecoder(r).Decode(&val); err != nil { + t.Errorf("decoding %s: %v", enc, err) + } + r.Close() + w.Close() + } +} + +func BenchmarkEncoderEncode(b *testing.B) { + b.ReportAllocs() + type T struct { + X, Y string + } + v := &T{"foo", "bar"} + for i := 0; i < b.N; i++ { + if err := NewEncoder(ioutil.Discard).Encode(v); err != nil { + b.Fatal(err) + } + } +} diff --git a/src/mongo/gotools/common/json/tagkey_test.go b/src/mongo/gotools/common/json/tagkey_test.go new file mode 100644 index 00000000000..1c42a903fd2 --- /dev/null +++ b/src/mongo/gotools/common/json/tagkey_test.go @@ -0,0 +1,110 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import ( + "testing" +) + +type basicLatin2xTag struct { + V string `json:"$%-/"` +} + +type basicLatin3xTag struct { + V string `json:"0123456789"` +} + +type basicLatin4xTag struct { + V string `json:"ABCDEFGHIJKLMO"` +} + +type basicLatin5xTag struct { + V string `json:"PQRSTUVWXYZ_"` +} + +type basicLatin6xTag struct { + V string `json:"abcdefghijklmno"` +} + +type basicLatin7xTag struct { + V string `json:"pqrstuvwxyz"` +} + +type miscPlaneTag struct { + V string `json:"色は匂へど"` +} + +type percentSlashTag struct { + V string `json:"text/html%"` // http://golang.org/issue/2718 +} + +type punctuationTag struct { + V string `json:"!#$%&()*+-./:<=>?@[]^_{|}~"` // http://golang.org/issue/3546 +} + +type emptyTag struct { + W string +} + +type misnamedTag struct { + X string `jsom:"Misnamed"` +} + +type badCodeTag struct { + Z string `json:" !\"#&'()*+,."` +} + +type spaceTag struct { + Q string `json:"With space"` +} + +type unicodeTag struct { + W string `json:"Ελλάδα"` +} + +var structTagObjectKeyTests = []struct { + raw interface{} + value string + key string +}{ + {basicLatin2xTag{"2x"}, "2x", "$%-/"}, + {basicLatin3xTag{"3x"}, "3x", "0123456789"}, + {basicLatin4xTag{"4x"}, "4x", "ABCDEFGHIJKLMO"}, + {basicLatin5xTag{"5x"}, "5x", "PQRSTUVWXYZ_"}, + {basicLatin6xTag{"6x"}, "6x", "abcdefghijklmno"}, + {basicLatin7xTag{"7x"}, "7x", "pqrstuvwxyz"}, + {miscPlaneTag{"いろはにほへと"}, "いろはにほへと", "色は匂へど"}, + {emptyTag{"Pour Moi"}, "Pour Moi", "W"}, + {misnamedTag{"Animal Kingdom"}, "Animal Kingdom", "X"}, + {badCodeTag{"Reliable Man"}, "Reliable Man", "Z"}, + {percentSlashTag{"brut"}, "brut", "text/html%"}, + {punctuationTag{"Union Rags"}, "Union Rags", "!#$%&()*+-./:<=>?@[]^_{|}~"}, + {spaceTag{"Perreddu"}, "Perreddu", "With space"}, + {unicodeTag{"Loukanikos"}, "Loukanikos", "Ελλάδα"}, +} + +func TestStructTagObjectKey(t *testing.T) { + for _, tt := range structTagObjectKeyTests { + b, err := Marshal(tt.raw) + if err != nil { + t.Fatalf("Marshal(%#q) failed: %v", tt.raw, err) + } + var f interface{} + err = Unmarshal(b, &f) + if err != nil { + t.Fatalf("Unmarshal(%#q) failed: %v", b, err) + } + for i, v := range f.(map[string]interface{}) { + switch i { + case tt.key: + if s, ok := v.(string); !ok || s != tt.value { + t.Fatalf("Unexpected value: %#q, want %v", s, tt.value) + } + default: + t.Fatalf("Unexpected key: %#q, from %#q", i, b) + } + } + } +} diff --git a/src/mongo/gotools/common/json/tags.go b/src/mongo/gotools/common/json/tags.go new file mode 100644 index 00000000000..c38fd5102f6 --- /dev/null +++ b/src/mongo/gotools/common/json/tags.go @@ -0,0 +1,44 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import ( + "strings" +) + +// tagOptions is the string following a comma in a struct field's "json" +// tag, or the empty string. It does not include the leading comma. +type tagOptions string + +// parseTag splits a struct field's json tag into its name and +// comma-separated options. +func parseTag(tag string) (string, tagOptions) { + if idx := strings.Index(tag, ","); idx != -1 { + return tag[:idx], tagOptions(tag[idx+1:]) + } + return tag, tagOptions("") +} + +// Contains reports whether a comma-separated list of options +// contains a particular substr flag. substr must be surrounded by a +// string boundary or commas. +func (o tagOptions) Contains(optionName string) bool { + if len(o) == 0 { + return false + } + s := string(o) + for s != "" { + var next string + i := strings.Index(s, ",") + if i >= 0 { + s, next = s[:i], s[i+1:] + } + if s == optionName { + return true + } + s = next + } + return false +} diff --git a/src/mongo/gotools/common/json/tags_test.go b/src/mongo/gotools/common/json/tags_test.go new file mode 100644 index 00000000000..91fb18831e2 --- /dev/null +++ b/src/mongo/gotools/common/json/tags_test.go @@ -0,0 +1,28 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import ( + "testing" +) + +func TestTagParsing(t *testing.T) { + name, opts := parseTag("field,foobar,foo") + if name != "field" { + t.Fatalf("name = %q, want field", name) + } + for _, tt := range []struct { + opt string + want bool + }{ + {"foobar", true}, + {"foo", true}, + {"bar", false}, + } { + if opts.Contains(tt.opt) != tt.want { + t.Errorf("Contains(%q) = %v", tt.opt, !tt.want) + } + } +} diff --git a/src/mongo/gotools/common/json/testdata/code.json.gz b/src/mongo/gotools/common/json/testdata/code.json.gz Binary files differnew file mode 100644 index 00000000000..0e2895b53ac --- /dev/null +++ b/src/mongo/gotools/common/json/testdata/code.json.gz diff --git a/src/mongo/gotools/common/json/timestamp.go b/src/mongo/gotools/common/json/timestamp.go new file mode 100644 index 00000000000..dbc38ef56e6 --- /dev/null +++ b/src/mongo/gotools/common/json/timestamp.go @@ -0,0 +1,67 @@ +package json + +import ( + "fmt" + "reflect" +) + +// Transition functions for recognizing Timestamp. +// Adapted from encoding/json/scanner.go. + +// stateUpperT is the state after reading `T`. +func stateUpperT(s *scanner, c int) int { + if c == 'i' { + s.step = generateState("Timestamp", []byte("mestamp"), stateConstructor) + return scanContinue + } + return s.error(c, "in literal Timestamp (expecting 'i')") +} + +// Decodes a Timestamp literal stored in the underlying byte data into v. +func (d *decodeState) storeTimestamp(v reflect.Value) { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginCtor { + d.error(fmt.Errorf("expected beginning of constructor")) + } + + args, err := d.ctor("Timestamp", []reflect.Type{uint32Type, uint32Type}) + if err != nil { + d.error(err) + } + switch kind := v.Kind(); kind { + case reflect.Interface: + arg0 := uint32(args[0].Uint()) + arg1 := uint32(args[1].Uint()) + v.Set(reflect.ValueOf(Timestamp{arg0, arg1})) + default: + d.error(fmt.Errorf("cannot store %v value into %v type", timestampType, kind)) + } +} + +// Returns a Timestamp literal from the underlying byte data. +func (d *decodeState) getTimestamp() interface{} { + op := d.scanWhile(scanSkipSpace) + if op != scanBeginCtor { + d.error(fmt.Errorf("expected beginning of constructor")) + } + + // Prevent d.convertNumber() from parsing the arguments as float64s. + useNumber := d.useNumber + d.useNumber = true + + args := d.ctorInterface() + if err := ctorNumArgsMismatch("Timestamp", 2, len(args)); err != nil { + d.error(err) + } + arg0, err := args[0].(Number).Uint32() + if err != nil { + d.error(fmt.Errorf("expected uint32 for first argument of Timestamp constructor")) + } + arg1, err := args[1].(Number).Uint32() + if err != nil { + d.error(fmt.Errorf("expected uint32 for second argument of Timestamp constructor")) + } + + d.useNumber = useNumber + return Timestamp{arg0, arg1} +} diff --git a/src/mongo/gotools/common/json/timestamp_test.go b/src/mongo/gotools/common/json/timestamp_test.go new file mode 100644 index 00000000000..027d9b4cb30 --- /dev/null +++ b/src/mongo/gotools/common/json/timestamp_test.go @@ -0,0 +1,85 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestTimestampValue(t *testing.T) { + + Convey("When unmarshalling JSON with Timestamp values", t, func() { + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "Timestamp(123, 321)" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(Timestamp) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, Timestamp{123, 321}) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + value1, value2, value3 := "Timestamp(123, 321)", + "Timestamp(456, 654)", "Timestamp(789, 987)" + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value1, key2, value2, key3, value3) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue1, ok := jsonMap[key1].(Timestamp) + So(ok, ShouldBeTrue) + So(jsonValue1, ShouldResemble, Timestamp{123, 321}) + + jsonValue2, ok := jsonMap[key2].(Timestamp) + So(ok, ShouldBeTrue) + So(jsonValue2, ShouldResemble, Timestamp{456, 654}) + + jsonValue3, ok := jsonMap[key3].(Timestamp) + So(ok, ShouldBeTrue) + So(jsonValue3, ShouldResemble, Timestamp{789, 987}) + }) + + Convey("works in an array", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "Timestamp(42, 10)" + data := fmt.Sprintf(`{"%v":[%v,%v,%v]}`, + key, value, value, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonArray, ok := jsonMap[key].([]interface{}) + So(ok, ShouldBeTrue) + + for _, _jsonValue := range jsonArray { + jsonValue, ok := _jsonValue.(Timestamp) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, Timestamp{42, 10}) + } + }) + + Convey("cannot use string as argument", func() { + var jsonMap map[string]interface{} + + key := "key" + value := `Timestamp("123", "321")` + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + }) + }) +} diff --git a/src/mongo/gotools/common/json/undefined.go b/src/mongo/gotools/common/json/undefined.go new file mode 100644 index 00000000000..20cb3e9a8b3 --- /dev/null +++ b/src/mongo/gotools/common/json/undefined.go @@ -0,0 +1,13 @@ +package json + +// Transition functions for recognizing undefined. +// Adapted from encoding/json/scanner.go. + +// stateU is the state after reading `u`. +func stateU(s *scanner, c int) int { + if c == 'n' { + s.step = generateState("undefined", []byte("defined"), stateEndValue) + return scanContinue + } + return s.error(c, "in literal undefined (expecting 'n')") +} diff --git a/src/mongo/gotools/common/json/undefined_test.go b/src/mongo/gotools/common/json/undefined_test.go new file mode 100644 index 00000000000..367220ff8cd --- /dev/null +++ b/src/mongo/gotools/common/json/undefined_test.go @@ -0,0 +1,89 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestUndefinedValue(t *testing.T) { + + Convey("When unmarshalling JSON with undefined values", t, func() { + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "undefined" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue, ok := jsonMap[key].(Undefined) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, Undefined{}) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + value := "undefined" + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value, key2, value, key3, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonValue1, ok := jsonMap[key1].(Undefined) + So(ok, ShouldBeTrue) + So(jsonValue1, ShouldResemble, Undefined{}) + + jsonValue2, ok := jsonMap[key2].(Undefined) + So(ok, ShouldBeTrue) + So(jsonValue2, ShouldResemble, Undefined{}) + + jsonValue3, ok := jsonMap[key3].(Undefined) + So(ok, ShouldBeTrue) + So(jsonValue3, ShouldResemble, Undefined{}) + }) + + Convey("works in an array", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "undefined" + data := fmt.Sprintf(`{"%v":[%v,%v,%v]}`, + key, value, value, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + jsonArray, ok := jsonMap[key].([]interface{}) + So(ok, ShouldBeTrue) + + for _, _jsonValue := range jsonArray { + jsonValue, ok := _jsonValue.(Undefined) + So(ok, ShouldBeTrue) + So(jsonValue, ShouldResemble, Undefined{}) + } + }) + + Convey("cannot have a sign ('+' or '-')", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "undefined" + data := fmt.Sprintf(`{"%v":+%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + + data = fmt.Sprintf(`{"%v":-%v}`, key, value) + + err = Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + }) + }) +} diff --git a/src/mongo/gotools/common/json/unquoted.go b/src/mongo/gotools/common/json/unquoted.go new file mode 100644 index 00000000000..91aa2485907 --- /dev/null +++ b/src/mongo/gotools/common/json/unquoted.go @@ -0,0 +1,31 @@ +package json + +// Transition function for recognizing unquoted strings. +// Adapted from encoding/json/scanner.go. + +func isBeginUnquotedString(c int) bool { + return c == '$' || c == '_' || 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' +} + +func isInUnquotedString(c int) bool { + return isBeginUnquotedString(c) || '0' <= c && c <= '9' +} + +func stateInUnquotedString(s *scanner, c int) int { + if isInUnquotedString(c) { + return scanContinue + } + return stateEndValue(s, c) +} + +// Decoder function that immediately returns an already unquoted string. +// Adapted from encoding/json/decode.go. +func maybeUnquoteBytes(s []byte) ([]byte, bool) { + if len(s) == 0 { + return nil, false + } + if s[0] != '"' && s[len(s)-1] != '"' && s[0] != '\'' && s[len(s)-1] != '\'' { + return s, true + } + return unquoteBytes(s) +} diff --git a/src/mongo/gotools/common/json/unquoted_test.go b/src/mongo/gotools/common/json/unquoted_test.go new file mode 100644 index 00000000000..4bf9a47638b --- /dev/null +++ b/src/mongo/gotools/common/json/unquoted_test.go @@ -0,0 +1,129 @@ +package json + +import ( + "fmt" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestUnquotedKeys(t *testing.T) { + + Convey("When unmarshalling JSON without quotes around its keys", t, func() { + + Convey("works for a single key", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "value" + data := fmt.Sprintf(`{%v:"%v"}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + So(jsonMap[key], ShouldEqual, value) + }) + + Convey("works for multiple keys", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + value1, value2, value3 := "value1", "value2", "value3" + data := fmt.Sprintf(`{%v:"%v",%v:"%v",%v:"%v"}`, + key1, value1, key2, value2, key3, value3) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + So(jsonMap[key1], ShouldEqual, value1) + So(jsonMap[key2], ShouldEqual, value2) + So(jsonMap[key3], ShouldEqual, value3) + }) + + Convey("can start with a dollar sign ('$')", func() { + var jsonMap map[string]interface{} + + key := "$dollar" + value := "money" + data := fmt.Sprintf(`{%v:"%v"}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + So(jsonMap[key], ShouldEqual, value) + }) + + Convey("can start with an underscore ('_')", func() { + var jsonMap map[string]interface{} + + key := "_id" + value := "unique" + data := fmt.Sprintf(`{%v:"%v"}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + So(jsonMap[key], ShouldEqual, value) + }) + + Convey("cannot start with a number ('[0-9]')", func() { + var jsonMap map[string]interface{} + + key := "073" + value := "octal" + data := fmt.Sprintf(`{%v:"%v"}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + }) + + Convey("can contain numbers ('[0-9]')", func() { + var jsonMap map[string]interface{} + + key := "b16" + value := "little" + data := fmt.Sprintf(`{%v:"%v"}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldBeNil) + + So(jsonMap[key], ShouldEqual, value) + }) + + Convey("cannot contain a period ('.')", func() { + var jsonMap map[string]interface{} + + key := "horse.horse" + value := "horse" + data := fmt.Sprintf(`{%v:"%v"}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + }) + }) + + Convey("When unmarshalling JSON without quotes around its values", t, func() { + + Convey("fails for a single value", func() { + var jsonMap map[string]interface{} + + key := "key" + value := "value" + data := fmt.Sprintf(`{"%v":%v}`, key, value) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + }) + + Convey("fails for multiple values", func() { + var jsonMap map[string]interface{} + + key1, key2, key3 := "key1", "key2", "key3" + value1, value2, value3 := "value1", "value2", "value3" + data := fmt.Sprintf(`{"%v":%v,"%v":%v,"%v":%v}`, + key1, value1, key2, value2, key3, value3) + + err := Unmarshal([]byte(data), &jsonMap) + So(err, ShouldNotBeNil) + }) + }) +} |