diff options
author | David Golden <xdg@xdg.me> | 2019-04-24 17:28:43 -0400 |
---|---|---|
committer | David Golden <xdg@xdg.me> | 2019-04-24 19:12:44 -0400 |
commit | de30bf68aa08bc967b0f601d2e5fdb63e783b6b4 (patch) | |
tree | 099e6ef23ece09a7daa2e49aaabd271594a40a2e /src/mongo/gotools/src/github.com | |
parent | c8debc4dbb35689fc79e036f6ee51d0c6f51e5fe (diff) | |
download | mongo-de30bf68aa08bc967b0f601d2e5fdb63e783b6b4.tar.gz |
Import tools: d91cc9fc08dd8f6dc5d71cdfcdefafd70f35149b from branch v4.1
ref: e4705b54b0..d91cc9fc08
for: 4.1.11
TOOLS-1843 Remove unused common packages
TOOLS-2258 mongoimport fails to connect when no URI or host is specified
Diffstat (limited to 'src/mongo/gotools/src/github.com')
212 files changed, 106 insertions, 10228 deletions
diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/Gopkg.lock b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/Gopkg.lock index c474c3df712..c2327119e89 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/Gopkg.lock +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/Gopkg.lock @@ -104,7 +104,7 @@ [[projects]] branch = "master" - digest = "1:34fd703595b17e19c3025578b7de549c8fb043995309506921df4ffc87eb9ce7" + digest = "1:64629229e2ed5c7af1cb1629b8b85ef8d1b3b7c949e6d96dfd19afa383e9030f" name = "github.com/mongodb/mongo-tools-common" packages = [ "archive", @@ -125,7 +125,7 @@ "util", ] pruneopts = "T" - revision = "cf9a7970421213e06e0f745fa2adeb861fa12f4a" + revision = "f450824d00a780709a0175633d91b50ffb766c8c" [[projects]] digest = "1:f363c75e8cac5653bc5c0c2b90cbd8a522fdc48c13a5f8d85078750f82d1a009" diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common.yml b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common.yml index 3ba3c28dd5b..38df99c0ccd 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common.yml +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common.yml @@ -281,7 +281,7 @@ mongo_tools_variables: mongod_args: "" mongod_port: 33333 ssl: &mongod_ssl_startup_args - mongod_args: "--sslMode requireSSL --sslCAFile common/db/openssl/testdata/ca.pem --sslPEMKeyFile common/db/openssl/testdata/server.pem" + mongod_args: "--sslMode requireSSL --sslCAFile testdata/certs/ca.pem --sslPEMKeyFile testdata/certs/server.pem" mongod_port: 33333 # Set storage engine as mmapv1 for 32 bit variants because WiredTiger requires 64 bit support. win32: &mongod_win32_startup_args @@ -293,7 +293,7 @@ mongo_tools_variables: mongo_args: &mongo_default_startup_args_string "--port 33333" mongod_port: 33333 ssl: &mongo_ssl_startup_args - mongo_args: "--port 33333 --ssl --sslCAFile ./common/db/openssl/testdata/ca.pem --sslPEMKeyFile ./common/db/openssl/testdata/server.pem --sslAllowInvalidCertificates" + mongo_args: "--port 33333 --ssl --sslCAFile ./testdata/certs/ca.pem --sslPEMKeyFile ./testdata/certs/server.pem --sslAllowInvalidCertificates" mongod_port: 33333 functions: @@ -1053,7 +1053,7 @@ tasks: set -e . ./set_goenv.sh GOROOT="" set_goenv || exit - retVal=$(go run vendor/github.com/3rf/mongo-lint/golint/golint.go mongo* bson* common/*); + retVal=$(go run vendor/github.com/3rf/mongo-lint/golint/golint.go mongo* bson* legacy/*); if [ "$retVal" = "" ]; then exit 0; else echo $retVal; exit 1; fi; - name: lint-js @@ -1453,7 +1453,7 @@ tasks: set -x set -v set -e - go tool vet bsondump common mongo* + go tool vet bsondump legacy mongo* - name: replay-dist commands: diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/archive.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/archive.go deleted file mode 100644 index 05f84ebaa44..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/archive.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package archive - -import "io" - -// NamespaceHeader is a data structure that, as BSON, is found in archives where it indicates -// that either the subsequent stream of BSON belongs to this new namespace, or that the -// indicated namespace will have no more documents (EOF) -type NamespaceHeader struct { - Database string `bson:"db"` - Collection string `bson:"collection"` - EOF bool `bson:"EOF"` - CRC int64 `bson:"CRC"` -} - -// CollectionMetadata is a data structure that, as BSON, is found in the prelude of the archive. -// There is one CollectionMetadata per collection that will be in the archive. -type CollectionMetadata struct { - Database string `bson:"db"` - Collection string `bson:"collection"` - Metadata string `bson:"metadata"` - Size int `bson:"size"` -} - -// Header is a data structure that, as BSON, is found immediately after the magic -// number in the archive, before any CollectionMetadatas. It is the home of any archive level information -type Header struct { - ConcurrentCollections int32 `bson:"concurrent_collections"` - FormatVersion string `bson:"version"` - ServerVersion string `bson:"server_version"` - ToolVersion string `bson:"tool_version"` -} - -const minBSONSize = 4 + 1 // an empty BSON document should be exactly five bytes long - -var terminator int32 = -1 -var terminatorBytes = []byte{0xFF, 0xFF, 0xFF, 0xFF} // TODO, rectify this with terminator - -// MagicNumber is four bytes that are found at the beginning of the archive that indicate that -// the byte stream is an archive, as opposed to anything else, including a stream of BSON documents -const MagicNumber uint32 = 0x8199e26d -const archiveFormatVersion = "0.1" - -// Writer is the top level object to contain information about archives in mongodump -type Writer struct { - Out io.WriteCloser - Prelude *Prelude - Mux *Multiplexer -} - -// Reader is the top level object to contain information about archives in mongorestore -type Reader struct { - In io.ReadCloser - Demux *Demultiplexer - Prelude *Prelude -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/demultiplexer.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/demultiplexer.go deleted file mode 100644 index c1659708115..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/demultiplexer.go +++ /dev/null @@ -1,487 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package archive - -import ( - "bytes" - "fmt" - "hash" - "hash/crc64" - "io" - "sync" - "sync/atomic" - - "github.com/mongodb/mongo-tools/common/db" - "github.com/mongodb/mongo-tools/common/intents" - "github.com/mongodb/mongo-tools/common/log" - "gopkg.in/mgo.v2/bson" -) - -// DemuxOut is a Demultiplexer output consumer -// The Write() and Close() occur in the same thread as the Demultiplexer runs in. -type DemuxOut interface { - Write([]byte) (int, error) - Close() error - Sum64() (uint64, bool) -} - -const ( - NamespaceUnopened = iota - NamespaceOpened - NamespaceClosed -) - -// Demultiplexer implements Parser. -type Demultiplexer struct { - In io.Reader - //TODO wrap up these three into a structure - outs map[string]DemuxOut - lengths map[string]int64 - currentNamespace string - buf [db.MaxBSONSize]byte - NamespaceChan chan string - NamespaceErrorChan chan error - NamespaceStatus map[string]int -} - -func CreateDemux(namespaceMetadatas []*CollectionMetadata, in io.Reader) *Demultiplexer { - demux := &Demultiplexer{ - NamespaceStatus: make(map[string]int), - In: in, - } - for _, cm := range namespaceMetadatas { - ns := cm.Database + "." + cm.Collection - demux.NamespaceStatus[ns] = NamespaceUnopened - - } - return demux -} - -// Run creates and runs a parser with the Demultiplexer as a consumer -func (demux *Demultiplexer) Run() error { - parser := Parser{In: demux.In} - err := parser.ReadAllBlocks(demux) - if len(demux.outs) > 0 { - log.Logvf(log.Always, "demux finishing when there are still outs (%v)", len(demux.outs)) - } - - log.Logvf(log.DebugLow, "demux finishing (err:%v)", err) - return err -} - -type demuxError struct { - Err error - Msg string -} - -// Error is part of the Error interface. It formats a demuxError for human readability. -func (pe *demuxError) Error() string { - err := fmt.Sprintf("error demultiplexing archive; %v", pe.Msg) - if pe.Err != nil { - err = fmt.Sprintf("%v ( %v )", err, pe.Err) - } - return err -} - -// newError creates a demuxError with just a message -func newError(msg string) error { - return &demuxError{ - Msg: msg, - } -} - -// newWrappedError creates a demuxError with a message as well as an underlying cause error -func newWrappedError(msg string, err error) error { - return &demuxError{ - Err: err, - Msg: msg, - } -} - -// HeaderBSON is part of the ParserConsumer interface and receives headers from parser. -// Its main role is to implement opens and EOFs of the embedded stream. -func (demux *Demultiplexer) HeaderBSON(buf []byte) error { - colHeader := NamespaceHeader{} - err := bson.Unmarshal(buf, &colHeader) - if err != nil { - return newWrappedError("header bson doesn't unmarshal as a collection header", err) - } - log.Logvf(log.DebugHigh, "demux namespaceHeader: %v", colHeader) - if colHeader.Collection == "" { - return newError("collection header is missing a Collection") - } - demux.currentNamespace = colHeader.Database + "." + colHeader.Collection - if _, ok := demux.outs[demux.currentNamespace]; !ok { - if demux.NamespaceStatus[demux.currentNamespace] != NamespaceUnopened { - return newError("namespace header for already opened namespace") - } - demux.NamespaceStatus[demux.currentNamespace] = NamespaceOpened - if demux.NamespaceChan != nil { - demux.NamespaceChan <- demux.currentNamespace - err := <-demux.NamespaceErrorChan - if err == io.EOF { - // if the Prioritizer sends us back an io.EOF then it's telling us that - // it's finishing and doesn't need any more namespace announcements. - close(demux.NamespaceChan) - demux.NamespaceChan = nil - return nil - } - if err != nil { - return newWrappedError("failed arranging a consumer for new namespace", err) - } - } - } - if colHeader.EOF { - if rcr, ok := demux.outs[demux.currentNamespace].(*RegularCollectionReceiver); ok { - rcr.err = io.EOF - } - demux.outs[demux.currentNamespace].Close() - demux.NamespaceStatus[demux.currentNamespace] = NamespaceClosed - length := int64(demux.lengths[demux.currentNamespace]) - crcUInt64, ok := demux.outs[demux.currentNamespace].Sum64() - if ok { - crc := int64(crcUInt64) - if crc != colHeader.CRC { - return fmt.Errorf("CRC mismatch for namespace %v, %v!=%v", - demux.currentNamespace, - crc, - colHeader.CRC, - ) - } - log.Logvf(log.DebugHigh, - "demux checksum for namespace %v is correct (%v), %v bytes", - demux.currentNamespace, crc, length) - } else { - log.Logvf(log.DebugHigh, - "demux checksum for namespace %v was not calculated.", - demux.currentNamespace) - } - delete(demux.outs, demux.currentNamespace) - delete(demux.lengths, demux.currentNamespace) - // in case we get a BSONBody with this block, - // we want to ensure that that causes an error - demux.currentNamespace = "" - } - return nil -} - -// End is part of the ParserConsumer interface and receives the end of archive notification. -func (demux *Demultiplexer) End() error { - log.Logvf(log.DebugHigh, "demux End") - var err error - if len(demux.outs) != 0 { - openNss := []string{} - for ns := range demux.outs { - openNss = append(openNss, ns) - if rcr, ok := demux.outs[ns].(*RegularCollectionReceiver); ok { - rcr.err = newError("archive io error") - } - demux.outs[ns].Close() - } - err = newError(fmt.Sprintf("archive finished but contained files were unfinished (%v)", openNss)) - } else { - for ns, status := range demux.NamespaceStatus { - if status != NamespaceClosed { - err = newError(fmt.Sprintf("archive finished before all collections were seen (%v)", ns)) - } - } - } - - if demux.NamespaceChan != nil { - close(demux.NamespaceChan) - } - - return err -} - -// BodyBSON is part of the ParserConsumer interface and receives BSON bodies from the parser. -// Its main role is to dispatch the body to the Read() function of the current DemuxOut. -func (demux *Demultiplexer) BodyBSON(buf []byte) error { - if demux.currentNamespace == "" { - return newError("collection data without a collection header") - } - - demux.lengths[demux.currentNamespace] += int64(len(buf)) - - out, ok := demux.outs[demux.currentNamespace] - if !ok { - return newError("no demux consumer currently consuming namespace " + demux.currentNamespace) - } - _, err := out.Write(buf) - return err -} - -// Open installs the DemuxOut as the handler for data for the namespace ns -func (demux *Demultiplexer) Open(ns string, out DemuxOut) { - // In the current implementation where this is either called before the demultiplexing is running - // or while the demutiplexer is inside of the NamespaceChan NamespaceErrorChan conversation - // I think that we don't need to lock outs, but I suspect that if the implementation changes - // we may need to lock when outs is accessed - log.Logvf(log.DebugHigh, "demux Open") - if demux.outs == nil { - demux.outs = make(map[string]DemuxOut) - demux.lengths = make(map[string]int64) - } - demux.outs[ns] = out - demux.lengths[ns] = 0 -} - -// RegularCollectionReceiver implements the intents.file interface. -type RegularCollectionReceiver struct { - pos int64 // updated atomically, aligned at the beginning of the struct - readLenChan chan int - readBufChan chan []byte - Intent *intents.Intent - Origin string - Demux *Demultiplexer - partialReadArray []byte - partialReadBuf []byte - hash hash.Hash64 - closeOnce sync.Once - openOnce sync.Once - err error -} - -func (receiver *RegularCollectionReceiver) Sum64() (uint64, bool) { - return receiver.hash.Sum64(), true -} - -// Read() runs in the restoring goroutine -func (receiver *RegularCollectionReceiver) Read(r []byte) (int, error) { - if receiver.partialReadBuf != nil && len(receiver.partialReadBuf) > 0 { - wLen := len(receiver.partialReadBuf) - copyLen := copy(r, receiver.partialReadBuf) - if wLen == copyLen { - receiver.partialReadBuf = nil - } else { - receiver.partialReadBuf = receiver.partialReadBuf[copyLen:] - } - atomic.AddInt64(&receiver.pos, int64(copyLen)) - return copyLen, nil - } - // Since we're the "reader" here, not the "writer" we need to start with a read, in case the chan is closed - wLen, ok := <-receiver.readLenChan - if !ok { - close(receiver.readBufChan) - return 0, receiver.err - } - if wLen > db.MaxBSONSize { - return 0, fmt.Errorf("incoming buffer size is too big %v", wLen) - } - rLen := len(r) - if wLen > rLen { - // if the incoming write size is larger then the incoming read buffer then we need to accept - // the write in a larger buffer, fill the read buffer, then cache the remainder - receiver.partialReadBuf = receiver.partialReadArray[:wLen] - receiver.readBufChan <- receiver.partialReadBuf - writtenLength := <-receiver.readLenChan - if wLen != writtenLength { - return 0, fmt.Errorf("regularCollectionReceiver didn't send what it said it would") - } - receiver.hash.Write(receiver.partialReadBuf) - copy(r, receiver.partialReadBuf) - receiver.partialReadBuf = receiver.partialReadBuf[rLen:] - atomic.AddInt64(&receiver.pos, int64(rLen)) - return rLen, nil - } - // Send the read buff to the BodyBSON ParserConsumer to fill - receiver.readBufChan <- r - // Receiver the wLen of data written - wLen = <-receiver.readLenChan - receiver.hash.Write(r[:wLen]) - atomic.AddInt64(&receiver.pos, int64(wLen)) - return wLen, nil -} - -func (receiver *RegularCollectionReceiver) Pos() int64 { - return atomic.LoadInt64(&receiver.pos) -} - -// Open is part of the intents.file interface. It creates the chan's in the -// RegularCollectionReceiver and adds the RegularCollectionReceiver to the set of -// RegularCollectionReceivers in the demultiplexer -func (receiver *RegularCollectionReceiver) Open() error { - // TODO move this implementation to some non intents.file method, to be called from prioritizer.Get - // So that we don't have to enable this double open stuff. - // Currently the open needs to finish before the prioritizer.Get finishes, so we open the intents.file - // in prioritizer.Get even though it's going to get opened again in DumpIntent. - receiver.openOnce.Do(func() { - receiver.readLenChan = make(chan int) - receiver.readBufChan = make(chan []byte) - receiver.hash = crc64.New(crc64.MakeTable(crc64.ECMA)) - receiver.Demux.Open(receiver.Origin, receiver) - }) - return nil -} - -func (receiver *RegularCollectionReceiver) TakeIOBuffer(ioBuf []byte) { - receiver.partialReadArray = ioBuf - -} -func (receiver *RegularCollectionReceiver) ReleaseIOBuffer() { - receiver.partialReadArray = nil -} - -// Write is part of the DemuxOut interface. -func (receiver *RegularCollectionReceiver) Write(buf []byte) (int, error) { - // As a writer, we need to write first, so that the reader can properly detect EOF - // Additionally, the reader needs to know the write size, so that it can give us a - // properly sized buffer. Sending the incoming buffersize fills both of these needs. - receiver.readLenChan <- len(buf) - // Receive from the reader a buffer to put the bytes into - readBuf := <-receiver.readBufChan - if len(readBuf) < len(buf) { - return 0, fmt.Errorf("readbuf is not large enough for incoming BodyBSON (%v<%v)", - len(readBuf), len(buf)) - } - copy(readBuf, buf) - // Send back the length of the data copied in to the buffer - receiver.readLenChan <- len(buf) - return len(buf), nil -} - -// Close is part of the DemuxOut as well as the intents.file interface. It only closes the readLenChan, as that is what will -// cause the RegularCollectionReceiver.Read() to receive EOF -// Close will get called twice, once in the demultiplexer, and again when the restore goroutine is done with its intent.file -func (receiver *RegularCollectionReceiver) Close() error { - receiver.closeOnce.Do(func() { - close(receiver.readLenChan) - // make sure that we don't return until any reader has finished - <-receiver.readBufChan - }) - return nil -} - -// SpecialCollectionCache implements both DemuxOut as well as intents.file -type SpecialCollectionCache struct { - pos int64 // updated atomically, aligned at the beginning of the struct - Intent *intents.Intent - Demux *Demultiplexer - buf bytes.Buffer - hash hash.Hash64 -} - -func NewSpecialCollectionCache(intent *intents.Intent, demux *Demultiplexer) *SpecialCollectionCache { - return &SpecialCollectionCache{ - Intent: intent, - Demux: demux, - hash: crc64.New(crc64.MakeTable(crc64.ECMA)), - } -} - -// Open is part of the both interfaces, and it does nothing -func (cache *SpecialCollectionCache) Open() error { - return nil -} - -// Close is part of the both interfaces, and it does nothing -func (cache *SpecialCollectionCache) Close() error { - cache.Intent.Size = int64(cache.buf.Len()) - return nil -} - -func (cache *SpecialCollectionCache) Read(p []byte) (int, error) { - n, err := cache.buf.Read(p) - atomic.AddInt64(&cache.pos, int64(n)) - return n, err -} - -func (cache *SpecialCollectionCache) Pos() int64 { - return atomic.LoadInt64(&cache.pos) -} - -func (cache *SpecialCollectionCache) Write(b []byte) (int, error) { - cache.hash.Write(b) - return cache.buf.Write(b) -} - -func (cache *SpecialCollectionCache) Sum64() (uint64, bool) { - return cache.hash.Sum64(), true -} - -// MutedCollection implements both DemuxOut as well as intents.file. It serves as a way to -// let the demutiplexer ignore certain embedded streams -type MutedCollection struct { - Intent *intents.Intent - Demux *Demultiplexer -} - -// Read is part of the intents.file interface, and does nothing -func (*MutedCollection) Read([]byte) (int, error) { - // Read is part of the intents.file interface, and does nothing - return 0, io.EOF -} - -// Write is part of the intents.file interface, and does nothing -func (*MutedCollection) Write(b []byte) (int, error) { - return len(b), nil -} - -// Close is part of the intents.file interface, and does nothing -func (*MutedCollection) Close() error { - return nil -} - -// Open is part of the intents.file interface, and does nothing -func (*MutedCollection) Open() error { - return nil -} - -// Sum64 is part of the DemuxOut interface -func (*MutedCollection) Sum64() (uint64, bool) { - return 0, false -} - -//===== Archive Manager Prioritizer ===== - -// NewPrioritizer creates a new Prioritizer and hooks up its Namespace channels to the ones in demux -func (demux *Demultiplexer) NewPrioritizer(mgr *intents.Manager) *Prioritizer { - return &Prioritizer{ - NamespaceChan: demux.NamespaceChan, - NamespaceErrorChan: demux.NamespaceErrorChan, - mgr: mgr, - } -} - -// Prioritizer is a completely reactive prioritizer -// Intents are handed out as they arrive in the archive -type Prioritizer struct { - NamespaceChan <-chan string - NamespaceErrorChan chan<- error - mgr *intents.Manager -} - -// Get waits for a new namespace from the NamespaceChan, and returns a Intent found for it -func (prioritizer *Prioritizer) Get() *intents.Intent { - namespace, ok := <-prioritizer.NamespaceChan - if !ok { - return nil - } - intent := prioritizer.mgr.IntentForNamespace(namespace) - if intent == nil { - prioritizer.NamespaceErrorChan <- fmt.Errorf("no intent for namespace %v", namespace) - } else { - if intent.BSONFile != nil { - intent.BSONFile.Open() - } - if intent.IsOplog() { - // once we see the oplog we - // cause the RestoreIntents to finish because we don't - // want RestoreIntents to restore the oplog - prioritizer.NamespaceErrorChan <- io.EOF - return nil - } - prioritizer.NamespaceErrorChan <- nil - } - return intent -} - -// Finish is part of the IntentPrioritizer interface, and does nothing -func (prioritizer *Prioritizer) Finish(*intents.Intent) { - // no-op - return -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/multiplexer.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/multiplexer.go deleted file mode 100644 index 5d075e0fec7..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/multiplexer.go +++ /dev/null @@ -1,321 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package archive - -import ( - "fmt" - "hash" - "hash/crc64" - "io" - "reflect" - - "github.com/mongodb/mongo-tools/common/db" - "github.com/mongodb/mongo-tools/common/intents" - "github.com/mongodb/mongo-tools/common/log" - "gopkg.in/mgo.v2/bson" -) - -// bufferSize enables or disables the MuxIn buffering -// TODO: remove this constant and the non-buffered MuxIn implementations -const bufferWrites = true -const bufferSize = db.MaxBSONSize - -// Multiplexer is what one uses to create interleaved intents in an archive -type Multiplexer struct { - Out io.WriteCloser - Control chan *MuxIn - Completed chan error - // shutdownInputs allows the mux to tell the intent dumping worker - // go routines to shutdown, so that we can shutdown - shutdownInputs notifier - // ins and selectCases are correlating slices - ins []*MuxIn - selectCases []reflect.SelectCase - currentNamespace string -} - -type notifier interface { - Notify() -} - -// NewMultiplexer creates a Multiplexer and populates its Control/Completed chans -// it takes a WriteCloser, which is where in inputs will get multiplexed on to, -// and it takes a notifier, which should allow the multiplexer to ask for the shutdown -// of the inputs. -func NewMultiplexer(out io.WriteCloser, shutdownInputs notifier) *Multiplexer { - mux := &Multiplexer{ - Out: out, - Control: make(chan *MuxIn), - Completed: make(chan error), - shutdownInputs: shutdownInputs, - ins: []*MuxIn{ - nil, // There is no MuxIn for the Control case - }, - } - mux.selectCases = []reflect.SelectCase{ - { - Dir: reflect.SelectRecv, - Chan: reflect.ValueOf(mux.Control), - Send: reflect.Value{}, - }, - } - return mux -} - -// Run multiplexes until it receives an EOF on its Control chan. -func (mux *Multiplexer) Run() { - var err, completionErr error - for { - index, value, notEOF := reflect.Select(mux.selectCases) - EOF := !notEOF - if index == 0 { //Control index - if EOF { - log.Logvf(log.DebugLow, "Mux finish") - mux.Out.Close() - if completionErr != nil { - mux.Completed <- completionErr - } else if len(mux.selectCases) != 1 { - mux.Completed <- fmt.Errorf("Mux ending but selectCases still open %v", - len(mux.selectCases)) - } else { - mux.Completed <- nil - } - return - } - muxIn, ok := value.Interface().(*MuxIn) - if !ok { - mux.Completed <- fmt.Errorf("non MuxIn received on Control chan") // one for the MuxIn.Open - return - } - log.Logvf(log.DebugLow, "Mux open namespace %v", muxIn.Intent.Namespace()) - mux.selectCases = append(mux.selectCases, reflect.SelectCase{ - Dir: reflect.SelectRecv, - Chan: reflect.ValueOf(muxIn.writeChan), - Send: reflect.Value{}, - }) - mux.ins = append(mux.ins, muxIn) - } else { - if EOF { - // We need to let the MuxIn know that we've - // noticed this close. This fixes a race where - // the intent processing threads finish, then the main - // thread closes the mux's control chan and the mux - // processes the close on the control chan before it processes - // the close on the MuxIn chan - mux.ins[index].writeCloseFinishedChan <- struct{}{} - - err = mux.formatEOF(index, mux.ins[index]) - if err != nil { - mux.shutdownInputs.Notify() - mux.Out = &nopCloseNopWriter{} - completionErr = err - } - log.Logvf(log.DebugLow, "Mux close namespace %v", mux.ins[index].Intent.Namespace()) - mux.currentNamespace = "" - mux.selectCases = append(mux.selectCases[:index], mux.selectCases[index+1:]...) - mux.ins = append(mux.ins[:index], mux.ins[index+1:]...) - } else { - bsonBytes, ok := value.Interface().([]byte) - if !ok { - mux.Completed <- fmt.Errorf("multiplexer received a value that wasn't a []byte") - return - } - err = mux.formatBody(mux.ins[index], bsonBytes) - if err != nil { - mux.shutdownInputs.Notify() - mux.Out = &nopCloseNopWriter{} - completionErr = err - } - } - } - } -} - -type nopCloseNopWriter struct{} - -func (*nopCloseNopWriter) Close() error { return nil } -func (*nopCloseNopWriter) Write(p []byte) (int, error) { return len(p), nil } - -// formatBody writes the BSON in to the archive, potentially writing a new header -// if the document belongs to a different namespace from the last header. -func (mux *Multiplexer) formatBody(in *MuxIn, bsonBytes []byte) error { - var err error - var length int - defer func() { - in.writeLenChan <- length - }() - if in.Intent.Namespace() != mux.currentNamespace { - // Handle the change of which DB/Collection we're writing docs for - // If mux.currentNamespace then we need to terminate the current block - if mux.currentNamespace != "" { - l, err := mux.Out.Write(terminatorBytes) - if err != nil { - return err - } - if l != len(terminatorBytes) { - return io.ErrShortWrite - } - } - header, err := bson.Marshal(NamespaceHeader{ - Database: in.Intent.DB, - Collection: in.Intent.C, - }) - if err != nil { - return err - } - l, err := mux.Out.Write(header) - if err != nil { - return err - } - if l != len(header) { - return io.ErrShortWrite - } - } - mux.currentNamespace = in.Intent.Namespace() - length, err = mux.Out.Write(bsonBytes) - if err != nil { - return err - } - return nil -} - -// formatEOF writes the EOF header in to the archive -func (mux *Multiplexer) formatEOF(index int, in *MuxIn) error { - var err error - if mux.currentNamespace != "" { - l, err := mux.Out.Write(terminatorBytes) - if err != nil { - return err - } - if l != len(terminatorBytes) { - return io.ErrShortWrite - } - } - eofHeader, err := bson.Marshal(NamespaceHeader{ - Database: in.Intent.DB, - Collection: in.Intent.C, - EOF: true, - CRC: int64(in.hash.Sum64()), - }) - if err != nil { - return err - } - l, err := mux.Out.Write(eofHeader) - if err != nil { - return err - } - if l != len(eofHeader) { - return io.ErrShortWrite - } - l, err = mux.Out.Write(terminatorBytes) - if err != nil { - return err - } - if l != len(terminatorBytes) { - return io.ErrShortWrite - } - return nil -} - -// MuxIn is an implementation of the intents.file interface. -// They live in the intents, and are potentially owned by different threads than -// the thread owning the Multiplexer. -// They are out the intents write data to the multiplexer -type MuxIn struct { - writeChan chan []byte - writeLenChan chan int - writeCloseFinishedChan chan struct{} - buf []byte - hash hash.Hash64 - Intent *intents.Intent - Mux *Multiplexer -} - -// Read does nothing for MuxIns -func (muxIn *MuxIn) Read([]byte) (int, error) { - return 0, nil -} - -func (muxIn *MuxIn) Pos() int64 { - return 0 -} - -// Close closes the chans in the MuxIn. -// Ultimately the multiplexer will detect that they are closed and cause a -// formatEOF to occur. -func (muxIn *MuxIn) Close() error { - // the mux side of this gets closed in the mux when it gets an eof on the read - log.Logvf(log.DebugHigh, "MuxIn close %v", muxIn.Intent.Namespace()) - if bufferWrites { - muxIn.writeChan <- muxIn.buf - length := <-muxIn.writeLenChan - if length != len(muxIn.buf) { - return io.ErrShortWrite - } - muxIn.buf = nil - } - close(muxIn.writeChan) - close(muxIn.writeLenChan) - // We need to wait for the close on the writeChan to be processed before proceeding - // Otherwise we might assume that all work is finished and exit the program before - // the mux finishes writing the end of the archive - <-muxIn.writeCloseFinishedChan - return nil -} - -// Open is implemented in Mux.open, but in short, it creates chans and a select case -// and adds the SelectCase and the MuxIn in to the Multiplexer. -func (muxIn *MuxIn) Open() error { - log.Logvf(log.DebugHigh, "MuxIn open %v", muxIn.Intent.Namespace()) - muxIn.writeChan = make(chan []byte) - muxIn.writeLenChan = make(chan int) - muxIn.writeCloseFinishedChan = make(chan struct{}) - muxIn.buf = make([]byte, 0, bufferSize) - muxIn.hash = crc64.New(crc64.MakeTable(crc64.ECMA)) - if bufferWrites { - muxIn.buf = make([]byte, 0, db.MaxBSONSize) - } - muxIn.Mux.Control <- muxIn - return nil -} - -// Write hands a buffer to the Multiplexer and receives a written length from the multiplexer -// after the length is received, the buffer is free to be reused. -func (muxIn *MuxIn) Write(buf []byte) (int, error) { - size := int( - (uint32(buf[0]) << 0) | - (uint32(buf[1]) << 8) | - (uint32(buf[2]) << 16) | - (uint32(buf[3]) << 24), - ) - // TODO remove these checks, they're for debugging - if len(buf) < size { - panic(fmt.Errorf("corrupt bson in MuxIn.Write (size %v/%v)", size, len(buf))) - } - if buf[size-1] != 0 { - panic(fmt.Errorf("corrupt bson in MuxIn.Write bson has no-zero terminator %v, (size %v/%v)", buf[size-1], size, len(buf))) - } - if bufferWrites { - if len(muxIn.buf)+len(buf) > cap(muxIn.buf) { - muxIn.writeChan <- muxIn.buf - length := <-muxIn.writeLenChan - if length != len(muxIn.buf) { - return 0, io.ErrShortWrite - } - muxIn.buf = muxIn.buf[:0] - } - muxIn.buf = append(muxIn.buf, buf...) - } else { - muxIn.writeChan <- buf - length := <-muxIn.writeLenChan - if length != len(buf) { - return 0, io.ErrShortWrite - } - } - muxIn.hash.Write(buf) - return len(buf), nil -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/multiplexer_roundtrip_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/multiplexer_roundtrip_test.go deleted file mode 100644 index 3876e6fd86e..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/multiplexer_roundtrip_test.go +++ /dev/null @@ -1,244 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package archive - -import ( - "bytes" - "hash" - "hash/crc32" - "io" - "os" - "testing" - - "github.com/mongodb/mongo-tools/common/db" - "github.com/mongodb/mongo-tools/common/intents" - "github.com/mongodb/mongo-tools/common/testtype" - . "github.com/smartystreets/goconvey/convey" - "gopkg.in/mgo.v2/bson" -) - -var testIntents = []*intents.Intent{ - { - DB: "foo", - C: "bar", - Location: "foo.bar", - }, - { - DB: "ding", - C: "bats", - Location: "ding.bats", - }, - { - DB: "flim", - C: "flam.fooey", - Location: "flim.flam.fooey", - }, - { - DB: "crow", - C: "bar", - Location: "crow.bar", - }, -} - -type testDoc struct { - Bar int - Baz string -} - -type closingBuffer struct { - bytes.Buffer -} - -func (*closingBuffer) Close() error { - return nil -} - -type testNotifier struct{} - -func (n *testNotifier) Notify() {} - -func TestBasicMux(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - var err error - - Convey("with 10000 docs in each of five collections", t, func() { - buf := &closingBuffer{bytes.Buffer{}} - - mux := NewMultiplexer(buf, new(testNotifier)) - muxIns := map[string]*MuxIn{} - - inChecksum := map[string]hash.Hash{} - inLengths := map[string]*int{} - outChecksum := map[string]hash.Hash{} - outLengths := map[string]*int{} - - // To confirm that what we multiplex is the same as what we demultiplex, we - // create input and output hashes for each namespace. After we finish - // multiplexing and demultiplexing we will compare all of the CRCs for each - // namespace - errChan := make(chan error) - makeIns(testIntents, mux, inChecksum, muxIns, inLengths, errChan) - - Convey("each document should be multiplexed", func() { - go mux.Run() - - for range testIntents { - err := <-errChan - So(err, ShouldBeNil) - } - close(mux.Control) - err = <-mux.Completed - So(err, ShouldBeNil) - - demux := &Demultiplexer{ - In: buf, - NamespaceStatus: make(map[string]int), - } - demuxOuts := map[string]*RegularCollectionReceiver{} - - errChan := make(chan error) - makeOuts(testIntents, demux, outChecksum, demuxOuts, outLengths, errChan) - - Convey("and demultiplexed successfully", func() { - demux.Run() - So(err, ShouldBeNil) - - for range testIntents { - err := <-errChan - So(err, ShouldBeNil) - } - for _, dbc := range testIntents { - ns := dbc.Namespace() - So(*inLengths[ns], ShouldEqual, *outLengths[ns]) - inSum := inChecksum[ns].Sum([]byte{}) - outSum := outChecksum[ns].Sum([]byte{}) - So(inSum, ShouldResemble, outSum) - } - }) - }) - }) - return -} - -func TestParallelMux(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - Convey("parallel mux/demux over a pipe", t, func() { - readPipe, writePipe, err := os.Pipe() - So(err, ShouldBeNil) - - mux := NewMultiplexer(writePipe, new(testNotifier)) - muxIns := map[string]*MuxIn{} - - demux := &Demultiplexer{ - In: readPipe, - NamespaceStatus: make(map[string]int), - } - demuxOuts := map[string]*RegularCollectionReceiver{} - - inChecksum := map[string]hash.Hash{} - inLengths := map[string]*int{} - - outChecksum := map[string]hash.Hash{} - outLengths := map[string]*int{} - - writeErrChan := make(chan error) - readErrChan := make(chan error) - - makeIns(testIntents, mux, inChecksum, muxIns, inLengths, writeErrChan) - makeOuts(testIntents, demux, outChecksum, demuxOuts, outLengths, readErrChan) - - go demux.Run() - go mux.Run() - - for range testIntents { - err := <-writeErrChan - So(err, ShouldBeNil) - err = <-readErrChan - So(err, ShouldBeNil) - } - close(mux.Control) - muxErr := <-mux.Completed - So(muxErr, ShouldBeNil) - - for _, dbc := range testIntents { - ns := dbc.Namespace() - So(*inLengths[ns], ShouldEqual, *outLengths[ns]) - inSum := inChecksum[ns].Sum([]byte{}) - outSum := outChecksum[ns].Sum([]byte{}) - So(inSum, ShouldResemble, outSum) - } - }) - return -} - -func makeIns(testIntents []*intents.Intent, mux *Multiplexer, inChecksum map[string]hash.Hash, muxIns map[string]*MuxIn, inLengths map[string]*int, errCh chan<- error) { - for index, dbc := range testIntents { - ns := dbc.Namespace() - sum := crc32.NewIEEE() - muxIn := &MuxIn{Intent: dbc, Mux: mux} - inLength := 0 - - inChecksum[ns] = sum - muxIns[ns] = muxIn - inLengths[ns] = &inLength - - go func(index int) { - err := muxIn.Open() - if err != nil { - errCh <- err - return - } - staticBSONBuf := make([]byte, db.MaxBSONSize) - for i := 0; i < 10000; i++ { - bsonBytes, _ := bson.Marshal(testDoc{Bar: index * i, Baz: ns}) - bsonBuf := staticBSONBuf[:len(bsonBytes)] - copy(bsonBuf, bsonBytes) - muxIn.Write(bsonBuf) - sum.Write(bsonBuf) - inLength += len(bsonBuf) - } - err = muxIn.Close() - errCh <- err - }(index) - } -} - -func makeOuts(testIntents []*intents.Intent, demux *Demultiplexer, outChecksum map[string]hash.Hash, demuxOuts map[string]*RegularCollectionReceiver, outLengths map[string]*int, errCh chan<- error) { - for _, dbc := range testIntents { - ns := dbc.Namespace() - sum := crc32.NewIEEE() - muxOut := &RegularCollectionReceiver{ - Intent: dbc, - Demux: demux, - Origin: ns, - } - outLength := 0 - - outChecksum[ns] = sum - demuxOuts[ns] = muxOut - outLengths[ns] = &outLength - - demuxOuts[ns].Open() - go func() { - bs := make([]byte, db.MaxBSONSize) - var err error - for { - var length int - length, err = muxOut.Read(bs) - if err != nil { - break - } - sum.Write(bs[:length]) - outLength += len(bs[:length]) - } - if err == io.EOF { - err = nil - } - errCh <- err - }() - } -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/parser.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/parser.go deleted file mode 100644 index eb53d4f0774..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/parser.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package archive - -import ( - "fmt" - "io" - - "github.com/mongodb/mongo-tools/common/db" -) - -// parser.go implements the parsing of the low-level archive format -// The low level archive format is defined as zero or more blocks -// where each block is defined as: -// a header BSON document -// zero or more body BSON documents -// a four byte terminator (0xFFFFFFFF) - -// ParserConsumer is the interface that one needs to implement to consume data from the Parser -type ParserConsumer interface { - HeaderBSON([]byte) error - BodyBSON([]byte) error - End() error -} - -// Parser encapsulates the small amount of state that the parser needs to keep -type Parser struct { - In io.Reader - buf [db.MaxBSONSize]byte - length int -} - -type parserError struct { - Err error - Msg string -} - -// Error is part of the Error interface. It formats a parserError for human readability. -func (pe *parserError) Error() string { - err := fmt.Sprintf("corruption found in archive; %v", pe.Msg) - if pe.Err != nil { - err = fmt.Sprintf("%v ( %v )", err, pe.Err) - } - return err -} - -// newParserError creates a parserError with just a message -func newParserError(msg string) error { - return &parserError{ - Msg: msg, - } -} - -// newParserWrappedError creates a parserError with a message as well as an underlying cause error -func newParserWrappedError(msg string, err error) error { - return &parserError{ - Err: err, - Msg: msg, - } -} - -// readBSONOrTerminator reads at least four bytes, determines -// if the first four bytes are a terminator, a bson length, or something else. -// If they are a terminator, true,nil are returned. If they are a BSON length, -// then the remainder of the BSON document are read in to the parser, otherwise -// an error is returned. -func (parse *Parser) readBSONOrTerminator() (isTerminator bool, err error) { - parse.length = 0 - _, err = io.ReadFull(parse.In, parse.buf[0:4]) - if err == io.EOF { - return false, err - } - if err != nil { - return false, newParserWrappedError("I/O error reading length or terminator", err) - } - size := int32( - (uint32(parse.buf[0]) << 0) | - (uint32(parse.buf[1]) << 8) | - (uint32(parse.buf[2]) << 16) | - (uint32(parse.buf[3]) << 24), - ) - if size == terminator { - return true, nil - } - if size < minBSONSize || size > db.MaxBSONSize { - return false, newParserError(fmt.Sprintf("%v is neither a valid bson length nor a archive terminator", size)) - } - // TODO Because we're reusing this same buffer for all of our IO, we are basically guaranteeing that we'll - // copy the bytes twice. At some point we should fix this. It's slightly complex, because we'll need consumer - // methods closing one buffer and acquiring another - _, err = io.ReadFull(parse.In, parse.buf[4:size]) - if err != nil { - // any error, including EOF is an error so we wrap it up - return false, newParserWrappedError("read bson", err) - } - if parse.buf[size-1] != 0x00 { - return false, newParserError(fmt.Sprintf("bson (size: %v, byte: %d) doesn't end with a null byte", size, parse.buf[size-1])) - } - parse.length = int(size) - return false, nil -} - -// ReadAllBlocks calls ReadBlock() until it returns an error. -// If the error is EOF, then nil is returned, otherwise it returns the error -func (parse *Parser) ReadAllBlocks(consumer ParserConsumer) (err error) { - for err == nil { - err = parse.ReadBlock(consumer) - } - endError := consumer.End() - if err == io.EOF { - return endError - } - return err -} - -// ReadBlock reads one archive block ( header + body* + terminator ) -// calling consumer.HeaderBSON() on the header, consumer.BodyBSON() on each piece of body, -// and consumer.EOF() when EOF is encountered before any data was read. -// It returns nil if a whole block was read, io.EOF if nothing was read, -// and a parserError if there was any io error in the middle of the block, -// if either of the consumer methods return error, or if there was any sort of -// parsing failure. -func (parse *Parser) ReadBlock(consumer ParserConsumer) (err error) { - isTerminator, err := parse.readBSONOrTerminator() - if err != nil { - return err - } - if isTerminator { - return newParserError("consecutive terminators / headerless blocks are not allowed") - } - err = consumer.HeaderBSON(parse.buf[:parse.length]) - if err != nil { - return newParserWrappedError("ParserConsumer.HeaderBSON()", err) - } - for { - isTerminator, err = parse.readBSONOrTerminator() - if err != nil { // all errors, including EOF are errors here - return newParserWrappedError("ParserConsumer.BodyBSON()", err) - } - if isTerminator { - return nil - } - err = consumer.BodyBSON(parse.buf[:parse.length]) - if err != nil { - return newParserWrappedError("ParserConsumer.BodyBSON()", err) - } - } -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/parser_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/parser_test.go deleted file mode 100644 index ef92ac0922f..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/parser_test.go +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package archive - -import ( - "bytes" - "fmt" - "io" - "testing" - - "github.com/mongodb/mongo-tools/common/testtype" - . "github.com/smartystreets/goconvey/convey" - "gopkg.in/mgo.v2/bson" -) - -type testConsumer struct { - headers []string // header data - bodies []string // body data - eof bool -} - -func (tc *testConsumer) HeaderBSON(b []byte) error { - ss := strStruct{} - err := bson.Unmarshal(b, &ss) - tc.headers = append(tc.headers, ss.Str) - return err -} - -func (tc *testConsumer) BodyBSON(b []byte) error { - ss := strStruct{} - err := bson.Unmarshal(b, &ss) - tc.bodies = append(tc.bodies, ss.Str) - return err -} - -func (tc *testConsumer) End() (err error) { - if tc.eof { - err = fmt.Errorf("double end") - } - tc.eof = true - return err -} - -type strStruct struct { - Str string -} - -var term = []byte{0xFF, 0xFF, 0xFF, 0xFF} -var notTerm = []byte{0xFF, 0xFF, 0xFF, 0xFE} - -func TestParsing(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("With a parser with a simple parser consumer", t, func() { - tc := &testConsumer{} - parser := Parser{} - Convey("a well formed header and body", func() { - buf := bytes.Buffer{} - b, _ := bson.Marshal(strStruct{"header"}) - buf.Write(b) - b, _ = bson.Marshal(strStruct{"body"}) - buf.Write(b) - buf.Write(term) - parser.In = &buf - Convey("ReadBlock data parses correctly", func() { - err := parser.ReadBlock(tc) - So(err, ShouldBeNil) - So(tc.eof, ShouldBeFalse) - So(tc.headers[0], ShouldEqual, "header") - So(tc.bodies[0], ShouldEqual, "body") - - err = parser.ReadBlock(tc) - So(err, ShouldEqual, io.EOF) - }) - Convey("ReadAllBlock data parses correctly", func() { - err := parser.ReadAllBlocks(tc) - So(err, ShouldEqual, nil) - So(tc.eof, ShouldBeTrue) - So(tc.headers[0], ShouldEqual, "header") - So(tc.bodies[0], ShouldEqual, "body") - - }) - }) - Convey("a well formed header and multiple body datas parse correctly", func() { - buf := bytes.Buffer{} - b, _ := bson.Marshal(strStruct{"header"}) - buf.Write(b) - b, _ = bson.Marshal(strStruct{"body0"}) - buf.Write(b) - b, _ = bson.Marshal(strStruct{"body1"}) - buf.Write(b) - b, _ = bson.Marshal(strStruct{"body2"}) - buf.Write(b) - buf.Write(term) - parser.In = &buf - err := parser.ReadBlock(tc) - So(err, ShouldBeNil) - So(tc.eof, ShouldBeFalse) - So(tc.headers[0], ShouldEqual, "header") - So(tc.bodies[0], ShouldEqual, "body0") - So(tc.bodies[1], ShouldEqual, "body1") - So(tc.bodies[2], ShouldEqual, "body2") - - err = parser.ReadBlock(tc) - So(err, ShouldEqual, io.EOF) - So(tc.eof, ShouldBeFalse) - }) - Convey("an incorrect terminator should cause an error", func() { - buf := bytes.Buffer{} - b, _ := bson.Marshal(strStruct{"header"}) - buf.Write(b) - b, _ = bson.Marshal(strStruct{"body"}) - buf.Write(b) - buf.Write(notTerm) - parser.In = &buf - err := parser.ReadBlock(tc) - So(err, ShouldNotBeNil) - }) - Convey("an empty block should result in EOF", func() { - buf := bytes.Buffer{} - parser.In = &buf - err := parser.ReadBlock(tc) - So(err, ShouldEqual, io.EOF) - So(tc.eof, ShouldBeFalse) - }) - Convey("an error comming from the consumer should propigate through the parser", func() { - tc.eof = true - buf := bytes.Buffer{} - parser.In = &buf - err := parser.ReadAllBlocks(tc) - So(err.Error(), ShouldContainSubstring, "double end") - }) - Convey("a partial block should result in a non-EOF error", func() { - buf := bytes.Buffer{} - b, _ := bson.Marshal(strStruct{"header"}) - buf.Write(b) - b, _ = bson.Marshal(strStruct{"body"}) - buf.Write(b) - parser.In = &buf - err := parser.ReadBlock(tc) - So(err, ShouldNotBeNil) - So(tc.eof, ShouldBeFalse) - So(tc.headers[0], ShouldEqual, "header") - So(tc.bodies[0], ShouldEqual, "body") - }) - Convey("a block with a missing terminator shoud result in a non-EOF error", func() { - buf := bytes.Buffer{} - b, _ := bson.Marshal(strStruct{"header"}) - buf.Write(b) - b, _ = bson.Marshal(strStruct{"body"}) - buf.Write(b[:len(b)-1]) - buf.Write([]byte{0x01}) - buf.Write(notTerm) - parser.In = &buf - err := parser.ReadBlock(tc) - So(err, ShouldNotBeNil) - So(tc.eof, ShouldBeFalse) - So(tc.headers[0], ShouldEqual, "header") - So(tc.bodies, ShouldBeNil) - }) - }) - return -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/prelude.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/prelude.go deleted file mode 100644 index 2d50bc66dce..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/prelude.go +++ /dev/null @@ -1,397 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package archive - -import ( - "bytes" - "fmt" - "io" - "path/filepath" - "sync/atomic" - - "github.com/mongodb/mongo-tools/common" - "github.com/mongodb/mongo-tools/common/intents" - "github.com/mongodb/mongo-tools/common/log" - "github.com/mongodb/mongo-tools/common/options" - "github.com/mongodb/mongo-tools/common/util" - "gopkg.in/mgo.v2/bson" -) - -//MetadataFile implements intents.file -type MetadataFile struct { - pos int64 // updated atomically, aligned at the beginning of the struct - *bytes.Buffer - Intent *intents.Intent -} - -func (md *MetadataFile) Open() error { - return nil -} -func (md *MetadataFile) Close() error { - return nil -} - -func (md *MetadataFile) Read(p []byte) (int, error) { - n, err := md.Buffer.Read(p) - atomic.AddInt64(&md.pos, int64(n)) - return n, err -} - -func (md *MetadataFile) Pos() int64 { - return atomic.LoadInt64(&md.pos) -} - -// DirLike represents the group of methods done on directories and files in dump directories, -// or in archives, when mongorestore is figuring out what intents to create. -type DirLike interface { - Name() string - Path() string - Size() int64 - IsDir() bool - Stat() (DirLike, error) - ReadDir() ([]DirLike, error) - Parent() DirLike -} - -// Prelude represents the knowledge gleaned from reading the prelude out of the archive. -type Prelude struct { - Header *Header - DBS []string - NamespaceMetadatas []*CollectionMetadata - NamespaceMetadatasByDB map[string][]*CollectionMetadata -} - -// Read consumes and checks the magic number at the beginning of the archive, -// then it runs the parser with a Prelude as its consumer. -func (prelude *Prelude) Read(in io.Reader) error { - readMagicNumberBuf := make([]byte, 4) - _, err := io.ReadAtLeast(in, readMagicNumberBuf, 4) - if err != nil { - return fmt.Errorf("I/O failure reading beginning of archive: %v", err) - } - readMagicNumber := uint32( - (uint32(readMagicNumberBuf[0]) << 0) | - (uint32(readMagicNumberBuf[1]) << 8) | - (uint32(readMagicNumberBuf[2]) << 16) | - (uint32(readMagicNumberBuf[3]) << 24), - ) - - if readMagicNumber != MagicNumber { - return fmt.Errorf("stream or file does not appear to be a mongodump archive") - } - - if prelude.NamespaceMetadatasByDB != nil { - prelude.NamespaceMetadatasByDB = make(map[string][]*CollectionMetadata, 0) - } - - parser := Parser{In: in} - parserConsumer := &preludeParserConsumer{prelude: prelude} - return parser.ReadBlock(parserConsumer) -} - -// NewPrelude generates a Prelude using the contents of an intent.Manager. -func NewPrelude(manager *intents.Manager, concurrentColls int, serverVersion string) (*Prelude, error) { - prelude := Prelude{ - Header: &Header{ - FormatVersion: archiveFormatVersion, - ServerVersion: serverVersion, - ToolVersion: options.VersionStr, - ConcurrentCollections: int32(concurrentColls), - }, - NamespaceMetadatasByDB: make(map[string][]*CollectionMetadata, 0), - } - allIntents := manager.Intents() - for _, intent := range allIntents { - if intent.MetadataFile != nil { - archiveMetadata, ok := intent.MetadataFile.(*MetadataFile) - if !ok { - return nil, fmt.Errorf("MetadataFile is not an archive.Metadata") - } - prelude.AddMetadata(&CollectionMetadata{ - Database: intent.DB, - Collection: intent.C, - Metadata: archiveMetadata.Buffer.String(), - }) - } else { - prelude.AddMetadata(&CollectionMetadata{ - Database: intent.DB, - Collection: intent.C, - }) - } - } - return &prelude, nil -} - -// AddMetadata adds a metadata data structure to a prelude and does the required bookkeeping. -func (prelude *Prelude) AddMetadata(cm *CollectionMetadata) { - prelude.NamespaceMetadatas = append(prelude.NamespaceMetadatas, cm) - if prelude.NamespaceMetadatasByDB == nil { - prelude.NamespaceMetadatasByDB = make(map[string][]*CollectionMetadata) - } - _, ok := prelude.NamespaceMetadatasByDB[cm.Database] - if !ok { - prelude.DBS = append(prelude.DBS, cm.Database) - } - prelude.NamespaceMetadatasByDB[cm.Database] = append(prelude.NamespaceMetadatasByDB[cm.Database], cm) - log.Logvf(log.Info, "archive prelude %v.%v", cm.Database, cm.Collection) -} - -// Write writes the archive header. -func (prelude *Prelude) Write(out io.Writer) error { - magicNumberBytes := make([]byte, 4) - for i := range magicNumberBytes { - magicNumberBytes[i] = byte(uint32(MagicNumber) >> uint(i*8)) - } - _, err := out.Write(magicNumberBytes) - if err != nil { - return err - } - buf, err := bson.Marshal(prelude.Header) - if err != nil { - return err - } - _, err = out.Write(buf) - if err != nil { - return err - } - for _, cm := range prelude.NamespaceMetadatas { - buf, err = bson.Marshal(cm) - if err != nil { - return err - } - _, err = out.Write(buf) - if err != nil { - return err - } - } - _, err = out.Write(terminatorBytes) - if err != nil { - return err - } - return nil -} - -// preludeParserConsumer wraps a Prelude, and implements ParserConsumer. -type preludeParserConsumer struct { - prelude *Prelude -} - -// HeaderBSON is part of the ParserConsumer interface, it unmarshals archive Headers. -func (hpc *preludeParserConsumer) HeaderBSON(data []byte) error { - hpc.prelude.Header = &Header{} - err := bson.Unmarshal(data, hpc.prelude.Header) - if err != nil { - return err - } - return nil -} - -// BodyBSON is part of the ParserConsumer interface, it unmarshals CollectionMetadata's. -func (hpc *preludeParserConsumer) BodyBSON(data []byte) error { - cm := &CollectionMetadata{} - err := bson.Unmarshal(data, cm) - if err != nil { - return err - } - cm.Collection, err = util.UnescapeCollectionName(cm.Collection) - if err != nil { - return err - } - - hpc.prelude.AddMetadata(cm) - return nil -} - -// BodyBSON is part of the ParserConsumer interface. -func (hpc *preludeParserConsumer) End() error { - return nil -} - -// PreludeExplorer implements DirLike. PreludeExplorer represent the databases, collections, -// and their metadata json files, of an archive, in such a way that they can be explored like a filesystem. -type PreludeExplorer struct { - prelude *Prelude - database string - collection string - isMetadata bool -} - -// NewPreludeExplorer creates a PreludeExplorer from a Prelude. -func (prelude *Prelude) NewPreludeExplorer() (*PreludeExplorer, error) { - pe := &PreludeExplorer{ - prelude: prelude, - } - return pe, nil -} - -// Name is part of the DirLike interface. It synthesizes a filename for the given "location" the prelude. -func (pe *PreludeExplorer) Name() string { - if pe.collection == "" { - return pe.database - } - if pe.isMetadata { - return util.EscapeCollectionName(pe.collection) + ".metadata.json" - } - return util.EscapeCollectionName(pe.collection) + ".bson" -} - -// Path is part of the DirLike interface. It creates the full path for the "location" in the prelude. -func (pe *PreludeExplorer) Path() string { - if pe.collection == "" { - return pe.database - } - if pe.database == "" { - return pe.Name() - } - return filepath.Join(pe.database, pe.Name()) -} - -// Size is part of the DirLike interface. It returns the size from the metadata -// of the prelude, if the "location" is a collection. -func (pe *PreludeExplorer) Size() int64 { - if pe.IsDir() { - return 0 - } - for _, ns := range pe.prelude.NamespaceMetadatas { - if ns.Database == pe.database && ns.Collection == pe.collection { - return int64(ns.Size) - } - } - return 0 -} - -// IsDir is part of the DirLike interface. All pes that are not collections are Dirs. -func (pe *PreludeExplorer) IsDir() bool { - return pe.collection == "" -} - -// Stat is part of the DirLike interface. os.Stat returns a FileInfo, and since -// DirLike is similar to FileInfo, we just return the pe, here. -func (pe *PreludeExplorer) Stat() (DirLike, error) { - return pe, nil -} - -// ReadDir is part of the DirLIke interface. ReadDir generates a list of PreludeExplorers -// whose "locations" are encapsulated by the current pes "location". -// -// "dump/oplog.bson" => &PreludeExplorer{ database: "", collection: "oplog.bson" } -// "dump/test/" => &PreludeExplorer{ database: "test", collection: "foo.bson" } -// "dump/test/foo.bson" => &PreludeExplorer{ database: "test", collection: "" } -// "dump/test/foo.json" => &PreludeExplorer{ database: "test", collection: "foo", isMetadata: true } -// -func (pe *PreludeExplorer) ReadDir() ([]DirLike, error) { - if !pe.IsDir() { - return nil, fmt.Errorf("not a directory") - } - pes := []DirLike{} - if pe.database == "" { - // when reading the top level of the archive, we need return all of the - // collections that are not bound to a database, aka, the oplog, and then all of - // the databases the prelude stores all top-level collections as collections in - // the "" database - topLevelNamespaceMetadatas, ok := pe.prelude.NamespaceMetadatasByDB[""] - if ok { - for _, topLevelNamespaceMetadata := range topLevelNamespaceMetadatas { - pes = append(pes, &PreludeExplorer{ - prelude: pe.prelude, - collection: topLevelNamespaceMetadata.Collection, - }) - if topLevelNamespaceMetadata.Metadata != "" { - pes = append(pes, &PreludeExplorer{ - prelude: pe.prelude, - collection: topLevelNamespaceMetadata.Collection, - isMetadata: true, - }) - } - } - } - for _, db := range pe.prelude.DBS { - pes = append(pes, &PreludeExplorer{ - prelude: pe.prelude, - database: db, - }) - } - } else { - // when reading the contents of a database directory, we just return all of the bson and - // json files for all of the collections bound to that database - namespaceMetadatas, ok := pe.prelude.NamespaceMetadatasByDB[pe.database] - if !ok { - return nil, fmt.Errorf("no such directory") //TODO: replace with real ERRNOs? - } - for _, namespaceMetadata := range namespaceMetadatas { - pes = append(pes, &PreludeExplorer{ - prelude: pe.prelude, - database: pe.database, - collection: namespaceMetadata.Collection, - }) - if namespaceMetadata.Metadata != "" { - pes = append(pes, &PreludeExplorer{ - prelude: pe.prelude, - database: pe.database, - collection: namespaceMetadata.Collection, - isMetadata: true, - }) - } - } - } - return pes, nil -} - -// Parent is part of the DirLike interface. It returns a pe without a collection, if there is one, -// otherwise, without a database. -func (pe *PreludeExplorer) Parent() DirLike { - if pe.collection != "" { - return &PreludeExplorer{ - prelude: pe.prelude, - database: pe.database, - } - } - return &PreludeExplorer{ - prelude: pe.prelude, - } -} - -// MetadataPreludeFile is part of the intents.file. It allows the metadata contained in the prelude to be opened and read -type MetadataPreludeFile struct { - pos int64 // updated atomically, aligned at the beginning of the struct - Intent *intents.Intent - Origin string - Prelude *Prelude - *bytes.Buffer -} - -// Open is part of the intents.file interface, it finds the metadata in the prelude and creates a bytes.Buffer from it. -func (mpf *MetadataPreludeFile) Open() error { - db, c := common.SplitNamespace(mpf.Origin) - dbMetadatas, ok := mpf.Prelude.NamespaceMetadatasByDB[db] - if !ok { - return fmt.Errorf("no metadata found for '%s'", db) - } - for _, metadata := range dbMetadatas { - if metadata.Collection == c { - mpf.Buffer = bytes.NewBufferString(metadata.Metadata) - return nil - } - } - return fmt.Errorf("no matching metadata found for '%s'", mpf.Origin) -} - -// Close is part of the intents.file interface. -func (mpf *MetadataPreludeFile) Close() error { - mpf.Buffer = nil - return nil -} - -func (mpf *MetadataPreludeFile) Read(p []byte) (int, error) { - n, err := mpf.Buffer.Read(p) - atomic.AddInt64(&mpf.pos, int64(n)) - return n, err -} - -func (mpf *MetadataPreludeFile) Pos() int64 { - return atomic.LoadInt64(&mpf.pos) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/prelude_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/prelude_test.go deleted file mode 100644 index b04e9e69f5f..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/archive/prelude_test.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package archive - -import ( - "bytes" - - . "github.com/smartystreets/goconvey/convey" - // "gopkg.in/mgo.v2/bson" - "github.com/mongodb/mongo-tools/common/testtype" - "testing" -) - -func TestPrelude(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - var err error - - Convey("WritePrelude/ReadPrelude roundtrip", t, func() { - - cm1 := &CollectionMetadata{ - Database: "db1", - Collection: "c1", - Metadata: "m1", - } - cm2 := &CollectionMetadata{ - Database: "db1", - Collection: "c2", - Metadata: "m2", - } - cm3 := &CollectionMetadata{ - Database: "db2", - Collection: "c3", - Metadata: "m3", - } - cm4 := &CollectionMetadata{ - Database: "db3", - Collection: "c4", - Metadata: "m4", - } - - archivePrelude := &Prelude{ - Header: &Header{ - FormatVersion: "version-foo", - }, - NamespaceMetadatas: []*CollectionMetadata{cm1, cm2, cm3, cm4}, - DBS: []string{"db1", "db2", "db3"}, - NamespaceMetadatasByDB: map[string][]*CollectionMetadata{ - "db1": {cm1, cm2}, - "db2": {cm3}, - "db3": {cm4}, - }, - } - buf := &bytes.Buffer{} - err = archivePrelude.Write(buf) - So(err, ShouldBeNil) - archivePrelude2 := &Prelude{} - err := archivePrelude2.Read(buf) - So(err, ShouldBeNil) - So(archivePrelude2, ShouldResemble, archivePrelude) - }) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/auth/auth_info.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/auth/auth_info.go deleted file mode 100644 index 41ad68cb3c3..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/auth/auth_info.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -// Package auth provides utilities for performing tasks related to authentication. -package auth - -import ( - "fmt" - "strings" - - "github.com/mongodb/mongo-tools/common/db" - "gopkg.in/mgo.v2/bson" -) - -// GetAuthVersion gets the authentication schema version of the connected server -// and returns that value as an integer along with any error that occurred. -func GetAuthVersion(commander db.CommandRunner) (int, error) { - results := bson.M{} - err := commander.Run( - bson.D{ - {"getParameter", 1}, - {"authSchemaVersion", 1}, - }, - &results, - "admin", - ) - - if err != nil { - errMessage := err.Error() - // as a necessary hack, if the error message takes a certain form, - // we can infer version 1. This is because early versions of mongodb - // had no concept of an "auth schema version", so asking for the - // authSchemaVersion value will return a "no option found" or "no such cmd" - if errMessage == "no option found to get" || - strings.Contains(errMessage, "no such cmd") { - return 1, nil - } - // otherwise it's a connection error, so bubble it up - return 0, err - } - - version, ok := results["authSchemaVersion"].(int) - if !ok { - // very unlikely this will ever happen - return 0, fmt.Errorf( - "getParameter command returned non-numeric result: %v", - results["authSchemaVersion"]) - } - return version, nil -} - -// VerifySystemAuthVersion returns an error if authentication is not set up for -// the given server. -func VerifySystemAuthVersion(sessionProvider *db.SessionProvider) error { - session, err := sessionProvider.GetSession() - if err != nil { - return fmt.Errorf("error getting session from server: %v", err) - } - defer session.Close() - - authSchemaQuery := bson.M{"_id": "authSchema"} - versionEntries := session.DB("admin").C("system.version").Find(authSchemaQuery) - if count, err := versionEntries.Count(); err != nil { - return fmt.Errorf("error checking pressence of auth version: %v", err) - } else if count == 0 { - return fmt.Errorf("found no auth version") - } - return nil -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/bsonutil.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/bsonutil.go deleted file mode 100644 index 3bf5e6ba056..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/bsonutil.go +++ /dev/null @@ -1,422 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -// Package bsonutil provides utilities for processing BSON data. -package bsonutil - -import ( - "encoding/base64" - "encoding/hex" - "errors" - "fmt" - "github.com/mongodb/mongo-tools/common/json" - "github.com/mongodb/mongo-tools/common/util" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" - "strconv" - "time" -) - -var ErrNoSuchField = errors.New("no such field") - -// ConvertJSONDocumentToBSON iterates through the document map and converts JSON -// values to their corresponding BSON values. It also replaces any extended JSON -// type value (e.g. $date) with the corresponding BSON type. -func ConvertJSONDocumentToBSON(doc map[string]interface{}) error { - for key, jsonValue := range doc { - var bsonValue interface{} - var err error - - switch v := jsonValue.(type) { - case map[string]interface{}, bson.D: // subdocument - bsonValue, err = ParseSpecialKeys(v) - default: - bsonValue, err = ConvertJSONValueToBSON(v) - } - if err != nil { - return err - } - - doc[key] = bsonValue - } - return nil -} - -// GetExtendedBsonD iterates through the document and returns a bson.D that adds type -// information for each key in document. -func GetExtendedBsonD(doc bson.D) (bson.D, error) { - var err error - var bsonDoc bson.D - for _, docElem := range doc { - var bsonValue interface{} - switch v := docElem.Value.(type) { - case map[string]interface{}, bson.D: // subdocument - bsonValue, err = ParseSpecialKeys(v) - default: - bsonValue, err = ConvertJSONValueToBSON(v) - } - if err != nil { - return nil, err - } - bsonDoc = append(bsonDoc, bson.DocElem{ - Name: docElem.Name, - Value: bsonValue, - }) - } - return bsonDoc, nil -} - -// FindValueByKey returns the value of keyName in document. If keyName is not found -// in the top-level of the document, ErrNoSuchField is returned as the error. -func FindValueByKey(keyName string, document *bson.D) (interface{}, error) { - for _, key := range *document { - if key.Name == keyName { - return key.Value, nil - } - } - return nil, ErrNoSuchField -} - -// ParseSpecialKeys takes a JSON document and inspects it for any extended JSON -// type (e.g $numberLong) and replaces any such values with the corresponding -// BSON type. -func ParseSpecialKeys(special interface{}) (interface{}, error) { - // first ensure we are using a correct document type - var doc map[string]interface{} - switch v := special.(type) { - case bson.D: - doc = v.Map() - case map[string]interface{}: - doc = v - default: - return nil, fmt.Errorf("%v (type %T) is not valid input to ParseSpecialKeys", special, special) - } - // check document to see if it is special - switch len(doc) { - case 1: // document has a single field - if jsonValue, ok := doc["$date"]; ok { - switch v := jsonValue.(type) { - case string: - return util.FormatDate(v) - case bson.D: - asMap := v.Map() - if jsonValue, ok := asMap["$numberLong"]; ok { - n, err := parseNumberLongField(jsonValue) - if err != nil { - return nil, err - } - return time.Unix(n/1e3, n%1e3*1e6), err - } - return nil, errors.New("expected $numberLong field in $date") - case map[string]interface{}: - if jsonValue, ok := v["$numberLong"]; ok { - n, err := parseNumberLongField(jsonValue) - if err != nil { - return nil, err - } - return time.Unix(n/1e3, n%1e3*1e6), err - } - return nil, errors.New("expected $numberLong field in $date") - - case json.Number: - n, err := v.Int64() - return time.Unix(n/1e3, n%1e3*1e6), err - case float64: - n := int64(v) - return time.Unix(n/1e3, n%1e3*1e6), nil - case int32: - n := int64(v) - return time.Unix(n/1e3, n%1e3*1e6), nil - case int64: - return time.Unix(v/1e3, v%1e3*1e6), nil - - case json.ISODate: - return v, nil - - default: - return nil, errors.New("invalid type for $date field") - } - } - - if jsonValue, ok := doc["$code"]; ok { - switch v := jsonValue.(type) { - case string: - return bson.JavaScript{Code: v}, nil - default: - return nil, errors.New("expected $code field to have string value") - } - } - - if jsonValue, ok := doc["$oid"]; ok { - switch v := jsonValue.(type) { - case string: - if !bson.IsObjectIdHex(v) { - return nil, errors.New("expected $oid field to contain 24 hexadecimal character") - } - return bson.ObjectIdHex(v), nil - - default: - return nil, errors.New("expected $oid field to have string value") - } - } - - if jsonValue, ok := doc["$numberLong"]; ok { - return parseNumberLongField(jsonValue) - } - - if jsonValue, ok := doc["$numberInt"]; ok { - switch v := jsonValue.(type) { - case string: - // all of decimal, hex, and octal are supported here - n, err := strconv.ParseInt(v, 0, 32) - return int32(n), err - - default: - return nil, errors.New("expected $numberInt field to have string value") - } - } - - if jsonValue, ok := doc["$timestamp"]; ok { - ts := json.Timestamp{} - - var tsDoc map[string]interface{} - switch internalDoc := jsonValue.(type) { - case map[string]interface{}: - tsDoc = internalDoc - case bson.D: - tsDoc = internalDoc.Map() - default: - return nil, errors.New("expected $timestamp key to have internal document") - } - - if seconds, ok := tsDoc["t"]; ok { - if asUint32, err := util.ToUInt32(seconds); err == nil { - ts.Seconds = asUint32 - } else { - return nil, errors.New("expected $timestamp 't' field to be a numeric type") - } - } else { - return nil, errors.New("expected $timestamp to have 't' field") - } - if inc, ok := tsDoc["i"]; ok { - if asUint32, err := util.ToUInt32(inc); err == nil { - ts.Increment = asUint32 - } else { - return nil, errors.New("expected $timestamp 'i' field to be a numeric type") - } - } else { - return nil, errors.New("expected $timestamp to have 'i' field") - } - // see BSON spec for details on the bit fiddling here - return bson.MongoTimestamp(int64(ts.Seconds)<<32 | int64(ts.Increment)), nil - } - - if jsonValue, ok := doc["$numberDecimal"]; ok { - switch v := jsonValue.(type) { - case string: - return bson.ParseDecimal128(v) - default: - return nil, errors.New("expected $numberDecimal field to have string value") - } - } - - if _, ok := doc["$undefined"]; ok { - return bson.Undefined, nil - } - - if _, ok := doc["$maxKey"]; ok { - return bson.MaxKey, nil - } - - if _, ok := doc["$minKey"]; ok { - return bson.MinKey, nil - } - - case 2: // document has two fields - if jsonValue, ok := doc["$code"]; ok { - code := bson.JavaScript{} - switch v := jsonValue.(type) { - case string: - code.Code = v - default: - return nil, errors.New("expected $code field to have string value") - } - - if jsonValue, ok = doc["$scope"]; ok { - switch v2 := jsonValue.(type) { - case map[string]interface{}, bson.D: - x, err := ParseSpecialKeys(v2) - if err != nil { - return nil, err - } - code.Scope = x - return code, nil - default: - return nil, errors.New("expected $scope field to contain map") - } - } else { - return nil, errors.New("expected $scope field with $code field") - } - } - - if jsonValue, ok := doc["$regex"]; ok { - regex := bson.RegEx{} - - switch pattern := jsonValue.(type) { - case string: - regex.Pattern = pattern - - default: - return nil, errors.New("expected $regex field to have string value") - } - if jsonValue, ok = doc["$options"]; !ok { - return nil, errors.New("expected $options field with $regex field") - } - - switch options := jsonValue.(type) { - case string: - regex.Options = options - - default: - return nil, errors.New("expected $options field to have string value") - } - - // Validate regular expression options - for i := range regex.Options { - switch o := regex.Options[i]; o { - default: - return nil, fmt.Errorf("invalid regular expression option '%v'", o) - - case 'g', 'i', 'm', 's': // allowed - } - } - return regex, nil - } - - if jsonValue, ok := doc["$binary"]; ok { - binary := bson.Binary{} - - switch data := jsonValue.(type) { - case string: - bytes, err := base64.StdEncoding.DecodeString(data) - if err != nil { - return nil, err - } - binary.Data = bytes - - default: - return nil, errors.New("expected $binary field to have string value") - } - if jsonValue, ok = doc["$type"]; !ok { - return nil, errors.New("expected $type field with $binary field") - } - - switch typ := jsonValue.(type) { - case string: - kind, err := hex.DecodeString(typ) - if err != nil { - return nil, err - } else if len(kind) != 1 { - return nil, errors.New("expected single byte (as hexadecimal string) for $type field") - } - binary.Kind = kind[0] - - default: - return nil, errors.New("expected $type field to have string value") - } - return binary, nil - } - - if jsonValue, ok := doc["$ref"]; ok { - dbRef := mgo.DBRef{} - - switch data := jsonValue.(type) { - case string: - dbRef.Collection = data - default: - return nil, errors.New("expected string for $ref field") - } - if jsonValue, ok = doc["$id"]; ok { - switch v2 := jsonValue.(type) { - case map[string]interface{}, bson.D: - x, err := ParseSpecialKeys(v2) - if err != nil { - return nil, fmt.Errorf("error parsing $id field: %v", err) - } - dbRef.Id = x - default: - dbRef.Id = v2 - } - return dbRef, nil - } - } - case 3: - if jsonValue, ok := doc["$ref"]; ok { - dbRef := mgo.DBRef{} - - switch data := jsonValue.(type) { - case string: - dbRef.Collection = data - default: - return nil, errors.New("expected string for $ref field") - } - if jsonValue, ok = doc["$id"]; ok { - switch v2 := jsonValue.(type) { - case map[string]interface{}, bson.D: - x, err := ParseSpecialKeys(v2) - if err != nil { - return nil, fmt.Errorf("error parsing $id field: %v", err) - } - dbRef.Id = x - default: - dbRef.Id = v2 - } - if dbValue, ok := doc["$db"]; ok { - switch v3 := dbValue.(type) { - case string: - dbRef.Database = v3 - default: - return nil, errors.New("expected string for $db field") - } - return dbRef, nil - } - } - } - } - - // nothing matched, so we recurse deeper - switch v := special.(type) { - case bson.D: - return GetExtendedBsonD(v) - case map[string]interface{}: - return ConvertJSONValueToBSON(v) - default: - return nil, fmt.Errorf("%v (type %T) is not valid input to ParseSpecialKeys", special, special) - } -} - -// ParseJSONValue takes any value generated by the json package and returns a -// BSON version of that value. -func ParseJSONValue(jsonValue interface{}) (interface{}, error) { - switch v := jsonValue.(type) { - case map[string]interface{}, bson.D: // subdocument - return ParseSpecialKeys(v) - - default: - return ConvertJSONValueToBSON(v) - } -} - -func parseNumberLongField(jsonValue interface{}) (int64, error) { - switch v := jsonValue.(type) { - case string: - // all of decimal, hex, and octal are supported here - return strconv.ParseInt(v, 0, 64) - - default: - return 0, errors.New("expected $numberLong field to have string value") - } -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/converter.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/converter.go deleted file mode 100644 index 3a43b430a74..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/converter.go +++ /dev/null @@ -1,394 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package bsonutil - -import ( - "encoding/base64" - "errors" - "fmt" - "github.com/mongodb/mongo-tools/common/json" - "github.com/mongodb/mongo-tools/common/util" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" - "time" -) - -// ConvertJSONValueToBSON walks through a document or an array and -// replaces any extended JSON value with its corresponding BSON type. -func ConvertJSONValueToBSON(x interface{}) (interface{}, error) { - switch v := x.(type) { - case nil: - return nil, nil - case bool: - return v, nil - case map[string]interface{}: // document - for key, jsonValue := range v { - bsonValue, err := ParseJSONValue(jsonValue) - if err != nil { - return nil, err - } - v[key] = bsonValue - } - return v, nil - case bson.D: - for i := range v { - var err error - v[i].Value, err = ParseJSONValue(v[i].Value) - if err != nil { - return nil, err - } - } - return v, nil - - case []interface{}: // array - for i, jsonValue := range v { - bsonValue, err := ParseJSONValue(jsonValue) - if err != nil { - return nil, err - } - v[i] = bsonValue - } - return v, nil - - case string, float64, int32, int64: - return v, nil // require no conversion - - case json.ObjectId: // ObjectId - s := string(v) - if !bson.IsObjectIdHex(s) { - return nil, errors.New("expected ObjectId to contain 24 hexadecimal characters") - } - return bson.ObjectIdHex(s), nil - - case json.Decimal128: - return v.Decimal128, nil - - case json.Date: // Date - n := int64(v) - return time.Unix(n/1e3, n%1e3*1e6), nil - - case json.ISODate: // ISODate - n := string(v) - return util.FormatDate(n) - - case json.NumberLong: // NumberLong - return int64(v), nil - - case json.NumberInt: // NumberInt - return int32(v), nil - - case json.NumberFloat: // NumberFloat - return float64(v), nil - case json.BinData: // BinData - data, err := base64.StdEncoding.DecodeString(v.Base64) - if err != nil { - return nil, err - } - return bson.Binary{v.Type, data}, nil - - case json.DBRef: // DBRef - var err error - v.Id, err = ParseJSONValue(v.Id) - if err != nil { - return nil, err - } - return mgo.DBRef{v.Collection, v.Id, v.Database}, nil - - case json.DBPointer: // DBPointer, for backwards compatibility - return bson.DBPointer{v.Namespace, v.Id}, nil - - case json.RegExp: // RegExp - return bson.RegEx{v.Pattern, v.Options}, nil - - case json.Timestamp: // Timestamp - ts := (int64(v.Seconds) << 32) | int64(v.Increment) - return bson.MongoTimestamp(ts), nil - - case json.JavaScript: // Javascript - return bson.JavaScript{v.Code, v.Scope}, nil - - case json.MinKey: // MinKey - return bson.MinKey, nil - - case json.MaxKey: // MaxKey - return bson.MaxKey, nil - - case json.Undefined: // undefined - return bson.Undefined, nil - - default: - return nil, fmt.Errorf("conversion of JSON value '%v' of type '%T' not supported", v, v) - } -} - -func convertKeys(v bson.M) (bson.M, error) { - for key, value := range v { - jsonValue, err := ConvertBSONValueToJSON(value) - if err != nil { - return nil, err - } - v[key] = jsonValue - } - return v, nil -} - -func getConvertedKeys(v bson.M) (bson.M, error) { - out := bson.M{} - for key, value := range v { - jsonValue, err := GetBSONValueAsJSON(value) - if err != nil { - return nil, err - } - out[key] = jsonValue - } - return out, nil -} - -// ConvertBSONValueToJSON walks through a document or an array and -// converts any BSON value to its corresponding extended JSON type. -// It returns the converted JSON document and any error encountered. -func ConvertBSONValueToJSON(x interface{}) (interface{}, error) { - switch v := x.(type) { - case nil: - return nil, nil - case bool: - return v, nil - - case *bson.M: // document - doc, err := convertKeys(*v) - if err != nil { - return nil, err - } - return doc, err - case bson.M: // document - return convertKeys(v) - case map[string]interface{}: - return convertKeys(v) - case bson.D: - for i, value := range v { - jsonValue, err := ConvertBSONValueToJSON(value.Value) - if err != nil { - return nil, err - } - v[i].Value = jsonValue - } - return MarshalD(v), nil - case MarshalD: - return v, nil - case []interface{}: // array - for i, value := range v { - jsonValue, err := ConvertBSONValueToJSON(value) - if err != nil { - return nil, err - } - v[i] = jsonValue - } - return v, nil - - case string: - return v, nil // require no conversion - - case int: - return json.NumberInt(v), nil - - case bson.ObjectId: // ObjectId - return json.ObjectId(v.Hex()), nil - - case bson.Decimal128: - return json.Decimal128{v}, nil - - case time.Time: // Date - return json.Date(v.Unix()*1000 + int64(v.Nanosecond()/1e6)), nil - - case int64: // NumberLong - return json.NumberLong(v), nil - - case int32: // NumberInt - return json.NumberInt(v), nil - - case float64: - return json.NumberFloat(v), nil - - case float32: - return json.NumberFloat(float64(v)), nil - - case []byte: // BinData (with generic type) - data := base64.StdEncoding.EncodeToString(v) - return json.BinData{0x00, data}, nil - - case bson.Binary: // BinData - data := base64.StdEncoding.EncodeToString(v.Data) - return json.BinData{v.Kind, data}, nil - - case mgo.DBRef: // DBRef - return json.DBRef{v.Collection, v.Id, v.Database}, nil - - case bson.DBPointer: // DBPointer - return json.DBPointer{v.Namespace, v.Id}, nil - - case bson.RegEx: // RegExp - return json.RegExp{v.Pattern, v.Options}, nil - - case bson.MongoTimestamp: // Timestamp - timestamp := int64(v) - return json.Timestamp{ - Seconds: uint32(timestamp >> 32), - Increment: uint32(timestamp), - }, nil - - case bson.JavaScript: // JavaScript - var scope interface{} - var err error - if v.Scope != nil { - scope, err = ConvertBSONValueToJSON(v.Scope) - if err != nil { - return nil, err - } - } - return json.JavaScript{v.Code, scope}, nil - - default: - switch x { - case bson.MinKey: // MinKey - return json.MinKey{}, nil - - case bson.MaxKey: // MaxKey - return json.MaxKey{}, nil - - case bson.Undefined: // undefined - return json.Undefined{}, nil - } - } - - return nil, fmt.Errorf("conversion of BSON value '%v' of type '%T' not supported", x, x) -} - -// GetBSONValueAsJSON is equivalent to ConvertBSONValueToJSON, but does not mutate its argument. -func GetBSONValueAsJSON(x interface{}) (interface{}, error) { - switch v := x.(type) { - case nil: - return nil, nil - case bool: - return v, nil - - case *bson.M: // document - doc, err := getConvertedKeys(*v) - if err != nil { - return nil, err - } - return doc, err - case bson.M: // document - return getConvertedKeys(v) - case map[string]interface{}: - return getConvertedKeys(v) - case bson.D: - out := bson.D{} - for _, value := range v { - jsonValue, err := GetBSONValueAsJSON(value.Value) - if err != nil { - return nil, err - } - out = append(out, bson.DocElem{ - Name: value.Name, - Value: jsonValue, - }) - } - return MarshalD(out), nil - case MarshalD: - out, err := GetBSONValueAsJSON(bson.D(v)) - if err != nil { - return nil, err - } - return MarshalD(out.(bson.D)), nil - case []interface{}: // array - out := []interface{}{} - for _, value := range v { - jsonValue, err := GetBSONValueAsJSON(value) - if err != nil { - return nil, err - } - out = append(out, jsonValue) - } - return out, nil - - case string: - return v, nil // require no conversion - - case int: - return json.NumberInt(v), nil - - case bson.ObjectId: // ObjectId - return json.ObjectId(v.Hex()), nil - - case bson.Decimal128: - return json.Decimal128{v}, nil - - case time.Time: // Date - return json.Date(v.Unix()*1000 + int64(v.Nanosecond()/1e6)), nil - - case int64: // NumberLong - return json.NumberLong(v), nil - - case int32: // NumberInt - return json.NumberInt(v), nil - - case float64: - return json.NumberFloat(v), nil - - case float32: - return json.NumberFloat(float64(v)), nil - - case []byte: // BinData (with generic type) - data := base64.StdEncoding.EncodeToString(v) - return json.BinData{0x00, data}, nil - - case bson.Binary: // BinData - data := base64.StdEncoding.EncodeToString(v.Data) - return json.BinData{v.Kind, data}, nil - - case mgo.DBRef: // DBRef - return json.DBRef{v.Collection, v.Id, v.Database}, nil - - case bson.DBPointer: // DBPointer - return json.DBPointer{v.Namespace, v.Id}, nil - - case bson.RegEx: // RegExp - return json.RegExp{v.Pattern, v.Options}, nil - - case bson.MongoTimestamp: // Timestamp - timestamp := int64(v) - return json.Timestamp{ - Seconds: uint32(timestamp >> 32), - Increment: uint32(timestamp), - }, nil - - case bson.JavaScript: // JavaScript - var scope interface{} - var err error - if v.Scope != nil { - scope, err = GetBSONValueAsJSON(v.Scope) - if err != nil { - return nil, err - } - } - return json.JavaScript{v.Code, scope}, nil - - default: - switch x { - case bson.MinKey: // MinKey - return json.MinKey{}, nil - - case bson.MaxKey: // MaxKey - return json.MaxKey{}, nil - - case bson.Undefined: // undefined - return json.Undefined{}, nil - } - } - - return nil, fmt.Errorf("conversion of BSON value '%v' of type '%T' not supported", x, x) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/converter_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/converter_test.go deleted file mode 100644 index e43ae011f0f..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/converter_test.go +++ /dev/null @@ -1,352 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package bsonutil - -import ( - "encoding/base64" - "fmt" - "testing" - "time" - - "github.com/mongodb/mongo-tools/common/json" - "github.com/mongodb/mongo-tools/common/testtype" - . "github.com/smartystreets/goconvey/convey" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" -) - -func TestObjectIdBSONToJSON(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("Converting a BSON ObjectId", t, func() { - Convey("that is valid to JSON should produce a json.ObjectId", func() { - bsonObjId := bson.NewObjectId() - jsonObjId := json.ObjectId(bsonObjId.Hex()) - - _jObjId, err := ConvertBSONValueToJSON(bsonObjId) - So(err, ShouldBeNil) - jObjId, ok := _jObjId.(json.ObjectId) - So(ok, ShouldBeTrue) - - So(jObjId, ShouldNotEqual, bsonObjId) - So(jObjId, ShouldEqual, jsonObjId) - }) - }) -} - -func TestArraysBSONToJSON(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("Converting BSON arrays to JSON arrays", t, func() { - Convey("should work for empty arrays", func() { - jArr, err := ConvertBSONValueToJSON([]interface{}{}) - So(err, ShouldBeNil) - - So(jArr, ShouldResemble, []interface{}{}) - }) - - Convey("should work for one-level deep arrays", func() { - objId := bson.NewObjectId() - bsonArr := []interface{}{objId, 28, 0.999, "plain"} - _jArr, err := ConvertBSONValueToJSON(bsonArr) - So(err, ShouldBeNil) - jArr, ok := _jArr.([]interface{}) - So(ok, ShouldBeTrue) - - So(len(jArr), ShouldEqual, 4) - So(jArr[0], ShouldEqual, json.ObjectId(objId.Hex())) - So(jArr[1], ShouldEqual, 28) - So(jArr[2], ShouldEqual, 0.999) - So(jArr[3], ShouldEqual, "plain") - }) - - Convey("should work for arrays with embedded objects", func() { - bsonObj := []interface{}{ - 80, - bson.M{ - "a": int64(20), - "b": bson.M{ - "c": bson.RegEx{Pattern: "hi", Options: "i"}, - }, - }, - } - - __jObj, err := ConvertBSONValueToJSON(bsonObj) - So(err, ShouldBeNil) - _jObj, ok := __jObj.([]interface{}) - So(ok, ShouldBeTrue) - jObj, ok := _jObj[1].(bson.M) - So(ok, ShouldBeTrue) - So(len(jObj), ShouldEqual, 2) - So(jObj["a"], ShouldEqual, json.NumberLong(20)) - jjObj, ok := jObj["b"].(bson.M) - So(ok, ShouldBeTrue) - - So(jjObj["c"], ShouldResemble, json.RegExp{"hi", "i"}) - So(jjObj["c"], ShouldNotResemble, json.RegExp{"i", "hi"}) - }) - - }) -} - -func TestDateBSONToJSON(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - timeNow := time.Now() - secs := int64(timeNow.Unix()) - nanosecs := timeNow.Nanosecond() - millis := int64(nanosecs / 1e6) - - timeNowSecs := time.Unix(secs, int64(0)) - timeNowMillis := time.Unix(secs, int64(millis*1e6)) - - Convey("Converting BSON time.Time 's dates to JSON", t, func() { - // json.Date is stored as an int64 representing the number of milliseconds since the epoch - Convey(fmt.Sprintf("should work with second granularity: %v", timeNowSecs), func() { - _jObj, err := ConvertBSONValueToJSON(timeNowSecs) - So(err, ShouldBeNil) - jObj, ok := _jObj.(json.Date) - So(ok, ShouldBeTrue) - - So(int64(jObj), ShouldEqual, secs*1e3) - }) - - Convey(fmt.Sprintf("should work with millisecond granularity: %v", timeNowMillis), func() { - _jObj, err := ConvertBSONValueToJSON(timeNowMillis) - So(err, ShouldBeNil) - jObj, ok := _jObj.(json.Date) - So(ok, ShouldBeTrue) - - So(int64(jObj), ShouldEqual, secs*1e3+millis) - }) - - Convey(fmt.Sprintf("should work with nanosecond granularity: %v", timeNow), func() { - _jObj, err := ConvertBSONValueToJSON(timeNow) - So(err, ShouldBeNil) - jObj, ok := _jObj.(json.Date) - So(ok, ShouldBeTrue) - - // we lose nanosecond precision - So(int64(jObj), ShouldEqual, secs*1e3+millis) - }) - - }) -} - -func TestMaxKeyBSONToJSON(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("Converting a BSON Maxkey to JSON", t, func() { - Convey("should produce a json.MaxKey", func() { - _jObj, err := ConvertBSONValueToJSON(bson.MaxKey) - So(err, ShouldBeNil) - jObj, ok := _jObj.(json.MaxKey) - So(ok, ShouldBeTrue) - - So(jObj, ShouldResemble, json.MaxKey{}) - }) - }) -} - -func TestMinKeyBSONToJSON(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("Converting a BSON Maxkey to JSON", t, func() { - Convey("should produce a json.MinKey", func() { - _jObj, err := ConvertBSONValueToJSON(bson.MinKey) - So(err, ShouldBeNil) - jObj, ok := _jObj.(json.MinKey) - So(ok, ShouldBeTrue) - - So(jObj, ShouldResemble, json.MinKey{}) - }) - }) -} - -func Test64BitIntBSONToJSON(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("Converting a BSON int64 to JSON", t, func() { - Convey("should produce a json.NumberLong", func() { - _jObj, err := ConvertBSONValueToJSON(int32(243)) - So(err, ShouldBeNil) - jObj, ok := _jObj.(json.NumberInt) - So(ok, ShouldBeTrue) - - So(jObj, ShouldEqual, json.NumberInt(243)) - }) - }) - -} - -func Test32BitIntBSONToJSON(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("Converting a BSON int32 integer to JSON", t, func() { - Convey("should produce a json.NumberInt", func() { - _jObj, err := ConvertBSONValueToJSON(int64(888234334343)) - So(err, ShouldBeNil) - jObj, ok := _jObj.(json.NumberLong) - So(ok, ShouldBeTrue) - - So(jObj, ShouldEqual, json.NumberLong(888234334343)) - }) - }) - -} - -func TestRegExBSONToJSON(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("Converting a BSON Regular Expression (= /decision/gi) to JSON", t, func() { - Convey("should produce a json.RegExp", func() { - _jObj, err := ConvertBSONValueToJSON(bson.RegEx{"decision", "gi"}) - So(err, ShouldBeNil) - jObj, ok := _jObj.(json.RegExp) - So(ok, ShouldBeTrue) - - So(jObj, ShouldResemble, json.RegExp{"decision", "gi"}) - }) - }) - -} - -func TestUndefinedValueBSONToJSON(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("Converting a BSON Undefined type to JSON", t, func() { - Convey("should produce a json.Undefined", func() { - _jObj, err := ConvertBSONValueToJSON(bson.Undefined) - So(err, ShouldBeNil) - jObj, ok := _jObj.(json.Undefined) - So(ok, ShouldBeTrue) - - So(jObj, ShouldResemble, json.Undefined{}) - }) - }) -} - -func TestDBRefBSONToJSON(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("Converting BSON DBRef to JSON", t, func() { - Convey("should produce a json.DBRef", func() { - _jObj, err := ConvertBSONValueToJSON(mgo.DBRef{"coll1", "some_id", "test"}) - So(err, ShouldBeNil) - jObj, ok := _jObj.(json.DBRef) - So(ok, ShouldBeTrue) - - So(jObj, ShouldResemble, json.DBRef{"coll1", "some_id", "test"}) - So(jObj, ShouldNotResemble, json.DBRef{"coll1", "test", "some_id"}) - }) - }) -} - -func TestTimestampBSONToJSON(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("Converting a BSON Timestamp to JSON", t, func() { - Convey("should produce a json.Timestamp", func() { - // {t:803434343, i:9} == bson.MongoTimestamp(803434343*2**32 + 9) - _jObj, err := ConvertBSONValueToJSON(bson.MongoTimestamp(uint64(803434343<<32) | uint64(9))) - So(err, ShouldBeNil) - jObj, ok := _jObj.(json.Timestamp) - So(ok, ShouldBeTrue) - - So(jObj, ShouldResemble, json.Timestamp{Seconds: 803434343, Increment: 9}) - So(jObj, ShouldNotResemble, json.Timestamp{Seconds: 803434343, Increment: 8}) - }) - }) -} - -func TestBinaryBSONToJSON(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("Converting BSON Binary data to JSON", t, func() { - Convey("should produce a json.BinData", func() { - _jObj, err := ConvertBSONValueToJSON(bson.Binary{'\x01', []byte("\x05\x20\x02\xae\xf7")}) - So(err, ShouldBeNil) - jObj, ok := _jObj.(json.BinData) - So(ok, ShouldBeTrue) - - base64data1 := base64.StdEncoding.EncodeToString([]byte("\x05\x20\x02\xae\xf7")) - base64data2 := base64.StdEncoding.EncodeToString([]byte("\x05\x20\x02\xaf\xf7")) - So(jObj, ShouldResemble, json.BinData{'\x01', base64data1}) - So(jObj, ShouldNotResemble, json.BinData{'\x01', base64data2}) - }) - }) -} - -func TestGenericBytesBSONToJSON(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("Converting Go bytes to JSON", t, func() { - Convey("should produce a json.BinData with Type=0x00 (Generic)", func() { - _jObj, err := ConvertBSONValueToJSON([]byte("this is something that's cool")) - So(err, ShouldBeNil) - jObj, ok := _jObj.(json.BinData) - So(ok, ShouldBeTrue) - - base64data := base64.StdEncoding.EncodeToString([]byte("this is something that's cool")) - So(jObj, ShouldResemble, json.BinData{0x00, base64data}) - So(jObj, ShouldNotResemble, json.BinData{0x01, base64data}) - }) - }) -} - -func TestUnknownBSONTypeToJSON(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("Converting an unknown BSON type to JSON", t, func() { - Convey("should produce an error", func() { - _, err := ConvertBSONValueToJSON(func() {}) - So(err, ShouldNotBeNil) - }) - }) -} - -func TestDBPointerBSONToJSON(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("Converting BSON DBPointer to JSON", t, func() { - Convey("should produce a json.DBPointer", func() { - objId := bson.NewObjectId() - _jObj, err := ConvertBSONValueToJSON(bson.DBPointer{"dbrefnamespace", objId}) - So(err, ShouldBeNil) - jObj, ok := _jObj.(json.DBPointer) - So(ok, ShouldBeTrue) - - So(jObj, ShouldResemble, json.DBPointer{"dbrefnamespace", objId}) - }) - }) -} - -func TestJSCodeBSONToJSON(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("Converting BSON Javascript code to JSON", t, func() { - Convey("should produce a json.Javascript", func() { - Convey("without scope if the scope for the BSON Javascript code is nil", func() { - _jObj, err := ConvertBSONValueToJSON(bson.JavaScript{"function() { return null; }", nil}) - So(err, ShouldBeNil) - jObj, ok := _jObj.(json.JavaScript) - So(ok, ShouldBeTrue) - - So(jObj, ShouldResemble, json.JavaScript{"function() { return null; }", nil}) - }) - - Convey("with scope if the scope for the BSON Javascript code is non-nil", func() { - _jObj, err := ConvertBSONValueToJSON(bson.JavaScript{"function() { return x; }", bson.M{"x": 2}}) - So(err, ShouldBeNil) - jObj, ok := _jObj.(json.JavaScript) - So(ok, ShouldBeTrue) - So(jObj.Scope.(bson.M)["x"], ShouldEqual, 2) - So(jObj.Code, ShouldEqual, "function() { return x; }") - }) - }) - }) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/date_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/date_test.go deleted file mode 100644 index f929c35c12b..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/date_test.go +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package bsonutil - -import ( - "fmt" - "github.com/mongodb/mongo-tools/common/json" - "github.com/mongodb/mongo-tools/common/testtype" - . "github.com/smartystreets/goconvey/convey" - "testing" - "time" -) - -func TestDateValue(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("When converting JSON with Date values", t, func() { - - Convey("works for Date object", func() { - key := "key" - jsonMap := map[string]interface{}{ - key: json.Date(100), - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - - jsonValue, ok := jsonMap[key].(time.Time) - So(ok, ShouldBeTrue) - So(jsonValue.Equal(time.Unix(0, int64(100*time.Millisecond))), ShouldBeTrue) - }) - - Convey("works for Date document", func() { - - dates := []string{ - "2006-01-02T15:04:05.000Z", - "2006-01-02T15:04:05.000-0700", - "2006-01-02T15:04:05Z", - "2006-01-02T15:04:05-0700", - "2006-01-02T15:04Z", - "2006-01-02T15:04-0700", - } - - for _, dateString := range dates { - example := fmt.Sprintf(`{ "$date": "%v" }`, dateString) - Convey(fmt.Sprintf("of string ('%v')", example), func() { - key := "key" - jsonMap := map[string]interface{}{ - key: map[string]interface{}{ - "$date": dateString, - }, - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - - // dateString is a valid time format string - date, err := time.Parse(dateString, dateString) - So(err, ShouldBeNil) - - jsonValue, ok := jsonMap[key].(time.Time) - So(ok, ShouldBeTrue) - So(jsonValue.Equal(date), ShouldBeTrue) - }) - } - - date := time.Unix(0, int64(time.Duration(1136214245000)*time.Millisecond)) - - Convey(`of $numberLong ('{ "$date": { "$numberLong": "1136214245000" } }')`, func() { - key := "key" - jsonMap := map[string]interface{}{ - key: map[string]interface{}{ - "$date": map[string]interface{}{ - "$numberLong": "1136214245000", - }, - }, - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - - jsonValue, ok := jsonMap[key].(time.Time) - So(ok, ShouldBeTrue) - So(jsonValue.Equal(date), ShouldBeTrue) - }) - - Convey(`of json.Number ('{ "$date": 1136214245000 }')`, func() { - key := "key" - jsonMap := map[string]interface{}{ - key: map[string]interface{}{ - "$date": json.Number("1136214245000"), - }, - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - - jsonValue, ok := jsonMap[key].(time.Time) - So(ok, ShouldBeTrue) - So(jsonValue.Equal(date), ShouldBeTrue) - }) - - Convey(`of numeric int64 ('{ "$date": 1136214245000 }')`, func() { - key := "key" - jsonMap := map[string]interface{}{ - key: map[string]interface{}{ - "$date": int64(1136214245000), - }, - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - - jsonValue, ok := jsonMap[key].(time.Time) - So(ok, ShouldBeTrue) - So(jsonValue.Equal(date), ShouldBeTrue) - }) - - Convey(`of numeric float64 ('{ "$date": 1136214245000 }')`, func() { - key := "key" - jsonMap := map[string]interface{}{ - key: map[string]interface{}{ - "$date": float64(1136214245000), - }, - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - - jsonValue, ok := jsonMap[key].(time.Time) - So(ok, ShouldBeTrue) - So(jsonValue.Equal(date), ShouldBeTrue) - }) - Convey(`of numeric int32 ('{ "$date": 2136800000 }')`, func() { - key := "key" - - date = time.Unix(0, int64(time.Duration(2136800000)*time.Millisecond)) - - jsonMap := map[string]interface{}{ - key: map[string]interface{}{ - "$date": int32(2136800000), - }, - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - - jsonValue, ok := jsonMap[key].(time.Time) - So(ok, ShouldBeTrue) - So(jsonValue.Equal(date), ShouldBeTrue) - }) - - Convey(`of negative numeric int32 ('{ "$date": -2136800000 }')`, func() { - key := "key" - - date = time.Unix(0, int64(time.Duration(-2136800000)*time.Millisecond)) - - jsonMap := map[string]interface{}{ - key: map[string]interface{}{ - "$date": int32(-2136800000), - }, - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - - jsonValue, ok := jsonMap[key].(time.Time) - So(ok, ShouldBeTrue) - So(jsonValue.Equal(date), ShouldBeTrue) - }) - }) - }) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/marshal_d.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/marshal_d.go deleted file mode 100644 index 591447814af..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/marshal_d.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package bsonutil - -import ( - "bytes" - "fmt" - "github.com/mongodb/mongo-tools/common/json" - "github.com/mongodb/mongo-tools/common/util" - "gopkg.in/mgo.v2/bson" -) - -// MarshalD is a wrapper for bson.D that allows unmarshalling -// of bson.D with preserved order. Necessary for printing -// certain database commands. -type MarshalD bson.D - -// MarshalJSON makes the MarshalD type usable by -// the encoding/json package. -func (md MarshalD) MarshalJSON() ([]byte, error) { - var buff bytes.Buffer - buff.WriteString("{") - for i, item := range md { - key, err := json.Marshal(item.Name) - if err != nil { - return nil, fmt.Errorf("cannot marshal key %v: %v", item.Name, err) - } - val, err := json.Marshal(item.Value) - if err != nil { - return nil, fmt.Errorf("cannot marshal value %v: %v", item.Value, err) - } - buff.Write(key) - buff.WriteString(":") - buff.Write(val) - if i != len(md)-1 { - buff.WriteString(",") - } - } - buff.WriteString("}") - return buff.Bytes(), nil -} - -// MakeSortString takes a bson.D object and converts it to a slice of strings -// that can be used as the input args to mgo's .Sort(...) function. -// For example: -// {a:1, b:-1} -> ["+a", "-b"] -func MakeSortString(sortObj bson.D) ([]string, error) { - sortStrs := make([]string, 0, len(sortObj)) - for _, docElem := range sortObj { - valueAsNumber, err := util.ToFloat64(docElem.Value) - if err != nil { - return nil, err - } - prefix := "+" - if valueAsNumber < 0 { - prefix = "-" - } - sortStrs = append(sortStrs, fmt.Sprintf("%v%v", prefix, docElem.Name)) - } - return sortStrs, nil -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/marshal_d_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/marshal_d_test.go deleted file mode 100644 index 9961e488b18..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/marshal_d_test.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package bsonutil - -import ( - "encoding/json" - "github.com/mongodb/mongo-tools/common/testtype" - . "github.com/smartystreets/goconvey/convey" - "gopkg.in/mgo.v2/bson" - "strings" - "testing" -) - -func TestMarshalDMarshalJSON(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("With a valid bson.D", t, func() { - testD := bson.D{ - {"cool", "rad"}, - {"aaa", 543.2}, - {"I", 0}, - {"E", 0}, - {"map", bson.M{"1": 1, "2": "two"}}, - } - - Convey("wrapping with MarshalD should allow json.Marshal to work", func() { - asJSON, err := json.Marshal(MarshalD(testD)) - So(err, ShouldBeNil) - strJSON := string(asJSON) - - Convey("with order preserved", func() { - So(strings.Index(strJSON, "cool"), ShouldBeLessThan, strings.Index(strJSON, "aaa")) - So(strings.Index(strJSON, "aaa"), ShouldBeLessThan, strings.Index(strJSON, "I")) - So(strings.Index(strJSON, "I"), ShouldBeLessThan, strings.Index(strJSON, "E")) - So(strings.Index(strJSON, "E"), ShouldBeLessThan, strings.Index(strJSON, "map")) - So(strings.Count(strJSON, ","), ShouldEqual, 5) // 4 + 1 from internal map - }) - - Convey("but still usable by the json parser", func() { - var asMap bson.M - err := json.Unmarshal(asJSON, &asMap) - So(err, ShouldBeNil) - - Convey("with types & values preserved", func() { - So(asMap["cool"], ShouldEqual, "rad") - So(asMap["aaa"], ShouldEqual, 543.2) - So(asMap["I"], ShouldEqual, 0) - So(asMap["E"], ShouldEqual, 0) - So(asMap["map"].(map[string]interface{})["1"], ShouldEqual, 1) - So(asMap["map"].(map[string]interface{})["2"], ShouldEqual, "two") - }) - }) - - Convey("putting it inside another map should still be usable by json.Marshal", func() { - _, err := json.Marshal(bson.M{"x": 0, "y": MarshalD(testD)}) - So(err, ShouldBeNil) - }) - }) - }) - - Convey("With en empty bson.D", t, func() { - testD := bson.D{} - - Convey("wrapping with MarshalD should allow json.Marshal to work", func() { - asJSON, err := json.Marshal(MarshalD(testD)) - So(err, ShouldBeNil) - strJSON := string(asJSON) - So(strJSON, ShouldEqual, "{}") - - Convey("but still usable by the json parser", func() { - var asInterface interface{} - err := json.Unmarshal(asJSON, &asInterface) - So(err, ShouldBeNil) - asMap, ok := asInterface.(map[string]interface{}) - So(ok, ShouldBeTrue) - So(len(asMap), ShouldEqual, 0) - }) - }) - }) -} - -func TestFindValueByKey(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - Convey("Given a bson.D document and a specific key", t, func() { - subDocument := &bson.D{ - bson.DocElem{Name: "field4", Value: "c"}, - } - document := &bson.D{ - bson.DocElem{Name: "field1", Value: "a"}, - bson.DocElem{Name: "field2", Value: "b"}, - bson.DocElem{Name: "field3", Value: subDocument}, - } - Convey("the corresponding value top-level keys should be returned", func() { - value, err := FindValueByKey("field1", document) - So(value, ShouldEqual, "a") - So(err, ShouldBeNil) - }) - Convey("the corresponding value top-level keys with sub-document values should be returned", func() { - value, err := FindValueByKey("field3", document) - So(value, ShouldEqual, subDocument) - So(err, ShouldBeNil) - }) - Convey("for non-existent keys nil and an error should be returned", func() { - value, err := FindValueByKey("field4", document) - So(value, ShouldBeNil) - So(err, ShouldNotBeNil) - }) - }) -} - -func TestEscapedKey(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - Convey("Given a bson.D document with a key that requires escaping", t, func() { - document := bson.D{ - bson.DocElem{Name: `foo"bar`, Value: "a"}, - } - Convey("it can be marshaled without error", func() { - asJSON, err := json.Marshal(MarshalD(document)) - So(err, ShouldBeNil) - Convey("and subsequently unmarshaled without error", func() { - var asMap bson.M - err := json.Unmarshal(asJSON, &asMap) - So(err, ShouldBeNil) - Convey("with the original value being correctly found with the unescaped key", func() { - So(asMap[`foo"bar`], ShouldEqual, "a") - }) - }) - }) - }) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/maxkey_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/maxkey_test.go deleted file mode 100644 index f64df05ad4d..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/maxkey_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package bsonutil - -import ( - "github.com/mongodb/mongo-tools/common/json" - "github.com/mongodb/mongo-tools/common/testtype" - . "github.com/smartystreets/goconvey/convey" - "gopkg.in/mgo.v2/bson" - "testing" -) - -func TestMaxKeyValue(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("When converting JSON with MaxKey values", t, func() { - - Convey("works for MaxKey literal", func() { - key := "key" - jsonMap := map[string]interface{}{ - key: json.MaxKey{}, - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - So(jsonMap[key], ShouldResemble, bson.MaxKey) - }) - - Convey(`works for MaxKey document ('{ "$maxKey": 1 }')`, func() { - key := "key" - jsonMap := map[string]interface{}{ - key: map[string]interface{}{ - "$maxKey": 1, - }, - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - So(jsonMap[key], ShouldResemble, bson.MaxKey) - }) - }) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/minkey_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/minkey_test.go deleted file mode 100644 index 22d6e3382ba..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/minkey_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package bsonutil - -import ( - "github.com/mongodb/mongo-tools/common/json" - "github.com/mongodb/mongo-tools/common/testtype" - . "github.com/smartystreets/goconvey/convey" - "gopkg.in/mgo.v2/bson" - "testing" -) - -func TestMinKeyValue(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("When converting JSON with MinKey values", t, func() { - - Convey("works for MinKey literal", func() { - key := "key" - jsonMap := map[string]interface{}{ - key: json.MinKey{}, - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - So(jsonMap[key], ShouldResemble, bson.MinKey) - }) - - Convey(`works for MinKey document ('{ "$minKey": 1 }')`, func() { - key := "key" - jsonMap := map[string]interface{}{ - key: map[string]interface{}{ - "$minKey": 1, - }, - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - So(jsonMap[key], ShouldResemble, bson.MinKey) - }) - }) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/number.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/number.go deleted file mode 100644 index 3a00bb615e8..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/number.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package bsonutil - -import ( - "fmt" - "reflect" -) - -var floatType = reflect.TypeOf(float64(0)) - -func getFloat(unk interface{}) (float64, error) { - v := reflect.ValueOf(unk) - v = reflect.Indirect(v) - if !v.Type().ConvertibleTo(floatType) { - return 0, fmt.Errorf("cannot convert %v to float64", v.Type()) - } - fv := v.Convert(floatType) - return fv.Float(), nil -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/numberint_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/numberint_test.go deleted file mode 100644 index 78050471fdd..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/numberint_test.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package bsonutil - -import ( - "github.com/mongodb/mongo-tools/common/json" - "github.com/mongodb/mongo-tools/common/testtype" - . "github.com/smartystreets/goconvey/convey" - "testing" -) - -func TestNumberIntValue(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("When converting JSON with NumberInt values", t, func() { - - Convey("works for NumberInt constructor", func() { - key := "key" - jsonMap := map[string]interface{}{ - key: json.NumberInt(42), - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - So(jsonMap[key], ShouldEqual, int32(42)) - }) - - Convey(`works for NumberInt document ('{ "$numberInt": "42" }')`, func() { - key := "key" - jsonMap := map[string]interface{}{ - key: map[string]interface{}{ - "$numberInt": "42", - }, - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - So(jsonMap[key], ShouldEqual, int32(42)) - }) - }) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/numberlong_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/numberlong_test.go deleted file mode 100644 index 9be09850f7c..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/numberlong_test.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package bsonutil - -import ( - "github.com/mongodb/mongo-tools/common/json" - "github.com/mongodb/mongo-tools/common/testtype" - . "github.com/smartystreets/goconvey/convey" - "testing" -) - -func TestNumberLongValue(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("When converting JSON with NumberLong values", t, func() { - - Convey("works for NumberLong constructor", func() { - key := "key" - jsonMap := map[string]interface{}{ - key: json.NumberLong(42), - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - So(jsonMap[key], ShouldEqual, int64(42)) - }) - - Convey(`works for NumberLong document ('{ "$numberLong": "42" }')`, func() { - key := "key" - jsonMap := map[string]interface{}{ - key: map[string]interface{}{ - "$numberLong": "42", - }, - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - So(jsonMap[key], ShouldEqual, int64(42)) - }) - }) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/objectid_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/objectid_test.go deleted file mode 100644 index 10a88ab9f8b..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/objectid_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package bsonutil - -import ( - "github.com/mongodb/mongo-tools/common/json" - "github.com/mongodb/mongo-tools/common/testtype" - . "github.com/smartystreets/goconvey/convey" - "gopkg.in/mgo.v2/bson" - "testing" -) - -func TestObjectIdValue(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("When converting JSON with ObjectId values", t, func() { - - Convey("works for ObjectId constructor", func() { - key := "key" - jsonMap := map[string]interface{}{ - key: json.ObjectId("0123456789abcdef01234567"), - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - So(jsonMap[key], ShouldEqual, bson.ObjectIdHex("0123456789abcdef01234567")) - }) - - Convey(`works for ObjectId document ('{ "$oid": "0123456789abcdef01234567" }')`, func() { - key := "key" - jsonMap := map[string]interface{}{ - key: map[string]interface{}{ - "$oid": "0123456789abcdef01234567", - }, - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - So(jsonMap[key], ShouldEqual, bson.ObjectIdHex("0123456789abcdef01234567")) - }) - }) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/regexp_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/regexp_test.go deleted file mode 100644 index efa0891d9e2..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/regexp_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package bsonutil - -import ( - "github.com/mongodb/mongo-tools/common/json" - "github.com/mongodb/mongo-tools/common/testtype" - . "github.com/smartystreets/goconvey/convey" - "gopkg.in/mgo.v2/bson" - "testing" -) - -func TestRegExpValue(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("When converting JSON with RegExp values", t, func() { - - Convey("works for RegExp constructor", func() { - key := "key" - jsonMap := map[string]interface{}{ - key: json.RegExp{"foo", "i"}, - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - So(jsonMap[key], ShouldResemble, bson.RegEx{"foo", "i"}) - }) - - Convey(`works for RegExp document ('{ "$regex": "foo", "$options": "i" }')`, func() { - key := "key" - jsonMap := map[string]interface{}{ - key: map[string]interface{}{ - "$regex": "foo", - "$options": "i", - }, - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - So(jsonMap[key], ShouldResemble, bson.RegEx{"foo", "i"}) - }) - - Convey(`can use multiple options ('{ "$regex": "bar", "$options": "gims" }')`, func() { - key := "key" - jsonMap := map[string]interface{}{ - key: map[string]interface{}{ - "$regex": "bar", - "$options": "gims", - }, - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - So(jsonMap[key], ShouldResemble, bson.RegEx{"bar", "gims"}) - }) - - Convey(`fails for an invalid option ('{ "$regex": "baz", "$options": "y" }')`, func() { - key := "key" - jsonMap := map[string]interface{}{ - key: map[string]interface{}{ - "$regex": "baz", - "$options": "y", - }, - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldNotBeNil) - }) - }) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/timestamp_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/timestamp_test.go deleted file mode 100644 index 9310e05ded0..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/timestamp_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package bsonutil - -import ( - "github.com/mongodb/mongo-tools/common/json" - "github.com/mongodb/mongo-tools/common/testtype" - . "github.com/smartystreets/goconvey/convey" - "gopkg.in/mgo.v2/bson" - "testing" -) - -func TestTimestampValue(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("When converting JSON with Timestamp values", t, func() { - testTS := bson.MongoTimestamp(123456<<32 | 55) - - Convey("works for Timestamp literal", func() { - - jsonMap := map[string]interface{}{ - "ts": json.Timestamp{123456, 55}, - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - So(jsonMap["ts"], ShouldEqual, testTS) - }) - - Convey(`works for Timestamp document`, func() { - Convey(`{"ts":{"$timestamp":{"t":123456, "i":55}}}`, func() { - jsonMap := map[string]interface{}{ - "ts": map[string]interface{}{ - "$timestamp": map[string]interface{}{ - "t": 123456.0, - "i": 55.0, - }, - }, - } - - bsonMap, err := ConvertJSONValueToBSON(jsonMap) - So(err, ShouldBeNil) - So(bsonMap.(map[string]interface{})["ts"], ShouldEqual, testTS) - }) - }) - }) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/undefined_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/undefined_test.go deleted file mode 100644 index 02acc63188e..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/bsonutil/undefined_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package bsonutil - -import ( - "github.com/mongodb/mongo-tools/common/json" - "github.com/mongodb/mongo-tools/common/testtype" - . "github.com/smartystreets/goconvey/convey" - "gopkg.in/mgo.v2/bson" - "testing" -) - -func TestUndefinedValue(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("When converting JSON with undefined values", t, func() { - - Convey("works for undefined literal", func() { - key := "key" - jsonMap := map[string]interface{}{ - key: json.Undefined{}, - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - So(jsonMap[key], ShouldResemble, bson.Undefined) - }) - - Convey(`works for undefined document ('{ "$undefined": true }')`, func() { - key := "key" - jsonMap := map[string]interface{}{ - key: map[string]interface{}{ - "$undefined": true, - }, - } - - err := ConvertJSONDocumentToBSON(jsonMap) - So(err, ShouldBeNil) - So(jsonMap[key], ShouldResemble, bson.Undefined) - }) - }) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/common.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/common.go deleted file mode 100644 index 4b11ae51a41..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/common.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -// Package common contains subpackages that are shared amongst the mongo -// tools. -package common - -import ( - "strings" -) - -// SplitNamespace returns the db and column from a single namespace string. -func SplitNamespace(ns string) (string, string) { - i := strings.Index(ns, ".") - if i != -1 { - return ns[:i], ns[i+1:] - } - return "", ns -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/bson_stream.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/bson_stream.go deleted file mode 100644 index 4679b9bdde4..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/bson_stream.go +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package db - -import ( - "fmt" - "gopkg.in/mgo.v2/bson" - "io" -) - -// BSONSource reads documents from the underlying io.ReadCloser, Stream which -// wraps a stream of BSON documents. -type BSONSource struct { - reusableBuf []byte - Stream io.ReadCloser - err error -} - -// DecodedBSONSource reads documents from the underlying io.ReadCloser, Stream which -// wraps a stream of BSON documents. -type DecodedBSONSource struct { - RawDocSource - err error -} - -// RawDocSource wraps basic functions for reading a BSON source file. -type RawDocSource interface { - LoadNext() []byte - Close() error - Err() error -} - -// NewBSONSource creates a BSONSource with a reusable I/O buffer -func NewBSONSource(in io.ReadCloser) *BSONSource { - return &BSONSource{make([]byte, MaxBSONSize), in, nil} -} - -// NewBufferlessBSONSource creates a BSONSource without a reusable I/O buffer -func NewBufferlessBSONSource(in io.ReadCloser) *BSONSource { - return &BSONSource{nil, in, nil} -} - -// Close closes the BSONSource, rendering it unusable for I/O. -// It returns an error, if any. -func (bs *BSONSource) Close() error { - return bs.Stream.Close() -} - -func NewDecodedBSONSource(ds RawDocSource) *DecodedBSONSource { - return &DecodedBSONSource{ds, nil} -} - -// Err returns any error in the DecodedBSONSource or its RawDocSource. -func (dbs *DecodedBSONSource) Err() error { - if dbs.err != nil { - return dbs.err - } - return dbs.RawDocSource.Err() -} - -// Next unmarshals the next BSON document into result. Returns true if no errors -// are encountered and false otherwise. -func (dbs *DecodedBSONSource) Next(result interface{}) bool { - doc := dbs.LoadNext() - if doc == nil { - return false - } - if err := bson.Unmarshal(doc, result); err != nil { - dbs.err = err - return false - } - dbs.err = nil - return true -} - -// LoadNext reads and returns the next BSON document in the stream. If the -// BSONSource was created with NewBSONSource then each returned []byte will be -// a slice of a single reused I/O buffer. If the BSONSource was created with -// NewBufferlessBSONSource then each returned []byte will be individually -// allocated -func (bs *BSONSource) LoadNext() []byte { - var into []byte - if bs.reusableBuf == nil { - into = make([]byte, 4) - } else { - into = bs.reusableBuf - } - // read the bson object size (a 4 byte integer) - _, err := io.ReadAtLeast(bs.Stream, into[0:4], 4) - if err != nil { - if err != io.EOF { - bs.err = err - return nil - } - // we hit EOF right away, so we're at the end of the stream. - bs.err = nil - return nil - } - - bsonSize := int32( - (uint32(into[0]) << 0) | - (uint32(into[1]) << 8) | - (uint32(into[2]) << 16) | - (uint32(into[3]) << 24), - ) - - // Verify that the size of the BSON object we are about to read can - // actually fit into the buffer that was provided. If not, either the BSON is - // invalid, or the buffer passed in is too small. - // Verify that we do not have an invalid BSON document with size < 5. - if bsonSize > MaxBSONSize || bsonSize < 5 { - bs.err = fmt.Errorf("invalid BSONSize: %v bytes", bsonSize) - return nil - } - if int(bsonSize) > cap(into) { - bigInto := make([]byte, bsonSize) - copy(bigInto, into) - into = bigInto - if bs.reusableBuf != nil { - bs.reusableBuf = bigInto - } - } - into = into[:int(bsonSize)] - _, err = io.ReadAtLeast(bs.Stream, into[4:], int(bsonSize-4)) - if err != nil { - if err != io.EOF { - bs.err = err - return nil - } - // this case means we hit EOF but read a partial document, - // so there's a broken doc in the stream. Treat this as error. - bs.err = fmt.Errorf("invalid bson: %v", err) - return nil - } - - bs.err = nil - return into -} - -func (bs *BSONSource) Err() error { - return bs.err -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/bson_stream_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/bson_stream_test.go deleted file mode 100644 index e26f6f3aeb0..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/bson_stream_test.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package db - -import ( - "bytes" - . "github.com/smartystreets/goconvey/convey" - "gopkg.in/mgo.v2/bson" - "io/ioutil" - "testing" - - "github.com/mongodb/mongo-tools/common/testtype" -) - -func TestBufferlessBSONSource(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.IntegrationTestType) - var testValues = []bson.M{ - {"_": bson.Binary{Kind: 0x80, Data: []byte("apples")}}, - {"_": bson.Binary{Kind: 0x80, Data: []byte("bananas")}}, - {"_": bson.Binary{Kind: 0x80, Data: []byte("cherries")}}, - } - Convey("with a buffer containing several bson documents with binary fields", t, func() { - writeBuf := bytes.NewBuffer(make([]byte, 0, 1024)) - for _, tv := range testValues { - data, err := bson.Marshal(&tv) - So(err, ShouldBeNil) - _, err = writeBuf.Write(data) - So(err, ShouldBeNil) - } - Convey("that we parse correctly with a BufferlessBSONSource", func() { - bsonSource := NewDecodedBSONSource( - NewBufferlessBSONSource(ioutil.NopCloser(writeBuf))) - docs := []bson.M{} - count := 0 - doc := &bson.M{} - for bsonSource.Next(doc) { - count++ - docs = append(docs, *doc) - doc = &bson.M{} - } - So(bsonSource.Err(), ShouldBeNil) - So(count, ShouldEqual, len(testValues)) - So(docs, ShouldResemble, testValues) - }) - }) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/buffered_bulk.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/buffered_bulk.go deleted file mode 100644 index 65b337c0154..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/buffered_bulk.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package db - -import ( - "fmt" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" -) - -// BufferedBulkInserter implements a bufio.Writer-like design for queuing up -// documents and inserting them in bulk when the given doc limit (or max -// message size) is reached. Must be flushed at the end to ensure that all -// documents are written. -type BufferedBulkInserter struct { - bulk *mgo.Bulk - collection *mgo.Collection - continueOnError bool - docLimit int - byteCount int - docCount int - unordered bool -} - -// NewBufferedBulkInserter returns an initialized BufferedBulkInserter -// for writing. -func NewBufferedBulkInserter(collection *mgo.Collection, docLimit int, - continueOnError bool) *BufferedBulkInserter { - bb := &BufferedBulkInserter{ - collection: collection, - continueOnError: continueOnError, - docLimit: docLimit, - } - bb.resetBulk() - return bb -} - -func (bb *BufferedBulkInserter) Unordered() { - bb.unordered = true - bb.bulk.Unordered() -} - -// throw away the old bulk and init a new one -func (bb *BufferedBulkInserter) resetBulk() { - bb.bulk = bb.collection.Bulk() - if bb.continueOnError || bb.unordered { - bb.bulk.Unordered() - } - bb.byteCount = 0 - bb.docCount = 0 -} - -// Insert adds a document to the buffer for bulk insertion. If the buffer is -// full, the bulk insert is made, returning any error that occurs. -func (bb *BufferedBulkInserter) Insert(doc interface{}) error { - rawBytes, err := bson.Marshal(doc) - if err != nil { - return fmt.Errorf("bson encoding error: %v", err) - } - // flush if we are full - if bb.docCount >= bb.docLimit || bb.byteCount+len(rawBytes) > MaxBSONSize { - err = bb.Flush() - } - // buffer the document - bb.docCount++ - bb.byteCount += len(rawBytes) - bb.bulk.Insert(bson.Raw{Data: rawBytes}) - return err -} - -// Flush writes all buffered documents in one bulk insert then resets the buffer. -func (bb *BufferedBulkInserter) Flush() error { - if bb.docCount == 0 { - return nil - } - defer bb.resetBulk() - if _, err := bb.bulk.Run(); err != nil { - return err - } - return nil -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/buffered_bulk_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/buffered_bulk_test.go deleted file mode 100644 index 9d7b98cfcd6..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/buffered_bulk_test.go +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package db - -import ( - "os" - "testing" - - "github.com/mongodb/mongo-tools/common/options" - "github.com/mongodb/mongo-tools/common/testtype" - . "github.com/smartystreets/goconvey/convey" - "gopkg.in/mgo.v2/bson" -) - -// Copied this var block, GetSSLOptions, and GetAuthOptions from common/testutil/auth.go to avoid circular dependency -var ( - UserAdmin = "uAdmin" - UserAdminPassword = "password" - CreatedUserNameEnv = "TOOLS_TESTING_AUTH_USERNAME" - CreatedUserPasswordEnv = "TOOLS_TESTING_AUTH_PASSWORD" -) - -func DBGetSSLOptions() options.SSL { - if testtype.HasTestType(testtype.SSLTestType) { - return options.SSL{ - UseSSL: true, - SSLCAFile: "./openssl/testdata/ca.pem", - SSLPEMKeyFile: "./openssl/testdata/server.pem", - } - } - - return options.SSL{ - UseSSL: false, - } -} - -func DBGetAuthOptions() options.Auth { - if testtype.HasTestType(testtype.AuthTestType) { - return options.Auth{ - Username: os.Getenv(CreatedUserNameEnv), - Password: os.Getenv(CreatedUserPasswordEnv), - Source: "admin", - } - } - - return options.Auth{} -} - -func TestBufferedBulkInserterInserts(t *testing.T) { - var bufBulk *BufferedBulkInserter - - testtype.SkipUnlessTestType(t, testtype.IntegrationTestType) - - auth := DBGetAuthOptions() - ssl := DBGetSSLOptions() - - Convey("With a valid session", t, func() { - opts := options.ToolOptions{ - Connection: &options.Connection{ - Port: DefaultTestPort, - }, - SSL: &ssl, - Auth: &auth, - } - provider, err := NewSessionProvider(opts) - So(provider, ShouldNotBeNil) - So(err, ShouldBeNil) - session, err := provider.GetSession() - So(session, ShouldNotBeNil) - So(err, ShouldBeNil) - - Convey("using a test collection and a doc limit of 3", func() { - testCol := session.DB("tools-test").C("bulk1") - bufBulk = NewBufferedBulkInserter(testCol, 3, false) - So(bufBulk, ShouldNotBeNil) - - Convey("inserting 10 documents into the BufferedBulkInserter", func() { - flushCount := 0 - for i := 0; i < 10; i++ { - So(bufBulk.Insert(bson.D{}), ShouldBeNil) - if bufBulk.docCount%3 == 0 { - flushCount++ - } - } - - Convey("should have flushed 3 times with one doc still buffered", func() { - So(flushCount, ShouldEqual, 3) - So(bufBulk.byteCount, ShouldBeGreaterThan, 0) - So(bufBulk.docCount, ShouldEqual, 1) - }) - }) - }) - - Convey("using a test collection and a doc limit of 1", func() { - testCol := session.DB("tools-test").C("bulk2") - bufBulk = NewBufferedBulkInserter(testCol, 1, false) - So(bufBulk, ShouldNotBeNil) - - Convey("inserting 10 documents into the BufferedBulkInserter and flushing", func() { - for i := 0; i < 10; i++ { - So(bufBulk.Insert(bson.D{}), ShouldBeNil) - } - So(bufBulk.Flush(), ShouldBeNil) - - Convey("should have no docs buffered", func() { - So(bufBulk.docCount, ShouldEqual, 0) - So(bufBulk.byteCount, ShouldEqual, 0) - }) - }) - }) - - Convey("using a test collection and a doc limit of 1000", func() { - testCol := session.DB("tools-test").C("bulk3") - bufBulk = NewBufferedBulkInserter(testCol, 100, false) - So(bufBulk, ShouldNotBeNil) - - Convey("inserting 1,000,000 documents into the BufferedBulkInserter and flushing", func() { - session.SetSocketTimeout(0) - - for i := 0; i < 1000000; i++ { - bufBulk.Insert(bson.M{"_id": i}) - } - So(bufBulk.Flush(), ShouldBeNil) - - Convey("should have inserted all of the documents", func() { - count, err := testCol.Count() - So(err, ShouldBeNil) - So(count, ShouldEqual, 1000000) - - // test values - testDoc := bson.M{} - err = testCol.Find(bson.M{"_id": 477232}).One(&testDoc) - So(err, ShouldBeNil) - So(testDoc["_id"], ShouldEqual, 477232) - err = testCol.Find(bson.M{"_id": 999999}).One(&testDoc) - So(err, ShouldBeNil) - So(testDoc["_id"], ShouldEqual, 999999) - err = testCol.Find(bson.M{"_id": 1}).One(&testDoc) - So(err, ShouldBeNil) - So(testDoc["_id"], ShouldEqual, 1) - - }) - }) - }) - - Reset(func() { - session.DB("tools-test").DropDatabase() - session.Close() - provider.Close() - }) - }) - -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/command.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/command.go deleted file mode 100644 index 09eda13307f..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/command.go +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package db - -import ( - mgo "gopkg.in/mgo.v2" -) - -// Query flags -const ( - Snapshot = 1 << iota - LogReplay - Prefetch -) - -type NodeType string - -const ( - Mongos NodeType = "mongos" - Standalone = "standalone" - ReplSet = "replset" - Unknown = "unknown" -) - -// CommandRunner exposes functions that can be run against a server -type CommandRunner interface { - Run(command interface{}, out interface{}, database string) error - FindOne(db, collection string, skip int, query interface{}, sort []string, into interface{}, opts int) error - Remove(db, collection string, query interface{}) error - DatabaseNames() ([]string, error) - CollectionNames(db string) ([]string, error) -} - -// Remove removes all documents matched by query q in the db database and c collection. -func (sp *SessionProvider) Remove(db, c string, q interface{}) error { - session, err := sp.GetSession() - if err != nil { - return err - } - defer session.Close() - _, err = session.DB(db).C(c).RemoveAll(q) - return err -} - -// Run issues the provided command on the db database and unmarshals its result -// into out. -func (sp *SessionProvider) Run(command interface{}, out interface{}, db string) error { - session, err := sp.GetSession() - if err != nil { - return err - } - defer session.Close() - return session.DB(db).Run(command, out) -} - -// DatabaseNames returns a slice containing the names of all the databases on the -// connected server. -func (sp *SessionProvider) DatabaseNames() ([]string, error) { - session, err := sp.GetSession() - if err != nil { - return nil, err - } - session.SetSocketTimeout(0) - defer session.Close() - return session.DatabaseNames() -} - -// CollectionNames returns the names of all the collections in the dbName database. -func (sp *SessionProvider) CollectionNames(dbName string) ([]string, error) { - session, err := sp.GetSession() - if err != nil { - return nil, err - } - defer session.Close() - session.SetSocketTimeout(0) - return session.DB(dbName).CollectionNames() -} - -// GetNodeType checks if the connected SessionProvider is a mongos, standalone, or replset, -// by looking at the result of calling isMaster. -func (sp *SessionProvider) GetNodeType() (NodeType, error) { - session, err := sp.GetSession() - if err != nil { - return Unknown, err - } - session.SetSocketTimeout(0) - defer session.Close() - masterDoc := struct { - SetName interface{} `bson:"setName"` - Hosts interface{} `bson:"hosts"` - Msg string `bson:"msg"` - }{} - err = session.Run("isMaster", &masterDoc) - if err != nil { - return Unknown, err - } - - if masterDoc.SetName != nil || masterDoc.Hosts != nil { - return ReplSet, nil - } else if masterDoc.Msg == "isdbgrid" { - // isdbgrid is always the msg value when calling isMaster on a mongos - // see http://docs.mongodb.org/manual/core/sharded-cluster-query-router/ - return Mongos, nil - } - return Standalone, nil -} - -// IsReplicaSet returns a boolean which is true if the connected server is part -// of a replica set. -func (sp *SessionProvider) IsReplicaSet() (bool, error) { - nodeType, err := sp.GetNodeType() - if err != nil { - return false, err - } - return nodeType == ReplSet, nil -} - -// IsMongos returns true if the connected server is a mongos. -func (sp *SessionProvider) IsMongos() (bool, error) { - nodeType, err := sp.GetNodeType() - if err != nil { - return false, err - } - return nodeType == Mongos, nil -} - -// SupportsCollectionUUID returns true if the connected server identifies -// collections with UUIDs -func (sp *SessionProvider) SupportsCollectionUUID() (bool, error) { - session, err := sp.GetSession() - if err != nil { - return false, err - } - defer session.Close() - - collInfo, err := GetCollectionInfo(session.DB("admin").C("system.version")) - if err != nil { - return false, err - } - - // On FCV 3.6+, admin.system.version will have a UUID - if collInfo != nil && collInfo.GetUUID() != "" { - return true, nil - } - - return false, nil -} - -// SupportsWriteCommands returns true if the connected server supports write -// commands, returns false otherwise. -func (sp *SessionProvider) SupportsWriteCommands() (bool, error) { - session, err := sp.GetSession() - if err != nil { - return false, err - } - session.SetSocketTimeout(0) - defer session.Close() - masterDoc := struct { - Ok int `bson:"ok"` - MaxWire int `bson:"maxWireVersion"` - }{} - err = session.Run("isMaster", &masterDoc) - if err != nil { - return false, err - } - // the connected server supports write commands if - // the maxWriteVersion field is present - return (masterDoc.Ok == 1 && masterDoc.MaxWire >= 2), nil -} - -// FindOne returns the first document in the collection and database that matches -// the query after skip, sort and query flags are applied. -func (sp *SessionProvider) FindOne(db, collection string, skip int, query interface{}, sort []string, into interface{}, flags int) error { - session, err := sp.GetSession() - if err != nil { - return err - } - defer session.Close() - - q := session.DB(db).C(collection).Find(query).Sort(sort...).Skip(skip) - q = ApplyFlags(q, session, flags) - return q.One(into) -} - -// ApplyFlags applies flags to the given query session. -func ApplyFlags(q *mgo.Query, session *mgo.Session, flags int) *mgo.Query { - if flags&Snapshot > 0 { - q = q.Hint("_id") - } - if flags&LogReplay > 0 { - q = q.LogReplay() - } - if flags&Prefetch > 0 { - session.SetPrefetch(1.0) - } - return q -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/connector.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/connector.go deleted file mode 100644 index c95196a0ed5..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/connector.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package db - -import ( - "net" - "time" - - "github.com/mongodb/mongo-tools/common/db/kerberos" - "github.com/mongodb/mongo-tools/common/options" - "github.com/mongodb/mongo-tools/common/util" - "gopkg.in/mgo.v2" -) - -// Interface type for connecting to the database. -type DBConnector interface { - // configure, based on the options passed in - Configure(options.ToolOptions) error - - // dial the database and get a fresh new session - GetNewSession() (*mgo.Session, error) -} - -// Basic connector for dialing the database, with no authentication. -type VanillaDBConnector struct { - dialInfo *mgo.DialInfo -} - -// Configure sets up the db connector using the options in opts. It parses the -// connection string and then sets up the dial information using the default -// dial timeout. -func (self *VanillaDBConnector) Configure(opts options.ToolOptions) error { - timeout := time.Duration(opts.Timeout) * time.Second - - // create the dialer func that will be used to connect - dialer := func(addr *mgo.ServerAddr) (net.Conn, error) { - conn, err := net.DialTimeout("tcp", addr.String(), timeout) - if err != nil { - return nil, err - } - // enable TCP keepalive - err = util.EnableTCPKeepAlive(conn, time.Duration(opts.TCPKeepAliveSeconds)*time.Second) - if err != nil { - return nil, err - } - return conn, nil - } - - // set up the dial info - self.dialInfo = &mgo.DialInfo{ - Direct: opts.Direct, - ReplicaSetName: opts.ReplicaSetName, - Username: opts.Auth.Username, - Password: opts.Auth.Password, - Source: opts.GetAuthenticationDatabase(), - Mechanism: opts.Auth.Mechanism, - DialServer: dialer, - Timeout: timeout, - } - - // create or fetch the addresses to be used to connect - if opts.URI != nil && opts.URI.ConnectionString != "" { - self.dialInfo.Addrs = opts.URI.GetConnectionAddrs() - } else { - self.dialInfo.Addrs = util.CreateConnectionAddrs(opts.Host, opts.Port) - } - - kerberos.AddKerberosOpts(opts, self.dialInfo) - - return nil -} - -// GetNewSession connects to the server and returns the established session and any -// error encountered. -func (self *VanillaDBConnector) GetNewSession() (*mgo.Session, error) { - return mgo.DialWithInfo(self.dialInfo) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/connector_sasl_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/connector_sasl_test.go deleted file mode 100644 index 00c6f640f4a..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/connector_sasl_test.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package db_test - -// This file runs Kerberos tests if the test.types includes 'kerberos' - -import ( - "fmt" - "os" - "runtime" - "testing" - - "github.com/mongodb/mongo-tools/common/db" - "github.com/mongodb/mongo-tools/common/options" - "github.com/mongodb/mongo-tools/common/testtype" - "github.com/mongodb/mongo-tools/common/testutil" - . "github.com/smartystreets/goconvey/convey" - "gopkg.in/mgo.v2/bson" -) - -var ( - KERBEROS_HOST = "ldaptest.10gen.cc" - KERBEROS_USER = "drivers@LDAPTEST.10GEN.CC" -) - -func TestKerberosAuthMechanism(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.KerberosTestType) - - Convey("should be able to successfully connect", t, func() { - connector := &db.VanillaDBConnector{} - - opts := options.ToolOptions{ - Connection: &options.Connection{ - Host: KERBEROS_HOST, - Port: "27017", - }, - Auth: &options.Auth{ - Username: KERBEROS_USER, - Mechanism: "GSSAPI", - }, - Kerberos: &options.Kerberos{ - Service: "mongodb", - ServiceHost: KERBEROS_HOST, - }, - } - - if runtime.GOOS == "windows" { - opts.Auth.Password = os.Getenv(testutil.WinKerberosPwdEnv) - if opts.Auth.Password == "" { - panic(fmt.Sprintf("Need to set %v environment variable to run kerberos tests on windows", - testutil.WinKerberosPwdEnv)) - } - } - - So(connector.Configure(opts), ShouldBeNil) - - session, err := connector.GetNewSession() - So(err, ShouldBeNil) - So(session, ShouldNotBeNil) - - n, err := session.DB("kerberos").C("test").Find(bson.M{}).Count() - So(err, ShouldBeNil) - So(n, ShouldEqual, 1) - }) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/connector_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/connector_test.go deleted file mode 100644 index 18eb9b4b64f..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/connector_test.go +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package db_test - -import ( - "testing" - "time" - - "github.com/mongodb/mongo-tools/common/db" - "github.com/mongodb/mongo-tools/common/options" - "github.com/mongodb/mongo-tools/common/testtype" - "github.com/mongodb/mongo-tools/common/testutil" - . "github.com/smartystreets/goconvey/convey" -) - -func TestVanillaDBConnector(t *testing.T) { - - testtype.SkipUnlessTestType(t, testtype.IntegrationTestType) - // VanillaDBConnector can't connect if SSL or Auth are enabled - if testtype.HasTestType(testtype.SSLTestType) || testtype.HasTestType(testtype.AuthTestType) { - t.SkipNow() - } - - Convey("With a vanilla db connector", t, func() { - - var connector *db.VanillaDBConnector - - Convey("calling Configure should populate the addrs and dial timeout"+ - " appropriately with no error", func() { - - connector = &db.VanillaDBConnector{} - - opts := options.ToolOptions{ - Connection: &options.Connection{ - Host: "host1,host2", - Port: "20000", - }, - Auth: &options.Auth{}, - } - So(connector.Configure(opts), ShouldBeNil) - info := db.VanillaTestWrapper(*connector).DialInfo() - So(info.Addrs, ShouldResemble, - []string{"host1:20000", "host2:20000"}) - So(info.Timeout, ShouldResemble, time.Duration(opts.Timeout)*time.Second) - - }) - - Convey("calling GetNewSession with a running mongod should connect"+ - " successfully", func() { - - connector = &db.VanillaDBConnector{} - - opts := options.ToolOptions{ - Connection: &options.Connection{ - Host: "localhost", - Port: db.DefaultTestPort, - }, - Auth: &options.Auth{}, - } - So(connector.Configure(opts), ShouldBeNil) - - session, err := connector.GetNewSession() - So(err, ShouldBeNil) - So(session, ShouldNotBeNil) - session.Close() - - }) - - }) - -} - -func TestVanillaDBConnectorWithAuth(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.AuthTestType) - // VanillaDBConnector can't connect if SSL is enabled - if testtype.HasTestType(testtype.SSLTestType) { - t.SkipNow() - } - - Convey("With a vanilla db connector and a mongod running with"+ - " auth", t, func() { - - auth := testutil.GetAuthOptions() - - var connector *db.VanillaDBConnector - - Convey("connecting without authentication should not be able"+ - " to run commands", func() { - - connector = &db.VanillaDBConnector{} - - opts := options.ToolOptions{ - Connection: &options.Connection{ - Host: "localhost", - Port: db.DefaultTestPort, - }, - Auth: &options.Auth{}, - } - So(connector.Configure(opts), ShouldBeNil) - - session, err := connector.GetNewSession() - So(err, ShouldBeNil) - So(session, ShouldNotBeNil) - - So(session.DB("admin").Run("top", &struct{}{}), ShouldNotBeNil) - session.Close() - - }) - - Convey("connecting with authentication should succeed and"+ - " authenticate properly", func() { - - connector = &db.VanillaDBConnector{} - - opts := options.ToolOptions{ - Connection: &options.Connection{ - Host: "localhost", - Port: db.DefaultTestPort, - }, - Auth: &auth, - } - So(connector.Configure(opts), ShouldBeNil) - - session, err := connector.GetNewSession() - So(err, ShouldBeNil) - So(session, ShouldNotBeNil) - - So(session.DB("admin").Run("top", &struct{}{}), ShouldBeNil) - session.Close() - - }) - - }) - -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/db.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/db.go deleted file mode 100644 index 839db12d491..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/db.go +++ /dev/null @@ -1,262 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -// Package db implements generic connection to MongoDB, and contains -// subpackages for specific methods of connection. -package db - -import ( - "github.com/mongodb/mongo-tools/common/options" - "github.com/mongodb/mongo-tools/common/password" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" - - "fmt" - "io" - "strings" - "sync" -) - -type ( - sessionFlag uint32 - // Used to get appropriate the DBConnector(s) based on opts - GetConnectorFunc func(opts options.ToolOptions) DBConnector -) - -// Session flags. -const ( - None sessionFlag = 0 - Monotonic sessionFlag = 1 << iota - DisableSocketTimeout -) - -// MongoDB enforced limits. -const ( - MaxBSONSize = 16 * 1024 * 1024 // 16MB - maximum BSON document size -) - -// Default port for integration tests -const ( - DefaultTestPort = "33333" -) - -const ( - ErrLostConnection = "lost connection to server" - ErrNoReachableServers = "no reachable servers" - ErrNsNotFound = "ns not found" - // replication errors list the replset name if we are talking to a mongos, - // so we can only check for this universal prefix - ErrReplTimeoutPrefix = "waiting for replication timed out" - ErrCouldNotContactPrimaryPrefix = "could not contact primary for replica set" - ErrWriteResultsUnavailable = "write results unavailable from" - ErrCouldNotFindPrimaryPrefix = `could not find host matching read preference { mode: "primary"` - ErrUnableToTargetPrefix = "unable to target" - ErrNotMaster = "not master" - ErrConnectionRefusedSuffix = "Connection refused" -) - -var ( - GetConnectorFuncs = []GetConnectorFunc{} -) - -// Used to manage database sessions -type SessionProvider struct { - - // For connecting to the database - connector DBConnector - - // used to avoid a race condition around creating the master session - masterSessionLock sync.Mutex - - // the master session to use for connection pooling - masterSession *mgo.Session - - // flags for generating the master session - bypassDocumentValidation bool - flags sessionFlag - readPreference mgo.Mode - tags bson.D -} - -// ApplyOpsResponse represents the response from an 'applyOps' command. -type ApplyOpsResponse struct { - Ok bool `bson:"ok"` - ErrMsg string `bson:"errmsg"` -} - -// Oplog represents a MongoDB oplog document. -type Oplog struct { - Timestamp bson.MongoTimestamp `bson:"ts"` - HistoryID int64 `bson:"h"` - Version int `bson:"v"` - Operation string `bson:"op"` - Namespace string `bson:"ns"` - Object bson.D `bson:"o"` - Query bson.D `bson:"o2"` - UI *bson.Binary `bson:"ui,omitempty"` -} - -// Returns a session connected to the database server for which the -// session provider is configured. -func (self *SessionProvider) GetSession() (*mgo.Session, error) { - self.masterSessionLock.Lock() - defer self.masterSessionLock.Unlock() - - // The master session is initialized - if self.masterSession != nil { - return self.masterSession.Copy(), nil - } - - // initialize the provider's master session - var err error - self.masterSession, err = self.connector.GetNewSession() - if err != nil { - return nil, fmt.Errorf("error connecting to db server: %v", err) - } - - // update masterSession based on flags - self.refresh() - - // copy the provider's master session, for connection pooling - return self.masterSession.Copy(), nil -} - -// Close closes the master session in the connection pool -func (self *SessionProvider) Close() { - self.masterSessionLock.Lock() - defer self.masterSessionLock.Unlock() - if self.masterSession != nil { - self.masterSession.Close() - } -} - -// refresh is a helper for modifying the session based on the -// session provider flags passed in with SetFlags. -// This helper assumes a lock is already taken. -func (self *SessionProvider) refresh() { - // handle bypassDocumentValidation - self.masterSession.SetBypassValidation(self.bypassDocumentValidation) - - // handle readPreference - self.masterSession.SetMode(self.readPreference, true) - - // disable timeouts - if (self.flags & DisableSocketTimeout) > 0 { - self.masterSession.SetSocketTimeout(0) - } - if self.tags != nil { - self.masterSession.SelectServers(self.tags) - } -} - -// SetFlags allows certain modifications to the masterSession after initial creation. -func (self *SessionProvider) SetFlags(flagBits sessionFlag) { - self.masterSessionLock.Lock() - defer self.masterSessionLock.Unlock() - - self.flags = flagBits - - // make sure we update the master session if one already exists - if self.masterSession != nil { - self.refresh() - } -} - -// SetReadPreference sets the read preference mode in the SessionProvider -// and eventually in the masterSession -func (self *SessionProvider) SetReadPreference(pref mgo.Mode) { - self.masterSessionLock.Lock() - defer self.masterSessionLock.Unlock() - - self.readPreference = pref - - if self.masterSession != nil { - self.refresh() - } -} - -// SetBypassDocumentValidation sets whether to bypass document validation in the SessionProvider -// and eventually in the masterSession -func (self *SessionProvider) SetBypassDocumentValidation(bypassDocumentValidation bool) { - self.masterSessionLock.Lock() - defer self.masterSessionLock.Unlock() - - self.bypassDocumentValidation = bypassDocumentValidation - - if self.masterSession != nil { - self.refresh() - } -} - -// SetTags sets the server selection tags in the SessionProvider -// and eventually in the masterSession -func (self *SessionProvider) SetTags(tags bson.D) { - self.masterSessionLock.Lock() - defer self.masterSessionLock.Unlock() - - self.tags = tags - - if self.masterSession != nil { - self.refresh() - } -} - -// NewSessionProvider constructs a session provider but does not attempt to -// create the initial session. -func NewSessionProvider(opts options.ToolOptions) (*SessionProvider, error) { - // create the provider - provider := &SessionProvider{ - readPreference: mgo.Primary, - bypassDocumentValidation: false, - } - - // finalize auth options, filling in missing passwords - if opts.Auth.ShouldAskForPassword() { - opts.Auth.Password = password.Prompt() - } - - // create the connector for dialing the database - provider.connector = getConnector(opts) - - // configure the connector - err := provider.connector.Configure(opts) - if err != nil { - return nil, fmt.Errorf("error configuring the connector: %v", err) - } - return provider, nil -} - -// IsConnectionError returns a boolean indicating if a given error is due to -// an error in an underlying DB connection (as opposed to some other write -// failure such as a duplicate key error) -func IsConnectionError(err error) bool { - if err == nil { - return false - } - lowerCaseError := strings.ToLower(err.Error()) - if lowerCaseError == ErrNoReachableServers || - err == io.EOF || - strings.Contains(lowerCaseError, ErrReplTimeoutPrefix) || - strings.Contains(lowerCaseError, ErrCouldNotContactPrimaryPrefix) || - strings.Contains(lowerCaseError, ErrWriteResultsUnavailable) || - strings.Contains(lowerCaseError, ErrCouldNotFindPrimaryPrefix) || - strings.Contains(lowerCaseError, ErrUnableToTargetPrefix) || - lowerCaseError == ErrNotMaster || - strings.HasSuffix(lowerCaseError, ErrConnectionRefusedSuffix) { - return true - } - return false -} - -// Get the right type of connector, based on the options -func getConnector(opts options.ToolOptions) DBConnector { - for _, getConnectorFunc := range GetConnectorFuncs { - if connector := getConnectorFunc(opts); connector != nil { - return connector - } - } - return &VanillaDBConnector{} -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/db_openssl.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/db_openssl.go deleted file mode 100644 index 2a7106a068e..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/db_openssl.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -// +build ssl,!openssl_pre_1.0 - -package db - -import ( - "github.com/mongodb/mongo-tools/common/db/openssl" - "github.com/mongodb/mongo-tools/common/options" -) - -func init() { - GetConnectorFuncs = append(GetConnectorFuncs, getSSLConnector) -} - -// return the SSL DB connector if using SSL, otherwise, return nil. -func getSSLConnector(opts options.ToolOptions) DBConnector { - if opts.SSL.UseSSL { - return &openssl.SSLDBConnector{} - } - return nil -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/db_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/db_test.go deleted file mode 100644 index a922bbdff44..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/db_test.go +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package db - -import ( - "reflect" - "testing" - - "github.com/mongodb/mongo-tools/common/options" - "github.com/mongodb/mongo-tools/common/testtype" - . "github.com/smartystreets/goconvey/convey" - "gopkg.in/mgo.v2" -) - -func TestNewSessionProvider(t *testing.T) { - - testtype.SkipUnlessTestType(t, testtype.IntegrationTestType) - auth := DBGetAuthOptions() - ssl := DBGetSSLOptions() - - Convey("When initializing a session provider", t, func() { - - Convey("with the standard options, a provider with a standard"+ - " connector should be returned", func() { - opts := options.ToolOptions{ - Connection: &options.Connection{ - Port: DefaultTestPort, - }, - SSL: &ssl, - Auth: &auth, - } - provider, err := NewSessionProvider(opts) - So(err, ShouldBeNil) - if !testtype.HasTestType(testtype.SSLTestType) { - So(reflect.TypeOf(provider.connector), ShouldEqual, - reflect.TypeOf(&VanillaDBConnector{})) - } - - Convey("and should be closeable", func() { - provider.Close() - }) - - }) - - Convey("the master session should be successfully "+ - " initialized", func() { - opts := options.ToolOptions{ - Connection: &options.Connection{ - Port: DefaultTestPort, - }, - SSL: &ssl, - Auth: &auth, - } - provider, err := NewSessionProvider(opts) - So(err, ShouldBeNil) - So(provider.masterSession, ShouldBeNil) - session, err := provider.GetSession() - So(err, ShouldBeNil) - So(session, ShouldNotBeNil) - session.Close() - So(provider.masterSession, ShouldNotBeNil) - err = provider.masterSession.Ping() - So(err, ShouldBeNil) - provider.Close() - So(func() { - provider.masterSession.Ping() - }, ShouldPanic) - - }) - - }) - -} - -func TestGetIndexes(t *testing.T) { - - testtype.SkipUnlessTestType(t, testtype.IntegrationTestType) - - Convey("With a valid session", t, func() { - auth := DBGetAuthOptions() - ssl := DBGetSSLOptions() - opts := options.ToolOptions{ - Connection: &options.Connection{ - Port: DefaultTestPort, - }, - SSL: &ssl, - Auth: &auth, - } - provider, err := NewSessionProvider(opts) - So(err, ShouldBeNil) - session, err := provider.GetSession() - So(err, ShouldBeNil) - - existing := session.DB("exists").C("collection") - missing := session.DB("exists").C("missing") - missingDB := session.DB("missingDB").C("missingCollection") - - err = existing.Database.DropDatabase() - So(err, ShouldBeNil) - err = existing.Create(&mgo.CollectionInfo{}) - So(err, ShouldBeNil) - err = missingDB.Database.DropDatabase() - So(err, ShouldBeNil) - - Convey("When GetIndexes is called on", func() { - Convey("an existing collection there should be no error", func() { - indexesIter, err := GetIndexes(existing) - So(err, ShouldBeNil) - Convey("and indexes should be returned", func() { - So(indexesIter, ShouldNotBeNil) - var indexes []mgo.Index - err := indexesIter.All(&indexes) - So(err, ShouldBeNil) - So(len(indexes), ShouldBeGreaterThan, 0) - }) - }) - - Convey("a missing collection there should be no error", func() { - indexesIter, err := GetIndexes(missing) - So(err, ShouldBeNil) - Convey("and there should be no indexes", func() { - So(indexesIter, ShouldBeNil) - }) - }) - - Convey("a missing database there should be no error", func() { - indexesIter, err := GetIndexes(missingDB) - So(err, ShouldBeNil) - Convey("and there should be no indexes", func() { - So(indexesIter, ShouldBeNil) - }) - }) - }) - - Reset(func() { - existing.Database.DropDatabase() - session.Close() - provider.Close() - }) - }) -} - -type listDatabasesCommand struct { - Databases []map[string]interface{} `json:"databases"` - Ok bool `json:"ok"` -} - -func (self *listDatabasesCommand) AsRunnable() interface{} { - return "listDatabases" -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/db_tlsgo.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/db_tlsgo.go deleted file mode 100644 index 6fa04a11a60..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/db_tlsgo.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -// +build ssl,openssl_pre_1.0 - -package db - -import ( - "github.com/mongodb/mongo-tools/common/db/tlsgo" - "github.com/mongodb/mongo-tools/common/options" -) - -func init() { - GetConnectorFuncs = append(GetConnectorFuncs, getSSLConnector) -} - -// return the SSL DB connector if using SSL, otherwise, return nil. -func getSSLConnector(opts options.ToolOptions) DBConnector { - if opts.SSL.UseSSL { - return &tlsgo.TLSDBConnector{} - } - return nil -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/export_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/export_test.go deleted file mode 100644 index 7699660290c..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/export_test.go +++ /dev/null @@ -1,12 +0,0 @@ -package db - -import ( - "gopkg.in/mgo.v2" -) - -// VanillaTestWrapper allows black box test access to private fields -type VanillaTestWrapper VanillaDBConnector - -func (v VanillaTestWrapper) DialInfo() *mgo.DialInfo { - return v.dialInfo -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/kerberos/gssapi.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/kerberos/gssapi.go deleted file mode 100644 index 0e6d8b058b1..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/kerberos/gssapi.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -// Package kerberos implements authentication to MongoDB using kerberos -package kerberos - -// #cgo windows CFLAGS: -Ic:/sasl/include -// #cgo windows LDFLAGS: -Lc:/sasl/lib - -import ( - "github.com/mongodb/mongo-tools/common/options" - "gopkg.in/mgo.v2" -) - -const authMechanism = "GSSAPI" - -func AddKerberosOpts(opts options.ToolOptions, dialInfo *mgo.DialInfo) { - if dialInfo == nil { - return - } - if opts.Kerberos == nil { - return - } - if opts.Auth == nil || (opts.Auth.Mechanism != authMechanism && - dialInfo.Mechanism != authMechanism) { - return - } - dialInfo.Service = opts.Kerberos.Service - dialInfo.ServiceHost = opts.Kerberos.ServiceHost - dialInfo.Mechanism = authMechanism -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/namespaces.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/namespaces.go deleted file mode 100644 index b2d5c7e8dda..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/namespaces.go +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package db - -import ( - "encoding/hex" - "fmt" - "strings" - - "github.com/mongodb/mongo-tools/common/log" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" -) - -type CollectionInfo struct { - Name string `bson:"name"` - Type string `bson:"type"` - Options *bson.D `bson:"options"` - Info *bson.D `bson:"info"` -} - -func (ci *CollectionInfo) IsView() bool { - return ci.Type == "view" -} - -func (ci *CollectionInfo) IsSystemCollection() bool { - return strings.HasPrefix(ci.Name, "system.") -} - -func (ci *CollectionInfo) GetUUID() string { - if ci.Info == nil { - return "" - } - for _, v := range *ci.Info { - if v.Name == "uuid" { - switch x := v.Value.(type) { - case bson.Binary: - if x.Kind == 4 { - return hex.EncodeToString(x.Data) - } - } - } - } - return "" -} - -// IsNoCmd returns true if err indicates a query command is not supported, -// otherwise, returns false. -func IsNoCmd(err error) bool { - e, ok := err.(*mgo.QueryError) - return ok && strings.HasPrefix(e.Message, "no such cmd:") -} - -// IsNoNamespace returns true if err indicates a query resulted in a -// "NamespaceNotFound" error otherwise, returns false. -func IsNoNamespace(err error) bool { - e, ok := err.(*mgo.QueryError) - return ok && e.Code == 26 -} - -// buildBsonArray takes a cursor iterator and returns an array of -// all of its documents as bson.D objects. -func buildBsonArray(iter *mgo.Iter) ([]bson.D, error) { - ret := make([]bson.D, 0, 0) - index := new(bson.D) - for iter.Next(index) { - ret = append(ret, *index) - index = new(bson.D) - } - - if iter.Err() != nil { - return nil, iter.Err() - } - return ret, nil - -} - -// GetIndexes returns an iterator to the raw index info for a collection by -// using the listIndexes command if available, or by falling back to querying -// against system.indexes (pre-3.0 systems). nil is returned if the collection -// does not exist. -func GetIndexes(coll *mgo.Collection) (*mgo.Iter, error) { - var cmdResult struct { - Cursor struct { - FirstBatch []bson.Raw `bson:"firstBatch"` - NS string - Id int64 - } - } - - err := coll.Database.Run(bson.D{{"listIndexes", coll.Name}, {"cursor", bson.M{}}}, &cmdResult) - switch { - case err == nil: - ns := strings.SplitN(cmdResult.Cursor.NS, ".", 2) - if len(ns) < 2 { - return nil, fmt.Errorf("server returned invalid cursor.ns `%v` on listIndexes for `%v`: %v", - cmdResult.Cursor.NS, coll.FullName, err) - } - - ses := coll.Database.Session - return ses.DB(ns[0]).C(ns[1]).NewIter(ses, cmdResult.Cursor.FirstBatch, cmdResult.Cursor.Id, nil), nil - case IsNoCmd(err): - log.Logvf(log.DebugLow, "No support for listIndexes command, falling back to querying system.indexes") - return getIndexesPre28(coll) - case IsNoNamespace(err): - return nil, nil - default: - return nil, fmt.Errorf("error running `listIndexes`. Collection: `%v` Err: %v", coll.FullName, err) - } -} - -func getIndexesPre28(coll *mgo.Collection) (*mgo.Iter, error) { - indexColl := coll.Database.C("system.indexes") - iter := indexColl.Find(&bson.M{"ns": coll.FullName}).Iter() - return iter, nil -} - -func GetCollections(database *mgo.Database, name string) (*mgo.Iter, bool, error) { - var cmdResult struct { - Cursor struct { - FirstBatch []bson.Raw `bson:"firstBatch"` - NS string - Id int64 - } - } - - command := bson.D{{"listCollections", 1}, {"cursor", bson.M{}}} - if len(name) > 0 { - command = bson.D{{"listCollections", 1}, {"filter", bson.M{"name": name}}, {"cursor", bson.M{}}} - } - - err := database.Run(command, &cmdResult) - switch { - case err == nil: - ns := strings.SplitN(cmdResult.Cursor.NS, ".", 2) - if len(ns) < 2 { - return nil, false, fmt.Errorf("server returned invalid cursor.ns `%v` on listCollections for `%v`: %v", - cmdResult.Cursor.NS, database.Name, err) - } - - return database.Session.DB(ns[0]).C(ns[1]).NewIter(database.Session, cmdResult.Cursor.FirstBatch, cmdResult.Cursor.Id, nil), false, nil - case IsNoCmd(err): - log.Logvf(log.DebugLow, "No support for listCollections command, falling back to querying system.namespaces") - iter, err := getCollectionsPre28(database, name) - return iter, true, err - default: - return nil, false, fmt.Errorf("error running `listCollections`. Database: `%v` Err: %v", - database.Name, err) - } -} - -func getCollectionsPre28(database *mgo.Database, name string) (*mgo.Iter, error) { - indexColl := database.C("system.namespaces") - selector := bson.M{} - if len(name) > 0 { - selector["name"] = database.Name + "." + name - } - iter := indexColl.Find(selector).Iter() - return iter, nil -} - -func GetCollectionInfo(coll *mgo.Collection) (*CollectionInfo, error) { - iter, useFullName, err := GetCollections(coll.Database, coll.Name) - if err != nil { - return nil, err - } - defer iter.Close() - comparisonName := coll.Name - if useFullName { - comparisonName = coll.FullName - } - - collInfo := &CollectionInfo{} - for iter.Next(collInfo) { - if collInfo.Name == comparisonName { - if useFullName { - collName, err := StripDBFromNamespace(collInfo.Name, coll.Database.Name) - if err != nil { - return nil, err - } - collInfo.Name = collName - } - break - } - } - if err := iter.Err(); err != nil { - return nil, err - } - return collInfo, nil -} - -func StripDBFromNamespace(namespace string, dbName string) (string, error) { - namespacePrefix := dbName + "." - // if the collection info came from querying system.indexes (2.6 or earlier) then the - // "name" we get includes the db name as well, so we must remove it - if strings.HasPrefix(namespace, namespacePrefix) { - return namespace[len(namespacePrefix):], nil - } - return "", fmt.Errorf("namespace '%v' format is invalid - expected to start with '%v'", namespace, namespacePrefix) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/namespaces_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/namespaces_test.go deleted file mode 100644 index 324ef59f151..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/namespaces_test.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package db - -import ( - "fmt" - "testing" - - "github.com/mongodb/mongo-tools/common/testtype" - . "github.com/smartystreets/goconvey/convey" -) - -type stripDBFromNamespaceTestCase struct { - inputNamespace string - inputDBName string - - outputNamespace string - outputError error -} - -func TestStripDBFromNamespace(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - Convey("When testing StripDBFromNamespace with cases", t, func() { - testCases := []stripDBFromNamespaceTestCase{ - { - inputNamespace: "database.col", - inputDBName: "database", - - outputNamespace: "col", - outputError: nil, - }, - { - inputNamespace: "database2.col", - inputDBName: "database", - - outputNamespace: "", - outputError: fmt.Errorf("namespace 'database2.col' format is invalid - expected to start with 'database.'"), - }, - { - inputNamespace: "database.col", - inputDBName: "notAPrefix", - - outputNamespace: "", - outputError: fmt.Errorf("namespace 'database.col' format is invalid - expected to start with 'notAPrefix.'"), - }, - } - Convey("cases should match expected", func() { - for _, tc := range testCases { - resultNamespace, resultError := StripDBFromNamespace(tc.inputNamespace, tc.inputDBName) - So(resultError, ShouldResemble, tc.outputError) - So(resultNamespace, ShouldEqual, tc.outputNamespace) - } - }) - }) - -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/openssl/openssl.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/openssl/openssl.go deleted file mode 100644 index cb744ad5dd0..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/openssl/openssl.go +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -// +build ssl,!openssl_pre_1.0 - -// Package openssl implements connection to MongoDB over ssl. -package openssl - -import ( - "fmt" - "net" - "time" - - "github.com/10gen/openssl" - "github.com/mongodb/mongo-tools/common/db/kerberos" - "github.com/mongodb/mongo-tools/common/log" - "github.com/mongodb/mongo-tools/common/options" - "github.com/mongodb/mongo-tools/common/util" - "gopkg.in/mgo.v2" -) - -// For connecting to the database over ssl -type SSLDBConnector struct { - dialInfo *mgo.DialInfo - ctx *openssl.Ctx -} - -// Configure the connector to connect to the server over ssl. Parses the -// connection string, and sets up the correct function to dial the server -// based on the ssl options passed in. -func (self *SSLDBConnector) Configure(opts options.ToolOptions) error { - - var err error - self.ctx, err = setupCtx(opts) - if err != nil { - return fmt.Errorf("openssl configuration: %v", err) - } - - var flags openssl.DialFlags - flags = 0 - if opts.SSLAllowInvalidCert || opts.SSLAllowInvalidHost { - flags = openssl.InsecureSkipHostVerification - } - // create the dialer func that will be used to connect - dialer := func(addr *mgo.ServerAddr) (net.Conn, error) { - conn, err := openssl.Dial("tcp", addr.String(), self.ctx, flags) - if err != nil { - // mgo discards dialer errors so log it now - log.Logvf(log.Always, "error dialing %v: %v", addr.String(), err) - return nil, err - } - // enable TCP keepalive - err = util.EnableTCPKeepAlive(conn.UnderlyingConn(), time.Duration(opts.TCPKeepAliveSeconds)*time.Second) - if err != nil { - // mgo discards dialer errors so log it now - log.Logvf(log.Always, "error enabling TCP keepalive on connection to %v: %v", addr.String(), err) - conn.Close() - return nil, err - } - return conn, nil - } - - timeout := time.Duration(opts.Timeout) * time.Second - - // set up the dial info - self.dialInfo = &mgo.DialInfo{ - Timeout: timeout, - Direct: opts.Direct, - ReplicaSetName: opts.ReplicaSetName, - DialServer: dialer, - Username: opts.Auth.Username, - Password: opts.Auth.Password, - Source: opts.GetAuthenticationDatabase(), - Mechanism: opts.Auth.Mechanism, - } - - // create or fetch the addresses to be used to connect - if opts.URI != nil && opts.URI.ConnectionString != "" { - self.dialInfo.Addrs = opts.URI.GetConnectionAddrs() - } else { - self.dialInfo.Addrs = util.CreateConnectionAddrs(opts.Host, opts.Port) - } - kerberos.AddKerberosOpts(opts, self.dialInfo) - return nil - -} - -// Dial the server. -func (self *SSLDBConnector) GetNewSession() (*mgo.Session, error) { - return mgo.DialWithInfo(self.dialInfo) -} - -// To be handed to mgo.DialInfo for connecting to the server. -type dialerFunc func(addr *mgo.ServerAddr) (net.Conn, error) - -// Handle optionally compiled SSL initialization functions (fips mode set) -type sslInitializationFunction func(options.ToolOptions) error - -var sslInitializationFunctions []sslInitializationFunction - -// Creates and configures an openssl.Ctx -func setupCtx(opts options.ToolOptions) (*openssl.Ctx, error) { - var ctx *openssl.Ctx - var err error - - for _, sslInitFunc := range sslInitializationFunctions { - err = sslInitFunc(opts) - if err != nil { - return nil, err - } - } - - if ctx, err = openssl.NewCtxWithVersion(openssl.AnyVersion); err != nil { - return nil, fmt.Errorf("failure creating new openssl context with "+ - "NewCtxWithVersion(AnyVersion): %v", err) - } - - // OpAll - Activate all bug workaround options, to support buggy client SSL's. - // NoSSLv2 - Disable SSL v2 support - ctx.SetOptions(openssl.OpAll | openssl.NoSSLv2) - - // HIGH - Enable strong ciphers - // !EXPORT - Disable export ciphers (40/56 bit) - // !aNULL - Disable anonymous auth ciphers - // @STRENGTH - Sort ciphers based on strength - ctx.SetCipherList("HIGH:!EXPORT:!aNULL@STRENGTH") - - // add the PEM key file with the cert and private key, if specified - if opts.SSLPEMKeyFile != "" { - if err = ctx.UseCertificateChainFile(opts.SSLPEMKeyFile); err != nil { - return nil, fmt.Errorf("UseCertificateChainFile: %v", err) - } - if opts.SSLPEMKeyPassword != "" { - if err = ctx.UsePrivateKeyFileWithPassword( - opts.SSLPEMKeyFile, openssl.FiletypePEM, opts.SSLPEMKeyPassword); err != nil { - return nil, fmt.Errorf("UsePrivateKeyFile: %v", err) - } - } else { - if err = ctx.UsePrivateKeyFile(opts.SSLPEMKeyFile, openssl.FiletypePEM); err != nil { - return nil, fmt.Errorf("UsePrivateKeyFile: %v", err) - } - } - // Verify that the certificate and the key go together. - if err = ctx.CheckPrivateKey(); err != nil { - return nil, fmt.Errorf("CheckPrivateKey: %v", err) - } - } - - // If renegotiation is needed, don't return from recv() or send() until it's successful. - // Note: this is for blocking sockets only. - ctx.SetMode(openssl.AutoRetry) - - // Disable session caching (see SERVER-10261) - ctx.SetSessionCacheMode(openssl.SessionCacheOff) - - if opts.SSLCAFile != "" { - calist, err := openssl.LoadClientCAFile(opts.SSLCAFile) - if err != nil { - return nil, fmt.Errorf("LoadClientCAFile: %v", err) - } - ctx.SetClientCAList(calist) - if err = ctx.LoadVerifyLocations(opts.SSLCAFile, ""); err != nil { - return nil, fmt.Errorf("LoadVerifyLocations: %v", err) - } - } else { - err = ctx.SetupSystemCA() - if err != nil { - return nil, fmt.Errorf("Error setting up system certificate authority: %v", err) - } - } - - var verifyOption openssl.VerifyOptions - if opts.SSLAllowInvalidCert { - verifyOption = openssl.VerifyNone - } else { - verifyOption = openssl.VerifyPeer - } - ctx.SetVerify(verifyOption, nil) - - if opts.SSLCRLFile != "" { - store := ctx.GetCertificateStore() - store.SetFlags(openssl.CRLCheck) - lookup, err := store.AddLookup(openssl.X509LookupFile()) - if err != nil { - return nil, fmt.Errorf("AddLookup(X509LookupFile()): %v", err) - } - lookup.LoadCRLFile(opts.SSLCRLFile) - } - - return ctx, nil -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/openssl/openssl_fips.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/openssl/openssl_fips.go deleted file mode 100644 index 68bf0a92d3d..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/openssl/openssl_fips.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -// +build ssl,!openssl_pre_1.0 - -package openssl - -import ( - "fmt" - - "github.com/10gen/openssl" - "github.com/mongodb/mongo-tools/common/options" -) - -func init() { - if openssl.FIPSModeDefined() { - sslInitializationFunctions = append(sslInitializationFunctions, SetUpFIPSMode) - } else { - sslInitializationFunctions = append(sslInitializationFunctions, NoFIPSModeAvailable) - } -} - -func SetUpFIPSMode(opts options.ToolOptions) error { - if err := openssl.FIPSModeSet(opts.SSLFipsMode); err != nil { - return fmt.Errorf("couldn't set FIPS mode to %v: %v", opts.SSLFipsMode, err) - } - return nil -} - -func NoFIPSModeAvailable(opts options.ToolOptions) error { - if opts.SSLFipsMode { - return fmt.Errorf("FIPS mode not supported") - } - return nil -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/read_preferences.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/read_preferences.go deleted file mode 100644 index a0fba3d611a..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/read_preferences.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package db - -import ( - "fmt" - - "github.com/mongodb/mongo-tools/common/json" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" -) - -type readPrefDoc struct { - Mode string - Tags bson.D -} - -const ( - WarningNonPrimaryMongosConnection = "Warning: using a non-primary readPreference with a " + - "connection to mongos may produce inconsistent duplicates or miss some documents." -) - -func ParseReadPreference(rp string) (mgo.Mode, bson.D, error) { - var mode string - var tags bson.D - if rp == "" { - return mgo.Nearest, nil, nil - } - if rp[0] != '{' { - mode = rp - } else { - var doc readPrefDoc - err := json.Unmarshal([]byte(rp), &doc) - if err != nil { - return 0, nil, fmt.Errorf("invalid --ReadPreferences json object: %v", err) - } - tags = doc.Tags - mode = doc.Mode - } - switch mode { - case "primary": - return mgo.Primary, tags, nil - case "primaryPreferred": - return mgo.PrimaryPreferred, tags, nil - case "secondary": - return mgo.Secondary, tags, nil - case "secondaryPreferred": - return mgo.SecondaryPreferred, tags, nil - case "nearest": - return mgo.Nearest, tags, nil - } - return 0, nil, fmt.Errorf("invalid readPreference mode '%v'", mode) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/config_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/config_test.go deleted file mode 100644 index be46a87dcf7..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/config_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package tlsgo - -import ( - "strings" - "testing" - - "github.com/mongodb/mongo-tools/common/testtype" -) - -func TestAddClientCert(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - cases := []struct { - Path string - Pass string - Valid bool - }{ - {Path: "testdata/pkcs1.pem", Valid: true}, - {Path: "testdata/pkcs1-rev.pem", Valid: true}, - {Path: "testdata/pkcs1-encrypted.pem", Pass: "qwerty", Valid: true}, - {Path: "testdata/pkcs1-encrypted-rev.pem", Pass: "qwerty", Valid: true}, - - {Path: "testdata/pkcs8.pem", Valid: true}, - {Path: "testdata/pkcs8-rev.pem", Valid: true}, - {Path: "testdata/pkcs8-encrypted.pem", Valid: false}, - {Path: "testdata/pkcs8-encrypted-rev.pem", Valid: false}, - } - - for _, v := range cases { - tlsc := NewTLSConfig() - _, err := tlsc.AddClientCertFromFile(v.Path, v.Pass) - switch v.Valid { - case true: - if err != nil { - t.Errorf("Error parsing %s: %s", v.Path, err.Error()) - } - case false: - if err == nil { - t.Errorf("Expected error parsing %s but parsed OK", v.Path) - } else if !strings.Contains(err.Error(), "encrypted private keys are not supported") { - t.Errorf("Incorrect error for %s: %s", v.Path, err.Error()) - } - } - } -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs1-encrypted-rev.pem b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs1-encrypted-rev.pem deleted file mode 100644 index 308e2263d4a..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs1-encrypted-rev.pem +++ /dev/null @@ -1,51 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDfjCCAmagAwIBAgIDBUEVMA0GCSqGSIb3DQEBBQUAMHQxFzAVBgNVBAMTDktl -cm5lbCBUZXN0IENBMQ8wDQYDVQQLEwZLZXJuZWwxEDAOBgNVBAoTB01vbmdvREIx -FjAUBgNVBAcTDU5ldyBZb3JrIENpdHkxETAPBgNVBAgTCE5ldyBZb3JrMQswCQYD -VQQGEwJVUzAeFw0xNjA5MjIxODE1MTJaFw0zNjA5MjIxODE1MTJaMG8xEjAQBgNV -BAMTCWxvY2FsaG9zdDEPMA0GA1UECxMGS2VybmVsMRAwDgYDVQQKEwdNb25nb0RC -MRYwFAYDVQQHEw1OZXcgWW9yayBDaXR5MREwDwYDVQQIEwhOZXcgWW9yazELMAkG -A1UEBhMCVVMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCTHMXV0LEZ -OCuDZ292e26NbbrMaib6IL3obp/5tOvNVCNnvfgYyJwCCTIZq/mwCjAV5N8Y7tJM -v0JrrGIWgJ3qtPMQ/1VxfzLLW598nnBuqZG2HiR3CTfhd0JBmnjKDMscz90+xB2x -DUDVe6PkbZWnN2otsBzVbW+AAJRVTgUb3cjSbGcC0eTMg3SGaWiB+DtiJIAe3bl8 -6TTmrUKVvbzbJrdrFWpz+NVxf5ejZje+Wlz6OXgkWki5U41PtA7aDFIX3mo1J3c0 -jW957fC/q76jrBoTCbufYPaLQIb5QSex+aJZ40rHpSSV75tsXNUkn22u83Bes+Ih -X0As7g5kW2TDAgMBAAGjHjAcMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATAN -BgkqhkiG9w0BAQUFAAOCAQEAEDzWG64/IlXSEFQZom1z0uBLSLVaxrNg4se6geLH -Bt63EW78H+JMf97AA32DsDiT3ih5uo8yUcOVoEUwontUOSjekHrYfagF/KxMvyMy -sWX+8m5SLrU6s4FysUCtlXa92g1Nh/rET074U2sNShhALgNB2XSw9P5n9GnKt5VT -Rkh0AeBJd09WcOGnSHs30+kKGNV8A5a2GTJbDma0dLa7zlhV6VU91Z9LA0aamyrX -eWwnymJvRcIYvxGqgNDxN/8MsaU1EcW0MNEDkc+kDE1LbOwlAQbCeLQDq/w6AlmC -smoCi0pp6Bf8tZM2RhcUN/xXxgEKcZzhlDOI4v8RNHOyMg== ------END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: AES-128-CBC,B57A03692CDD397E50317A829B4A4698 - -V/V8LyrTJtyxBZYcodeO7xyS/+pmjmbEEYWC6ugP+MgeStTINrjfiYbc3QPkfUEg -SpWFgeq6rFbnszeWrcuk9U0NCv+vg3SjMuprrisCJerpv9bCldF9lbbqElL6o3ov -Q3EIS5JJWxXOJN/FAvTF4nNhh+0aasMmnZyHZMsT2aqBrswDQ7h51wCV4IRHk5Xr -StqjV314kQHMRQfybYkPKZkABtpghSLGGLguwch1cbKPCKHTinFuIuZGHJlQmnOz -tfXtnjlrAG7LtyfddrTlWkm/fGF6lhewC15HLLgpNVkLmFtHCyOtDVTkInT6CM+x -DaDnXebj4gghvJ0kmm7uX3rLX/pvnne+iNpNLaZcjVx20+iGhYaJdy4yUq+nH/UU -+dHlyublzcsDHmZG8CX297DT5kRgkH6Nh3VQdhygQNRNCEHQbR8Gsff/3bJ+KDO8 -6vw/xtcjnbIsOVM8Wxp+lkvKmwk+tTVEhL4bG/+6sq1Cd9jDnf0fzWx7t+IA4t/Y -OJ2K65T6I7QVgu0y3jSyLN1MH3oLPF3VGlF6NZlRZUObDL/HzWCFWCpBMtdAxfjI -Wxh5QyQix5lo9IuvYMYmCGk6d+N/fhpLmp3mcURkZrSZCIvfLFF7jrlO4z68j0Os -XODkuYgBXhHKf+tYc0Scokd5cbHlLZ986ngPsSClTtdovouHMxRfWoLQBdlXvxi0 -CjC7SRPuvLSSRLXzF72Htgb7U/W+JflSwvpZrO8VJ7ngR4sU2s1fO1K7x+fLIHEx -M1V6OTQfmJoumg6DIYqAqO8QD6JVIn+JfZ8Ympt7zFaPCJtpxxmKjmpQ1BWatDP6 -dLrdxW9uV6VKYBQuVv+k+jFcjNMrRfJHfeUxrOjCIo3dUDfju+DOdJUAMxWPzdZZ -OmcTG/4AIzw0BJirIAuAsz1RE3V8UXjefnO3YOBZMJPx22iBOacRtcYZXX5Vi/hs -UMmBWrjrsgmtb8KxIvDED3fnfWI6JdK92x+yIJAOB920z//XP1XmiLV6QjwXgXIw -g85ZceCh7Z6E62GYRQ3xboelbKlOzeRXqwM9Tz75677pqnloeEZfN/0GCABX4SAi -jDmb1dt9DiwHsVnt2zvY85V14qNq5QkCTkD+34l+ASLrwgYj8iJ8f3NQMXvBatY0 -eKUonwjSD0odxgvgdwvGlsx1++ec6TWB7jUD/dLxPqPy+m+SpsrxmJG9/WxFXIA/ -UHcW8n5xy1D1mKgeGxTPgWLwYlbcLD3HBaIFj6s9vDfP7+ztcg6Xdsslf8irHByp -JZgnG3ptQFAVEftM7oWvM2eXdBp1mgxuSGgZohURNOAdW0m5VEsmMp95k/iN4vXI -+aTYuVmeWJhQY9pvRW38RDhwxBXIiN2dCkijUPHCi7fc1k9ox06rGsX3doW6UBu+ -H45w0BTVpJR8kv7y+Ep5yd0VTKnGy16PVL/K9GqNahzwb72JxLP+hI4Amlp7rSAG -Mfq0O3SvSrDks5PsPgBHEKnBfMMgKgTQOWICLtFG7Xoh1aJA9ykge2TniaUZeRuN -Wm4FEDBqhCEZpOOFdtq/P9v8KV/IDuyMhFEMb6tSn9P6EDTIS7feJnhXn7JFMdJT ------END RSA PRIVATE KEY----- diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs1-encrypted.pem b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs1-encrypted.pem deleted file mode 100644 index fa92cebe1d7..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs1-encrypted.pem +++ /dev/null @@ -1,51 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: AES-128-CBC,B57A03692CDD397E50317A829B4A4698 - -V/V8LyrTJtyxBZYcodeO7xyS/+pmjmbEEYWC6ugP+MgeStTINrjfiYbc3QPkfUEg -SpWFgeq6rFbnszeWrcuk9U0NCv+vg3SjMuprrisCJerpv9bCldF9lbbqElL6o3ov -Q3EIS5JJWxXOJN/FAvTF4nNhh+0aasMmnZyHZMsT2aqBrswDQ7h51wCV4IRHk5Xr -StqjV314kQHMRQfybYkPKZkABtpghSLGGLguwch1cbKPCKHTinFuIuZGHJlQmnOz -tfXtnjlrAG7LtyfddrTlWkm/fGF6lhewC15HLLgpNVkLmFtHCyOtDVTkInT6CM+x -DaDnXebj4gghvJ0kmm7uX3rLX/pvnne+iNpNLaZcjVx20+iGhYaJdy4yUq+nH/UU -+dHlyublzcsDHmZG8CX297DT5kRgkH6Nh3VQdhygQNRNCEHQbR8Gsff/3bJ+KDO8 -6vw/xtcjnbIsOVM8Wxp+lkvKmwk+tTVEhL4bG/+6sq1Cd9jDnf0fzWx7t+IA4t/Y -OJ2K65T6I7QVgu0y3jSyLN1MH3oLPF3VGlF6NZlRZUObDL/HzWCFWCpBMtdAxfjI -Wxh5QyQix5lo9IuvYMYmCGk6d+N/fhpLmp3mcURkZrSZCIvfLFF7jrlO4z68j0Os -XODkuYgBXhHKf+tYc0Scokd5cbHlLZ986ngPsSClTtdovouHMxRfWoLQBdlXvxi0 -CjC7SRPuvLSSRLXzF72Htgb7U/W+JflSwvpZrO8VJ7ngR4sU2s1fO1K7x+fLIHEx -M1V6OTQfmJoumg6DIYqAqO8QD6JVIn+JfZ8Ympt7zFaPCJtpxxmKjmpQ1BWatDP6 -dLrdxW9uV6VKYBQuVv+k+jFcjNMrRfJHfeUxrOjCIo3dUDfju+DOdJUAMxWPzdZZ -OmcTG/4AIzw0BJirIAuAsz1RE3V8UXjefnO3YOBZMJPx22iBOacRtcYZXX5Vi/hs -UMmBWrjrsgmtb8KxIvDED3fnfWI6JdK92x+yIJAOB920z//XP1XmiLV6QjwXgXIw -g85ZceCh7Z6E62GYRQ3xboelbKlOzeRXqwM9Tz75677pqnloeEZfN/0GCABX4SAi -jDmb1dt9DiwHsVnt2zvY85V14qNq5QkCTkD+34l+ASLrwgYj8iJ8f3NQMXvBatY0 -eKUonwjSD0odxgvgdwvGlsx1++ec6TWB7jUD/dLxPqPy+m+SpsrxmJG9/WxFXIA/ -UHcW8n5xy1D1mKgeGxTPgWLwYlbcLD3HBaIFj6s9vDfP7+ztcg6Xdsslf8irHByp -JZgnG3ptQFAVEftM7oWvM2eXdBp1mgxuSGgZohURNOAdW0m5VEsmMp95k/iN4vXI -+aTYuVmeWJhQY9pvRW38RDhwxBXIiN2dCkijUPHCi7fc1k9ox06rGsX3doW6UBu+ -H45w0BTVpJR8kv7y+Ep5yd0VTKnGy16PVL/K9GqNahzwb72JxLP+hI4Amlp7rSAG -Mfq0O3SvSrDks5PsPgBHEKnBfMMgKgTQOWICLtFG7Xoh1aJA9ykge2TniaUZeRuN -Wm4FEDBqhCEZpOOFdtq/P9v8KV/IDuyMhFEMb6tSn9P6EDTIS7feJnhXn7JFMdJT ------END RSA PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIDfjCCAmagAwIBAgIDBUEVMA0GCSqGSIb3DQEBBQUAMHQxFzAVBgNVBAMTDktl -cm5lbCBUZXN0IENBMQ8wDQYDVQQLEwZLZXJuZWwxEDAOBgNVBAoTB01vbmdvREIx -FjAUBgNVBAcTDU5ldyBZb3JrIENpdHkxETAPBgNVBAgTCE5ldyBZb3JrMQswCQYD -VQQGEwJVUzAeFw0xNjA5MjIxODE1MTJaFw0zNjA5MjIxODE1MTJaMG8xEjAQBgNV -BAMTCWxvY2FsaG9zdDEPMA0GA1UECxMGS2VybmVsMRAwDgYDVQQKEwdNb25nb0RC -MRYwFAYDVQQHEw1OZXcgWW9yayBDaXR5MREwDwYDVQQIEwhOZXcgWW9yazELMAkG -A1UEBhMCVVMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCTHMXV0LEZ -OCuDZ292e26NbbrMaib6IL3obp/5tOvNVCNnvfgYyJwCCTIZq/mwCjAV5N8Y7tJM -v0JrrGIWgJ3qtPMQ/1VxfzLLW598nnBuqZG2HiR3CTfhd0JBmnjKDMscz90+xB2x -DUDVe6PkbZWnN2otsBzVbW+AAJRVTgUb3cjSbGcC0eTMg3SGaWiB+DtiJIAe3bl8 -6TTmrUKVvbzbJrdrFWpz+NVxf5ejZje+Wlz6OXgkWki5U41PtA7aDFIX3mo1J3c0 -jW957fC/q76jrBoTCbufYPaLQIb5QSex+aJZ40rHpSSV75tsXNUkn22u83Bes+Ih -X0As7g5kW2TDAgMBAAGjHjAcMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATAN -BgkqhkiG9w0BAQUFAAOCAQEAEDzWG64/IlXSEFQZom1z0uBLSLVaxrNg4se6geLH -Bt63EW78H+JMf97AA32DsDiT3ih5uo8yUcOVoEUwontUOSjekHrYfagF/KxMvyMy -sWX+8m5SLrU6s4FysUCtlXa92g1Nh/rET074U2sNShhALgNB2XSw9P5n9GnKt5VT -Rkh0AeBJd09WcOGnSHs30+kKGNV8A5a2GTJbDma0dLa7zlhV6VU91Z9LA0aamyrX -eWwnymJvRcIYvxGqgNDxN/8MsaU1EcW0MNEDkc+kDE1LbOwlAQbCeLQDq/w6AlmC -smoCi0pp6Bf8tZM2RhcUN/xXxgEKcZzhlDOI4v8RNHOyMg== ------END CERTIFICATE----- diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs1-rev.pem b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs1-rev.pem deleted file mode 100644 index 0bb7b967c9d..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs1-rev.pem +++ /dev/null @@ -1,48 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDfjCCAmagAwIBAgIDBUEVMA0GCSqGSIb3DQEBBQUAMHQxFzAVBgNVBAMTDktl -cm5lbCBUZXN0IENBMQ8wDQYDVQQLEwZLZXJuZWwxEDAOBgNVBAoTB01vbmdvREIx -FjAUBgNVBAcTDU5ldyBZb3JrIENpdHkxETAPBgNVBAgTCE5ldyBZb3JrMQswCQYD -VQQGEwJVUzAeFw0xNjA5MjIxODE1MTJaFw0zNjA5MjIxODE1MTJaMG8xEjAQBgNV -BAMTCWxvY2FsaG9zdDEPMA0GA1UECxMGS2VybmVsMRAwDgYDVQQKEwdNb25nb0RC -MRYwFAYDVQQHEw1OZXcgWW9yayBDaXR5MREwDwYDVQQIEwhOZXcgWW9yazELMAkG -A1UEBhMCVVMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCTHMXV0LEZ -OCuDZ292e26NbbrMaib6IL3obp/5tOvNVCNnvfgYyJwCCTIZq/mwCjAV5N8Y7tJM -v0JrrGIWgJ3qtPMQ/1VxfzLLW598nnBuqZG2HiR3CTfhd0JBmnjKDMscz90+xB2x -DUDVe6PkbZWnN2otsBzVbW+AAJRVTgUb3cjSbGcC0eTMg3SGaWiB+DtiJIAe3bl8 -6TTmrUKVvbzbJrdrFWpz+NVxf5ejZje+Wlz6OXgkWki5U41PtA7aDFIX3mo1J3c0 -jW957fC/q76jrBoTCbufYPaLQIb5QSex+aJZ40rHpSSV75tsXNUkn22u83Bes+Ih -X0As7g5kW2TDAgMBAAGjHjAcMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATAN -BgkqhkiG9w0BAQUFAAOCAQEAEDzWG64/IlXSEFQZom1z0uBLSLVaxrNg4se6geLH -Bt63EW78H+JMf97AA32DsDiT3ih5uo8yUcOVoEUwontUOSjekHrYfagF/KxMvyMy -sWX+8m5SLrU6s4FysUCtlXa92g1Nh/rET074U2sNShhALgNB2XSw9P5n9GnKt5VT -Rkh0AeBJd09WcOGnSHs30+kKGNV8A5a2GTJbDma0dLa7zlhV6VU91Z9LA0aamyrX -eWwnymJvRcIYvxGqgNDxN/8MsaU1EcW0MNEDkc+kDE1LbOwlAQbCeLQDq/w6AlmC -smoCi0pp6Bf8tZM2RhcUN/xXxgEKcZzhlDOI4v8RNHOyMg== ------END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAkxzF1dCxGTgrg2dvdntujW26zGom+iC96G6f+bTrzVQjZ734 -GMicAgkyGav5sAowFeTfGO7STL9Ca6xiFoCd6rTzEP9VcX8yy1uffJ5wbqmRth4k -dwk34XdCQZp4ygzLHM/dPsQdsQ1A1Xuj5G2VpzdqLbAc1W1vgACUVU4FG93I0mxn -AtHkzIN0hmlogfg7YiSAHt25fOk05q1Clb282ya3axVqc/jVcX+Xo2Y3vlpc+jl4 -JFpIuVONT7QO2gxSF95qNSd3NI1vee3wv6u+o6waEwm7n2D2i0CG+UEnsfmiWeNK -x6Ukle+bbFzVJJ9trvNwXrPiIV9ALO4OZFtkwwIDAQABAoIBAAuueTclPyrVfv8M -M5mg64JneDHLLBUojGvsfN+DMkY3rCgMuaqeI2U1/bh0I3uLE45pgh2kuSZG+as7 -IP7Qb7m3bKWo4MwGYa4sNFnc6uiepmdjtVmObdWFdslmzrick3RSPStCv2jTuJY7 -HySAXyXMDK5cEa4Q5o2vfhfX/ifcMZrS2Pz1o4k2Wh8EtzmRxJR+QR8d+XLtVsKf -WIvtlhwGqWkmocFOsWW/6Mtf7IPWC3QAPjVYNcxe/8LSE/FhnRr7L6Uv1K7vGImE -/+QVScl5sP2bpvo+9LxzOMANMdTWWX5ZZJhIdvwpsyctcZovuJq/Lrh9A0j40nRJ -LuR6wUECgYEA2AgCKimqgpf7WCZMv72Kbur2banm1nwRsnPENGK4e6ZuYwHXu5n6 -HLgk/zp2lJdE4yGr8EBE5hvoFCosxEuvF2ldlqnKDqRUC9IKNtXJEisadWCEPmOp -v04zPaV5hWOXaK3ZoCQ7D8xvzThcZderMMdGoeacv050nJnSkPEhissCgYEArlSG -x2KRa1AvAYwMnEIeABrzjSzLLPHyYmCByouo3ljjWiBu7gSsJCO+O8QBjIklpW24 -g+Cek8d9X3oMw2aKKukgecxTR/XE7StB6RXngEIWvIqLj3CNWn1l+K/F95rrQrBr -6Fea6qWnMYeZrnuGGvBX7PwjJncE1nvn+ey/9ukCgYB/1N1TDayz8jLsil1H6GSO -FcMUSUErEed27UHgrbn0kRsowuJhRE/Xxq89x957NrewnzAaziz27PR7Wil7Tj2h -YNvcV0QVPe/tvrAEmqSMd60EX8RhFqBPb3qqs8wgvjnN23G3bTj1tEdD7GHgcan/ -BywoiUmfelFOiUcsNUNf2QKBgBUuig3R6S9r17pNZP7bfb+vhqZBqhI096mCZmLQ -41zY2g8KX9Al2zCs8yFZ6IJF68AU+9VyRnJYS+B8+O4JGIKsPtjtvbTBpQLYPbLv -iWhpH1AbWWe9Wj+Dew9jdB9owGsi+omJk3YtWIpJGqA7vAir6VMPM8oprfnNplsU -rCJ5AoGBANRKxMsriiA/sDLPxDCQOZg4JGRy1ycRVu75wETwoWnDMUP6J6BxE/pK -keA1nmrVXLheVs3kB7Bg7Jm+53E6RPATRGbvJ+5nqtDDxjL3HL8Jg6uKFjPmnpJ4 -crTsbc7nrAxo0cRmUlgbzQqhgAfnb8B7Fai2T1qoFixPtIFicehx ------END RSA PRIVATE KEY----- diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs1.pem b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs1.pem deleted file mode 100644 index 9f6124b5fa2..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs1.pem +++ /dev/null @@ -1,48 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAkxzF1dCxGTgrg2dvdntujW26zGom+iC96G6f+bTrzVQjZ734 -GMicAgkyGav5sAowFeTfGO7STL9Ca6xiFoCd6rTzEP9VcX8yy1uffJ5wbqmRth4k -dwk34XdCQZp4ygzLHM/dPsQdsQ1A1Xuj5G2VpzdqLbAc1W1vgACUVU4FG93I0mxn -AtHkzIN0hmlogfg7YiSAHt25fOk05q1Clb282ya3axVqc/jVcX+Xo2Y3vlpc+jl4 -JFpIuVONT7QO2gxSF95qNSd3NI1vee3wv6u+o6waEwm7n2D2i0CG+UEnsfmiWeNK -x6Ukle+bbFzVJJ9trvNwXrPiIV9ALO4OZFtkwwIDAQABAoIBAAuueTclPyrVfv8M -M5mg64JneDHLLBUojGvsfN+DMkY3rCgMuaqeI2U1/bh0I3uLE45pgh2kuSZG+as7 -IP7Qb7m3bKWo4MwGYa4sNFnc6uiepmdjtVmObdWFdslmzrick3RSPStCv2jTuJY7 -HySAXyXMDK5cEa4Q5o2vfhfX/ifcMZrS2Pz1o4k2Wh8EtzmRxJR+QR8d+XLtVsKf -WIvtlhwGqWkmocFOsWW/6Mtf7IPWC3QAPjVYNcxe/8LSE/FhnRr7L6Uv1K7vGImE -/+QVScl5sP2bpvo+9LxzOMANMdTWWX5ZZJhIdvwpsyctcZovuJq/Lrh9A0j40nRJ -LuR6wUECgYEA2AgCKimqgpf7WCZMv72Kbur2banm1nwRsnPENGK4e6ZuYwHXu5n6 -HLgk/zp2lJdE4yGr8EBE5hvoFCosxEuvF2ldlqnKDqRUC9IKNtXJEisadWCEPmOp -v04zPaV5hWOXaK3ZoCQ7D8xvzThcZderMMdGoeacv050nJnSkPEhissCgYEArlSG -x2KRa1AvAYwMnEIeABrzjSzLLPHyYmCByouo3ljjWiBu7gSsJCO+O8QBjIklpW24 -g+Cek8d9X3oMw2aKKukgecxTR/XE7StB6RXngEIWvIqLj3CNWn1l+K/F95rrQrBr -6Fea6qWnMYeZrnuGGvBX7PwjJncE1nvn+ey/9ukCgYB/1N1TDayz8jLsil1H6GSO -FcMUSUErEed27UHgrbn0kRsowuJhRE/Xxq89x957NrewnzAaziz27PR7Wil7Tj2h -YNvcV0QVPe/tvrAEmqSMd60EX8RhFqBPb3qqs8wgvjnN23G3bTj1tEdD7GHgcan/ -BywoiUmfelFOiUcsNUNf2QKBgBUuig3R6S9r17pNZP7bfb+vhqZBqhI096mCZmLQ -41zY2g8KX9Al2zCs8yFZ6IJF68AU+9VyRnJYS+B8+O4JGIKsPtjtvbTBpQLYPbLv -iWhpH1AbWWe9Wj+Dew9jdB9owGsi+omJk3YtWIpJGqA7vAir6VMPM8oprfnNplsU -rCJ5AoGBANRKxMsriiA/sDLPxDCQOZg4JGRy1ycRVu75wETwoWnDMUP6J6BxE/pK -keA1nmrVXLheVs3kB7Bg7Jm+53E6RPATRGbvJ+5nqtDDxjL3HL8Jg6uKFjPmnpJ4 -crTsbc7nrAxo0cRmUlgbzQqhgAfnb8B7Fai2T1qoFixPtIFicehx ------END RSA PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIDfjCCAmagAwIBAgIDBUEVMA0GCSqGSIb3DQEBBQUAMHQxFzAVBgNVBAMTDktl -cm5lbCBUZXN0IENBMQ8wDQYDVQQLEwZLZXJuZWwxEDAOBgNVBAoTB01vbmdvREIx -FjAUBgNVBAcTDU5ldyBZb3JrIENpdHkxETAPBgNVBAgTCE5ldyBZb3JrMQswCQYD -VQQGEwJVUzAeFw0xNjA5MjIxODE1MTJaFw0zNjA5MjIxODE1MTJaMG8xEjAQBgNV -BAMTCWxvY2FsaG9zdDEPMA0GA1UECxMGS2VybmVsMRAwDgYDVQQKEwdNb25nb0RC -MRYwFAYDVQQHEw1OZXcgWW9yayBDaXR5MREwDwYDVQQIEwhOZXcgWW9yazELMAkG -A1UEBhMCVVMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCTHMXV0LEZ -OCuDZ292e26NbbrMaib6IL3obp/5tOvNVCNnvfgYyJwCCTIZq/mwCjAV5N8Y7tJM -v0JrrGIWgJ3qtPMQ/1VxfzLLW598nnBuqZG2HiR3CTfhd0JBmnjKDMscz90+xB2x -DUDVe6PkbZWnN2otsBzVbW+AAJRVTgUb3cjSbGcC0eTMg3SGaWiB+DtiJIAe3bl8 -6TTmrUKVvbzbJrdrFWpz+NVxf5ejZje+Wlz6OXgkWki5U41PtA7aDFIX3mo1J3c0 -jW957fC/q76jrBoTCbufYPaLQIb5QSex+aJZ40rHpSSV75tsXNUkn22u83Bes+Ih -X0As7g5kW2TDAgMBAAGjHjAcMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATAN -BgkqhkiG9w0BAQUFAAOCAQEAEDzWG64/IlXSEFQZom1z0uBLSLVaxrNg4se6geLH -Bt63EW78H+JMf97AA32DsDiT3ih5uo8yUcOVoEUwontUOSjekHrYfagF/KxMvyMy -sWX+8m5SLrU6s4FysUCtlXa92g1Nh/rET074U2sNShhALgNB2XSw9P5n9GnKt5VT -Rkh0AeBJd09WcOGnSHs30+kKGNV8A5a2GTJbDma0dLa7zlhV6VU91Z9LA0aamyrX -eWwnymJvRcIYvxGqgNDxN/8MsaU1EcW0MNEDkc+kDE1LbOwlAQbCeLQDq/w6AlmC -smoCi0pp6Bf8tZM2RhcUN/xXxgEKcZzhlDOI4v8RNHOyMg== ------END CERTIFICATE----- diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs8-encrypted-rev.pem b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs8-encrypted-rev.pem deleted file mode 100644 index 2a9b8ea4aa4..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs8-encrypted-rev.pem +++ /dev/null @@ -1,51 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDfjCCAmagAwIBAgIDBUEVMA0GCSqGSIb3DQEBBQUAMHQxFzAVBgNVBAMTDktl -cm5lbCBUZXN0IENBMQ8wDQYDVQQLEwZLZXJuZWwxEDAOBgNVBAoTB01vbmdvREIx -FjAUBgNVBAcTDU5ldyBZb3JrIENpdHkxETAPBgNVBAgTCE5ldyBZb3JrMQswCQYD -VQQGEwJVUzAeFw0xNjA5MjIxODE1MTJaFw0zNjA5MjIxODE1MTJaMG8xEjAQBgNV -BAMTCWxvY2FsaG9zdDEPMA0GA1UECxMGS2VybmVsMRAwDgYDVQQKEwdNb25nb0RC -MRYwFAYDVQQHEw1OZXcgWW9yayBDaXR5MREwDwYDVQQIEwhOZXcgWW9yazELMAkG -A1UEBhMCVVMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCTHMXV0LEZ -OCuDZ292e26NbbrMaib6IL3obp/5tOvNVCNnvfgYyJwCCTIZq/mwCjAV5N8Y7tJM -v0JrrGIWgJ3qtPMQ/1VxfzLLW598nnBuqZG2HiR3CTfhd0JBmnjKDMscz90+xB2x -DUDVe6PkbZWnN2otsBzVbW+AAJRVTgUb3cjSbGcC0eTMg3SGaWiB+DtiJIAe3bl8 -6TTmrUKVvbzbJrdrFWpz+NVxf5ejZje+Wlz6OXgkWki5U41PtA7aDFIX3mo1J3c0 -jW957fC/q76jrBoTCbufYPaLQIb5QSex+aJZ40rHpSSV75tsXNUkn22u83Bes+Ih -X0As7g5kW2TDAgMBAAGjHjAcMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATAN -BgkqhkiG9w0BAQUFAAOCAQEAEDzWG64/IlXSEFQZom1z0uBLSLVaxrNg4se6geLH -Bt63EW78H+JMf97AA32DsDiT3ih5uo8yUcOVoEUwontUOSjekHrYfagF/KxMvyMy -sWX+8m5SLrU6s4FysUCtlXa92g1Nh/rET074U2sNShhALgNB2XSw9P5n9GnKt5VT -Rkh0AeBJd09WcOGnSHs30+kKGNV8A5a2GTJbDma0dLa7zlhV6VU91Z9LA0aamyrX -eWwnymJvRcIYvxGqgNDxN/8MsaU1EcW0MNEDkc+kDE1LbOwlAQbCeLQDq/w6AlmC -smoCi0pp6Bf8tZM2RhcUN/xXxgEKcZzhlDOI4v8RNHOyMg== ------END CERTIFICATE----- ------BEGIN ENCRYPTED PRIVATE KEY----- -MIIFHzBJBgkqhkiG9w0BBQ0wPDAbBgkqhkiG9w0BBQwwDgQIP80PLbXYYHUCAggA -MB0GCWCGSAFlAwQBAgQQu1qZnln9ymhZVDJmGJpIJQSCBNDufC1nGCgwBWtkzqP+ -MN3/UJD4cX6TQDjGotN484gLvCm138yB8EPSuFz2RUcOFQImKm3fuqBKgx08jen6 -DQRNekzW1ngIV3BZwn5kMwr0lJK4ibpfEmdTYu/2INq55ljsFx7pq+69PLOqskPa -l+1CzPub0xPC8spG6H0xxOV3HYZlzNX6SKgpK/GPCyGzspgijdacn+x+KFpvMRG3 -fDvdGTP5F/lk6++EHFM/LBfitNV0qkd9GoOIbcDkinu6EytSfJY/mY337AhitWQZ -zdhgC3nA+QYy9s/hs2hXBepkIsFzLMRF162Cqc7KPNObpVGBPxFS+an3c7FyYXVw -ekTf1XrUpdsqNIgvSQkUhzkPc01jHWd4paHgSCLayLx6c9jPXiCxgASZ7BcjAZOC -VLqoi9RHYrEdpoZBwMnSheHa6OVdqPbitlx4vA41s1ERuRktz9hXuhl/Rje+IF5i -2N2l4q3ix4K2yvtZ4wmoc92/WPy2XVudeBinupIxLbrq82HIs1KvLZZ78s+s2Gfh -PDH/1gMiraOWyBY1/4DtAnptl2qKW3YsTwMGCfrX8euRC7WCk/QBw6SBy1XlV2pc -uc1ZOAgWQHwDSRK6XJHgElrQkgVRlszg5vofJ1RdRxJo6XossIc3vx/IUqv2+7xx -mGBE+71FYDg4vmN5nAgN2MjEGdyMEGL4WiKT6Y/WSOTrtRVKRFTilzxuOmx6Hq37 -rldBokhttrx0JikU0fqDWSaDbERSslmv5TinygKyq/PnGOHtcBzHC0c+AIlp2Rj8 -Z5TbgMVcxjV0GZ0SojjO6DO9weJ5c5iBom+VJrniYNDc4jqn0OqIQEembgGuTdHk -37Dqp7oxonLZS1Qi+YNljxQvGUeaoy0hSJS/9C2ANWoo+POB/BkhdS3NT2CQAxNZ -ca4ThdtyLvhSjLIEEMJH7J+LFVuE32hbivWtjKcha8vJ/sYz5gZE193Jfz5H92Zq -3Ee7ipvaKQrxATCp7xJdX5ftHp2+dMsiRKxff8TOO9TVwoJkWOw9zSOMidI+znuL -IF2kTMMPu/o1EbOzEvgck/dcvPlTzWQEGy6eCSixndB4c9yjcVnZpzYnWJEhV7to -W9OfcBkQ/3V5jn96yQPCXm1br2j9FS5QDmWP+GOlLUEPwb27jUajTs3emeqvC0qJ -OALtJsKkwT9L7Cq/cZNByBrbmimEI1NkaVRPjauHhQSzPYIJWBkaJPoZIkbCJ5eO -vRi/2Bd74fda8pVFxm9kUNP8wwpz9JSXmzVRzGXOJ3lS1TKAXl++gb5HX+bieSNy -QHcjw6rBwOkdac40vs5mxGb0XHtP8Qqvn0+fzmKan4MBGKGrB2nlfBrhI2Uopni5 -WRSWbZjDe3ofsjlaj39rxQksvSnZEN/us4JHl2QWfYhpg9tYiCmO0zPREqdWKoi2 -IgydR30JXmNx+W2UBoh1iIPgxeqkDXsxWusGbAgyZs4s7/dcrlcVQz5vzHm0zXsK -hix58eAuxTJORkGKaxva5fmdwvHJJPt5/nPPsGdm81WVqm79yKRRE4mjl+PTBryE -4IuFZjGksVDHpi1LMpW4FMmaYjf/oNm9/ZAqOtxJYC8CFIyyVbqSMOwrqSDxmE8O -gHyWskGclbX/lOH8H83lXnh2xw== ------END ENCRYPTED PRIVATE KEY----- diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs8-encrypted.pem b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs8-encrypted.pem deleted file mode 100644 index 88773490b0f..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs8-encrypted.pem +++ /dev/null @@ -1,51 +0,0 @@ ------BEGIN ENCRYPTED PRIVATE KEY----- -MIIFHzBJBgkqhkiG9w0BBQ0wPDAbBgkqhkiG9w0BBQwwDgQIP80PLbXYYHUCAggA -MB0GCWCGSAFlAwQBAgQQu1qZnln9ymhZVDJmGJpIJQSCBNDufC1nGCgwBWtkzqP+ -MN3/UJD4cX6TQDjGotN484gLvCm138yB8EPSuFz2RUcOFQImKm3fuqBKgx08jen6 -DQRNekzW1ngIV3BZwn5kMwr0lJK4ibpfEmdTYu/2INq55ljsFx7pq+69PLOqskPa -l+1CzPub0xPC8spG6H0xxOV3HYZlzNX6SKgpK/GPCyGzspgijdacn+x+KFpvMRG3 -fDvdGTP5F/lk6++EHFM/LBfitNV0qkd9GoOIbcDkinu6EytSfJY/mY337AhitWQZ -zdhgC3nA+QYy9s/hs2hXBepkIsFzLMRF162Cqc7KPNObpVGBPxFS+an3c7FyYXVw -ekTf1XrUpdsqNIgvSQkUhzkPc01jHWd4paHgSCLayLx6c9jPXiCxgASZ7BcjAZOC -VLqoi9RHYrEdpoZBwMnSheHa6OVdqPbitlx4vA41s1ERuRktz9hXuhl/Rje+IF5i -2N2l4q3ix4K2yvtZ4wmoc92/WPy2XVudeBinupIxLbrq82HIs1KvLZZ78s+s2Gfh -PDH/1gMiraOWyBY1/4DtAnptl2qKW3YsTwMGCfrX8euRC7WCk/QBw6SBy1XlV2pc -uc1ZOAgWQHwDSRK6XJHgElrQkgVRlszg5vofJ1RdRxJo6XossIc3vx/IUqv2+7xx -mGBE+71FYDg4vmN5nAgN2MjEGdyMEGL4WiKT6Y/WSOTrtRVKRFTilzxuOmx6Hq37 -rldBokhttrx0JikU0fqDWSaDbERSslmv5TinygKyq/PnGOHtcBzHC0c+AIlp2Rj8 -Z5TbgMVcxjV0GZ0SojjO6DO9weJ5c5iBom+VJrniYNDc4jqn0OqIQEembgGuTdHk -37Dqp7oxonLZS1Qi+YNljxQvGUeaoy0hSJS/9C2ANWoo+POB/BkhdS3NT2CQAxNZ -ca4ThdtyLvhSjLIEEMJH7J+LFVuE32hbivWtjKcha8vJ/sYz5gZE193Jfz5H92Zq -3Ee7ipvaKQrxATCp7xJdX5ftHp2+dMsiRKxff8TOO9TVwoJkWOw9zSOMidI+znuL -IF2kTMMPu/o1EbOzEvgck/dcvPlTzWQEGy6eCSixndB4c9yjcVnZpzYnWJEhV7to -W9OfcBkQ/3V5jn96yQPCXm1br2j9FS5QDmWP+GOlLUEPwb27jUajTs3emeqvC0qJ -OALtJsKkwT9L7Cq/cZNByBrbmimEI1NkaVRPjauHhQSzPYIJWBkaJPoZIkbCJ5eO -vRi/2Bd74fda8pVFxm9kUNP8wwpz9JSXmzVRzGXOJ3lS1TKAXl++gb5HX+bieSNy -QHcjw6rBwOkdac40vs5mxGb0XHtP8Qqvn0+fzmKan4MBGKGrB2nlfBrhI2Uopni5 -WRSWbZjDe3ofsjlaj39rxQksvSnZEN/us4JHl2QWfYhpg9tYiCmO0zPREqdWKoi2 -IgydR30JXmNx+W2UBoh1iIPgxeqkDXsxWusGbAgyZs4s7/dcrlcVQz5vzHm0zXsK -hix58eAuxTJORkGKaxva5fmdwvHJJPt5/nPPsGdm81WVqm79yKRRE4mjl+PTBryE -4IuFZjGksVDHpi1LMpW4FMmaYjf/oNm9/ZAqOtxJYC8CFIyyVbqSMOwrqSDxmE8O -gHyWskGclbX/lOH8H83lXnh2xw== ------END ENCRYPTED PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIDfjCCAmagAwIBAgIDBUEVMA0GCSqGSIb3DQEBBQUAMHQxFzAVBgNVBAMTDktl -cm5lbCBUZXN0IENBMQ8wDQYDVQQLEwZLZXJuZWwxEDAOBgNVBAoTB01vbmdvREIx -FjAUBgNVBAcTDU5ldyBZb3JrIENpdHkxETAPBgNVBAgTCE5ldyBZb3JrMQswCQYD -VQQGEwJVUzAeFw0xNjA5MjIxODE1MTJaFw0zNjA5MjIxODE1MTJaMG8xEjAQBgNV -BAMTCWxvY2FsaG9zdDEPMA0GA1UECxMGS2VybmVsMRAwDgYDVQQKEwdNb25nb0RC -MRYwFAYDVQQHEw1OZXcgWW9yayBDaXR5MREwDwYDVQQIEwhOZXcgWW9yazELMAkG -A1UEBhMCVVMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCTHMXV0LEZ -OCuDZ292e26NbbrMaib6IL3obp/5tOvNVCNnvfgYyJwCCTIZq/mwCjAV5N8Y7tJM -v0JrrGIWgJ3qtPMQ/1VxfzLLW598nnBuqZG2HiR3CTfhd0JBmnjKDMscz90+xB2x -DUDVe6PkbZWnN2otsBzVbW+AAJRVTgUb3cjSbGcC0eTMg3SGaWiB+DtiJIAe3bl8 -6TTmrUKVvbzbJrdrFWpz+NVxf5ejZje+Wlz6OXgkWki5U41PtA7aDFIX3mo1J3c0 -jW957fC/q76jrBoTCbufYPaLQIb5QSex+aJZ40rHpSSV75tsXNUkn22u83Bes+Ih -X0As7g5kW2TDAgMBAAGjHjAcMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATAN -BgkqhkiG9w0BAQUFAAOCAQEAEDzWG64/IlXSEFQZom1z0uBLSLVaxrNg4se6geLH -Bt63EW78H+JMf97AA32DsDiT3ih5uo8yUcOVoEUwontUOSjekHrYfagF/KxMvyMy -sWX+8m5SLrU6s4FysUCtlXa92g1Nh/rET074U2sNShhALgNB2XSw9P5n9GnKt5VT -Rkh0AeBJd09WcOGnSHs30+kKGNV8A5a2GTJbDma0dLa7zlhV6VU91Z9LA0aamyrX -eWwnymJvRcIYvxGqgNDxN/8MsaU1EcW0MNEDkc+kDE1LbOwlAQbCeLQDq/w6AlmC -smoCi0pp6Bf8tZM2RhcUN/xXxgEKcZzhlDOI4v8RNHOyMg== ------END CERTIFICATE----- diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs8-rev.pem b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs8-rev.pem deleted file mode 100644 index 7d902b28e11..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs8-rev.pem +++ /dev/null @@ -1,50 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDuDCCAqCgAwIBAgIJAIJdodI/q6hqMA0GCSqGSIb3DQEBCwUAMFIxCzAJBgNV -BAYTAlVTMREwDwYDVQQIDAhOZXcgWW9yazEQMA4GA1UECgwHTW9uZ29EQjERMA8G -A1UECwwIU2VjdXJpdHkxCzAJBgNVBAMMAmNhMB4XDTE3MDMxNzEwMTQ0MVoXDTI3 -MDMxNTEwMTQ0MVowaDELMAkGA1UEBhMCVVMxETAPBgNVBAgMCE5ldyBZb3JrMREw -DwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHTW9uZ29EQjEQMA4GA1UECwwHRHJp -dmVyczEPMA0GA1UEAwwGY2xpZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB -CgKCAQEA0nYSydRYw7eM1KtyM6s49A7SIPGUua+wStu9KkTzFcaJ8y/hcrek5J/4 -PcpY5gf8tkf3GrXxumtPnWCJJP+wbNh4U9HJgtFrzkIHnYmOxjLERGgu/w+4W3J+ -/RUSOOHK2DeOzIYZd79d48716kNWYFV80nhQRJexJSD1fGgQLll947HBh50f4Jne -JMtq3Bw/YoJfKDa8AcsWj80U5yGF6BUhVddteIwXlHbTUJxFu5cZ3iVOEr7sTd8O -gpJ1XZgUGOW9fVBxwRRiLe1MXHrljvaNOT532W+kQDw9U94teD6pDTIRPrOxJ8l4 -GWiP3hyKVqcbx2fPumj5zqRz8nlZSQIDAQABo3sweTAJBgNVHRMEAjAAMCwGCWCG -SAGG+EIBDQQfFh1PcGVuU1NMIEdlbmVyYXRlZCBDZXJ0aWZpY2F0ZTAdBgNVHQ4E -FgQUd7IrLVymryD9EI7JqSDy61B0hiYwHwYDVR0jBBgwFoAU6V78cAw4dTLrQwZE -x8Vf5k+rHDkwDQYJKoZIhvcNAQELBQADggEBAGxTZl9WrjlXd9UFIFKiTx3io/YR -NuAfStSuLwoNAi3P+XYLwvfUScyHOambqBmBFsMSNiQe6h4tepcVIFLeGcsTsoyf -JkTMwiJH1iIdAchNJmsdkWrPlzUc8s7modmzBx6TBokiGL79vVuh20SW8IyWJZaf -79A1vFR7PRRPsJWfbXkEOP+CoyQfJtPLz+fFcX2CFkvtn5T8IM97OBBckyE3pjRQ -nZ7bDc+mM/2T23KMnSWNvqP68Yt+7YMyQ+uj1+HJOHfHQSD0nU/Mn0+EqLhZbzvL -EKJ5z1meByoriHlMGvZjGMIIcH1Gt/QAi8sVzZBJr+Cq0c9P7F+uNFcODaM= ------END CERTIFICATE----- ------BEGIN PRIVATE KEY----- -MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDSdhLJ1FjDt4zU -q3Izqzj0DtIg8ZS5r7BK270qRPMVxonzL+Fyt6Tkn/g9yljmB/y2R/catfG6a0+d -YIkk/7Bs2HhT0cmC0WvOQgediY7GMsREaC7/D7hbcn79FRI44crYN47Mhhl3v13j -zvXqQ1ZgVXzSeFBEl7ElIPV8aBAuWX3jscGHnR/gmd4ky2rcHD9igl8oNrwByxaP -zRTnIYXoFSFV1214jBeUdtNQnEW7lxneJU4SvuxN3w6CknVdmBQY5b19UHHBFGIt -7UxceuWO9o05PnfZb6RAPD1T3i14PqkNMhE+s7EnyXgZaI/eHIpWpxvHZ8+6aPnO -pHPyeVlJAgMBAAECggEBALR2AAhF51Ly2XQmCkeZor1K1AzhePh7WDvoDVzoQFPE -qNb4kGTwaRiMvqwlDHM6GAwoyw6BQmPpzhuRAifSgvHh79NXiGV+suTqI2OG5wC1 -2Ssa9mlIjnkDRTY3UieqHGenw+9FcSMH2TcUaDLWSINT6jMCbTlTpNbEWxqwlGdY -URP4I0lN/NiKyfGemctXTuKj3YOB+6feQZaL1RWYJ9pneSad6rbsLYLOc0JAK8zG -wu3mPhPTrMqaj7DXAbhz4NWmIjosIp4/5bnV2HSvpd4UiB7/yK/gknZ4XesHOz4z -aIKbYxbaUkrKIGO/mwuZOjCDSw/rSDmuxuDWFRU3NE0CgYEA6LW2dFPfxIxEmeXl -EzFxJhsCeCOcPo4ueWxbMboILl2KjMhTUGxKZEjJtWpK+FwVqashU2CrDfW/zfzp -ekb1EVAeq+bDsHKRXMJfHQ3qky733nqsKbpQonJwyQ64AVhQVLgr8Xi+gNiYaOWo -g5ZftrlNlJu59GJv+St2eipAlwsCgYEA54ZTpYAodRkMcZOGqiG8mHwNwxPSRIRO -7iRRT+8NFLVfC3Y1oPD7o2tmwFDpSzybgIOpdKuqHOG6/ed82AyqnODhmdNPcCpv -FgyWZaurgJepe8Y61vjoaV6y7geLJAOL/WAbqzRRq6tDI708t21lsFCTvtoyW/0I -0kggr/+ytXsCgYEAkaW5jlE4ilGoVhI3L64QPWNGRl8zWUuv9rtE0Hi4yhwtrTNs -QbelT+LmrC7cwVkRDeJXt1GXfeNDqu8SSj/C/pUAvWJvNC5goIfe89ZT7M7GwG5S -9sLv2Nx7jrsxm1Xk4UFr73Q893OY4H5s2/7v5PNRhSN6XWSG5JK5UnjDeEUCgYEA -iMQnAWsVeybS3Pzi3fmT6RfPIV/CJEtsPO0jQ27ZcVQ60xB/WZVBcSXuysiBJ7qj -uWUNYyhNE0adKYPnkdDZsFZ/rljPYlkOyh2hcmnYo9vzeHR/KaJb2HLijA3Uue+G -cKSnc5kybZB71s7g4RI0sdTHkkRe30w4O8/zz0PjE6UCgYEAzARJZItdMu9wGu3U -X7tSSXJL2avVKv/lBDUfZAChBhpXOQf7MvgmKUCiZC/BMZ/plw/AxBL8swrfKgsw -TdrZwrhK3wOgqYWIHCAfzR+Qa0rRTqVmRQERFylqXzNmUWMG5iq7D9rp3Ht9/Ozn -6NGsAa53FvCDeBkFzi/dsbhxvjk= ------END PRIVATE KEY----- diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs8.pem b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs8.pem deleted file mode 100644 index 305c67658c9..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/testdata/pkcs8.pem +++ /dev/null @@ -1,50 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDSdhLJ1FjDt4zU -q3Izqzj0DtIg8ZS5r7BK270qRPMVxonzL+Fyt6Tkn/g9yljmB/y2R/catfG6a0+d -YIkk/7Bs2HhT0cmC0WvOQgediY7GMsREaC7/D7hbcn79FRI44crYN47Mhhl3v13j -zvXqQ1ZgVXzSeFBEl7ElIPV8aBAuWX3jscGHnR/gmd4ky2rcHD9igl8oNrwByxaP -zRTnIYXoFSFV1214jBeUdtNQnEW7lxneJU4SvuxN3w6CknVdmBQY5b19UHHBFGIt -7UxceuWO9o05PnfZb6RAPD1T3i14PqkNMhE+s7EnyXgZaI/eHIpWpxvHZ8+6aPnO -pHPyeVlJAgMBAAECggEBALR2AAhF51Ly2XQmCkeZor1K1AzhePh7WDvoDVzoQFPE -qNb4kGTwaRiMvqwlDHM6GAwoyw6BQmPpzhuRAifSgvHh79NXiGV+suTqI2OG5wC1 -2Ssa9mlIjnkDRTY3UieqHGenw+9FcSMH2TcUaDLWSINT6jMCbTlTpNbEWxqwlGdY -URP4I0lN/NiKyfGemctXTuKj3YOB+6feQZaL1RWYJ9pneSad6rbsLYLOc0JAK8zG -wu3mPhPTrMqaj7DXAbhz4NWmIjosIp4/5bnV2HSvpd4UiB7/yK/gknZ4XesHOz4z -aIKbYxbaUkrKIGO/mwuZOjCDSw/rSDmuxuDWFRU3NE0CgYEA6LW2dFPfxIxEmeXl -EzFxJhsCeCOcPo4ueWxbMboILl2KjMhTUGxKZEjJtWpK+FwVqashU2CrDfW/zfzp -ekb1EVAeq+bDsHKRXMJfHQ3qky733nqsKbpQonJwyQ64AVhQVLgr8Xi+gNiYaOWo -g5ZftrlNlJu59GJv+St2eipAlwsCgYEA54ZTpYAodRkMcZOGqiG8mHwNwxPSRIRO -7iRRT+8NFLVfC3Y1oPD7o2tmwFDpSzybgIOpdKuqHOG6/ed82AyqnODhmdNPcCpv -FgyWZaurgJepe8Y61vjoaV6y7geLJAOL/WAbqzRRq6tDI708t21lsFCTvtoyW/0I -0kggr/+ytXsCgYEAkaW5jlE4ilGoVhI3L64QPWNGRl8zWUuv9rtE0Hi4yhwtrTNs -QbelT+LmrC7cwVkRDeJXt1GXfeNDqu8SSj/C/pUAvWJvNC5goIfe89ZT7M7GwG5S -9sLv2Nx7jrsxm1Xk4UFr73Q893OY4H5s2/7v5PNRhSN6XWSG5JK5UnjDeEUCgYEA -iMQnAWsVeybS3Pzi3fmT6RfPIV/CJEtsPO0jQ27ZcVQ60xB/WZVBcSXuysiBJ7qj -uWUNYyhNE0adKYPnkdDZsFZ/rljPYlkOyh2hcmnYo9vzeHR/KaJb2HLijA3Uue+G -cKSnc5kybZB71s7g4RI0sdTHkkRe30w4O8/zz0PjE6UCgYEAzARJZItdMu9wGu3U -X7tSSXJL2avVKv/lBDUfZAChBhpXOQf7MvgmKUCiZC/BMZ/plw/AxBL8swrfKgsw -TdrZwrhK3wOgqYWIHCAfzR+Qa0rRTqVmRQERFylqXzNmUWMG5iq7D9rp3Ht9/Ozn -6NGsAa53FvCDeBkFzi/dsbhxvjk= ------END PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIDuDCCAqCgAwIBAgIJAIJdodI/q6hqMA0GCSqGSIb3DQEBCwUAMFIxCzAJBgNV -BAYTAlVTMREwDwYDVQQIDAhOZXcgWW9yazEQMA4GA1UECgwHTW9uZ29EQjERMA8G -A1UECwwIU2VjdXJpdHkxCzAJBgNVBAMMAmNhMB4XDTE3MDMxNzEwMTQ0MVoXDTI3 -MDMxNTEwMTQ0MVowaDELMAkGA1UEBhMCVVMxETAPBgNVBAgMCE5ldyBZb3JrMREw -DwYDVQQHDAhOZXcgWW9yazEQMA4GA1UECgwHTW9uZ29EQjEQMA4GA1UECwwHRHJp -dmVyczEPMA0GA1UEAwwGY2xpZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB -CgKCAQEA0nYSydRYw7eM1KtyM6s49A7SIPGUua+wStu9KkTzFcaJ8y/hcrek5J/4 -PcpY5gf8tkf3GrXxumtPnWCJJP+wbNh4U9HJgtFrzkIHnYmOxjLERGgu/w+4W3J+ -/RUSOOHK2DeOzIYZd79d48716kNWYFV80nhQRJexJSD1fGgQLll947HBh50f4Jne -JMtq3Bw/YoJfKDa8AcsWj80U5yGF6BUhVddteIwXlHbTUJxFu5cZ3iVOEr7sTd8O -gpJ1XZgUGOW9fVBxwRRiLe1MXHrljvaNOT532W+kQDw9U94teD6pDTIRPrOxJ8l4 -GWiP3hyKVqcbx2fPumj5zqRz8nlZSQIDAQABo3sweTAJBgNVHRMEAjAAMCwGCWCG -SAGG+EIBDQQfFh1PcGVuU1NMIEdlbmVyYXRlZCBDZXJ0aWZpY2F0ZTAdBgNVHQ4E -FgQUd7IrLVymryD9EI7JqSDy61B0hiYwHwYDVR0jBBgwFoAU6V78cAw4dTLrQwZE -x8Vf5k+rHDkwDQYJKoZIhvcNAQELBQADggEBAGxTZl9WrjlXd9UFIFKiTx3io/YR -NuAfStSuLwoNAi3P+XYLwvfUScyHOambqBmBFsMSNiQe6h4tepcVIFLeGcsTsoyf -JkTMwiJH1iIdAchNJmsdkWrPlzUc8s7modmzBx6TBokiGL79vVuh20SW8IyWJZaf -79A1vFR7PRRPsJWfbXkEOP+CoyQfJtPLz+fFcX2CFkvtn5T8IM97OBBckyE3pjRQ -nZ7bDc+mM/2T23KMnSWNvqP68Yt+7YMyQ+uj1+HJOHfHQSD0nU/Mn0+EqLhZbzvL -EKJ5z1meByoriHlMGvZjGMIIcH1Gt/QAi8sVzZBJr+Cq0c9P7F+uNFcODaM= ------END CERTIFICATE----- diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/tlsgo.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/tlsgo.go deleted file mode 100644 index c26b7e2dc4f..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/tlsgo.go +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -// Package tlsgo implements connection to MongoDB with Go native TLS. -package tlsgo - -import ( - "crypto/tls" - "fmt" - "net" - "strings" - "time" - - "github.com/mongodb/mongo-tools/common/db/kerberos" - "github.com/mongodb/mongo-tools/common/log" - "github.com/mongodb/mongo-tools/common/options" - "github.com/mongodb/mongo-tools/common/util" - "gopkg.in/mgo.v2" -) - -// TLSDBConnector makes a connection to the database with Go native TLS. -type TLSDBConnector struct { - dialInfo *mgo.DialInfo - config *TLSConfig -} - -// Configure the connector to connect to the server over ssl. Sets up the -// correct function to dial the server based on the ssl options passed in. -func (c *TLSDBConnector) Configure(opts options.ToolOptions) error { - if opts.SSLFipsMode { - return fmt.Errorf("FIPS mode not supported") - } - - if opts.SSLCRLFile != "" { - return fmt.Errorf("CRL files are not supported on this platform") - } - - c.config = NewTLSConfig() - - if opts.SSLAllowInvalidCert || opts.SSLAllowInvalidHost { - c.config.SetInsecure(true) - } - - if opts.SSLPEMKeyFile != "" { - subject, err := c.config.AddClientCertFromFile(opts.SSLPEMKeyFile, opts.SSLPEMKeyPassword) - if err != nil { - return err - } - if opts.Auth.Mechanism == "MONGODB-X509" && opts.Auth.Username == "" { - opts.Auth.Username = subject - } - } - - if opts.SSLCAFile != "" { - c.config.AddCaCertFromFile(opts.SSLCAFile) - } - - // set up the dial info - c.dialInfo = &mgo.DialInfo{ - Timeout: time.Duration(opts.Timeout) * time.Second, - Direct: opts.Direct, - ReplicaSetName: opts.ReplicaSetName, - DialServer: c.makeDialer(opts), - Username: opts.Auth.Username, - Password: opts.Auth.Password, - Source: opts.GetAuthenticationDatabase(), - Mechanism: opts.Auth.Mechanism, - } - - // create or fetch the addresses to be used to connect - if opts.URI != nil && opts.URI.ConnectionString != "" { - c.dialInfo.Addrs = opts.URI.GetConnectionAddrs() - } else { - c.dialInfo.Addrs = util.CreateConnectionAddrs(opts.Host, opts.Port) - } - kerberos.AddKerberosOpts(opts, c.dialInfo) - return nil -} - -// GetNewSession dials the server. -func (c *TLSDBConnector) GetNewSession() (*mgo.Session, error) { - return mgo.DialWithInfo(c.dialInfo) -} - -// To be handed to mgo.DialInfo for connecting to the server. -type dialerFunc func(addr *mgo.ServerAddr) (net.Conn, error) - -func (c *TLSDBConnector) makeDialer(opts options.ToolOptions) dialerFunc { - return func(addr *mgo.ServerAddr) (net.Conn, error) { - address := addr.String() - conn, err := net.Dial("tcp", address) - if err != nil { - // mgo discards dialer errors so log it now - log.Logvf(log.Always, "error dialing %v: %v", address, err) - return nil, err - } - // enable TCP keepalive - err = util.EnableTCPKeepAlive(conn, time.Duration(opts.TCPKeepAliveSeconds)*time.Second) - if err != nil { - // mgo discards dialer errors so log it now - log.Logvf(log.Always, "error enabling TCP keepalive on connection to %v: %v", address, err) - conn.Close() - return nil, err - } - - tlsConfig, err := c.config.MakeConfig() - if err != nil { - return nil, err - } - - if !tlsConfig.InsecureSkipVerify { - colonPos := strings.LastIndex(address, ":") - if colonPos == -1 { - colonPos = len(address) - } - - hostname := address[:colonPos] - tlsConfig.ServerName = hostname - } - - client := tls.Client(conn, tlsConfig) - err = client.Handshake() - if err != nil { - // mgo discards dialer errors so log it now - log.Logvf(log.Always, "error doing TLS handshake with %v: %v", address, err) - client.Close() - return nil, err - } - - return client, nil - } -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/write_concern.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/write_concern.go deleted file mode 100644 index b7b0377bd70..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/write_concern.go +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package db - -import ( - "github.com/mongodb/mongo-tools/common/connstring" - "github.com/mongodb/mongo-tools/common/json" - "github.com/mongodb/mongo-tools/common/log" - "github.com/mongodb/mongo-tools/common/util" - "gopkg.in/mgo.v2" - - "fmt" - "strconv" - "time" -) - -// write concern fields -const ( - j = "j" - w = "w" - fSync = "fsync" - wTimeout = "wtimeout" -) - -// constructWCObject takes in a write concern and attempts to construct an -// mgo.Safe object from it. It returns an error if it is unable to parse the -// string or if a parsed write concern field value is invalid. -func constructWCObject(writeConcern string) (sessionSafety *mgo.Safe, err error) { - sessionSafety = &mgo.Safe{} - defer func() { - // If the user passes a w value of 0, we set the session to use the - // unacknowledged write concern but only if journal commit acknowledgment, - // is not required. If commit acknowledgment is required, it prevails, - // and the server will require that mongod acknowledge the write operation - if sessionSafety.WMode == "" && sessionSafety.W == 0 && !sessionSafety.J { - sessionSafety = nil - } - }() - jsonWriteConcern := map[string]interface{}{} - - if err = json.Unmarshal([]byte(writeConcern), &jsonWriteConcern); err != nil { - // if the writeConcern string cannot be unmarshalled into JSON, this - // allows a default to the old behavior wherein the entire argument - // passed in is assigned to the 'w' field - thus allowing users to pass - // a write concern that looks like: "majority", 0, "4", etc. - wValue, err := strconv.Atoi(writeConcern) - if err != nil { - sessionSafety.WMode = writeConcern - } else { - sessionSafety.W = wValue - if wValue < 0 { - return sessionSafety, fmt.Errorf("invalid '%v' argument: %v", w, wValue) - } - } - return sessionSafety, nil - } - - if jVal, ok := jsonWriteConcern[j]; ok && util.IsTruthy(jVal) { - sessionSafety.J = true - } - - if fsyncVal, ok := jsonWriteConcern[fSync]; ok && util.IsTruthy(fsyncVal) { - sessionSafety.FSync = true - } - - if wtimeout, ok := jsonWriteConcern[wTimeout]; ok { - wtimeoutValue, err := util.ToInt(wtimeout) - if err != nil { - return sessionSafety, fmt.Errorf("invalid '%v' argument: %v", wTimeout, wtimeout) - } - sessionSafety.WTimeout = wtimeoutValue - } - - if wInterface, ok := jsonWriteConcern[w]; ok { - wValue, err := util.ToInt(wInterface) - if err != nil { - // if the argument is neither a string nor int, error out - wStrVal, ok := wInterface.(string) - if !ok { - return sessionSafety, fmt.Errorf("invalid '%v' argument: %v", w, wInterface) - } - sessionSafety.WMode = wStrVal - } else { - sessionSafety.W = wValue - if wValue < 0 { - return sessionSafety, fmt.Errorf("invalid '%v' argument: %v", w, wValue) - } - } - } - return sessionSafety, nil -} - -// constructSafetyFromConnString takes in a parsed connection string and attempts -// to construct an mgo.Safe object from it. It returns an error if it is unable -// to parse the write concern value. -func constructSafetyFromConnString(cs *connstring.ConnString) (*mgo.Safe, error) { - safe := &mgo.Safe{} - - wValue, err := strconv.Atoi(cs.W) - if err != nil { - safe.WMode = cs.W - } else { - safe.W = wValue - if wValue < 0 { - return nil, fmt.Errorf("invalid '%v' argument: %v", w, wValue) - } - } - - safe.WTimeout = int(cs.WTimeout / time.Second) - safe.FSync = cs.FSync - safe.J = cs.Journal - - if safe.WMode == "" && safe.W == 0 && !safe.J { - return nil, nil - } - - return safe, nil -} - -// BuildWriteConcern takes a string and a NodeType indicating the type of node the write concern -// is intended to be used against, and converts the write concern string argument into an -// mgo.Safe object that's usable on sessions for that node type. -func BuildWriteConcern(writeConcern string, nodeType NodeType, cs *connstring.ConnString) (*mgo.Safe, error) { - var sessionSafety *mgo.Safe - var err error - - if cs != nil && writeConcern != "" { - return nil, fmt.Errorf("cannot specify writeConcern string and connectionString object") - } - - if cs != nil { - if cs.W == "" { - cs.W = "majority" - } - sessionSafety, err = constructSafetyFromConnString(cs) - if err != nil { - return nil, err - } - } else { - if writeConcern == "" { - writeConcern = "majority" - } - sessionSafety, err = constructWCObject(writeConcern) - if err != nil { - return nil, err - } - } - - if sessionSafety == nil { - log.Logvf(log.DebugLow, "using unacknowledged write concern") - return nil, nil - } - - // for standalone mongods, set the default write concern to 1 - if nodeType == Standalone { - log.Logvf(log.DebugLow, "standalone server: setting write concern %v to 1", w) - sessionSafety.W = 1 - sessionSafety.WMode = "" - } - - var writeConcernStr interface{} - - if sessionSafety.WMode != "" { - writeConcernStr = sessionSafety.WMode - } else { - writeConcernStr = sessionSafety.W - } - log.Logvf(log.Info, "using write concern: %v='%v', %v=%v, %v=%v, %v=%v", - w, writeConcernStr, - j, sessionSafety.J, - fSync, sessionSafety.FSync, - wTimeout, sessionSafety.WTimeout, - ) - return sessionSafety, nil -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/write_concern_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/write_concern_test.go deleted file mode 100644 index e9783525eeb..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/write_concern_test.go +++ /dev/null @@ -1,271 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package db - -import ( - "github.com/mongodb/mongo-tools/common/connstring" - "github.com/mongodb/mongo-tools/common/testtype" - . "github.com/smartystreets/goconvey/convey" - - "fmt" - "testing" - "time" -) - -func TestBuildWriteConcern(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - Convey("When building write concern object", t, func() { - Convey("and given a write concern string value, and a boolean indicating if the "+ - "write concern is to be used on a replica set, on calling BuildWriteConcern...", func() { - Convey("no error should be returned if the write concern is valid", func() { - writeConcern, err := BuildWriteConcern(`{w:34}`, ReplSet, nil) - So(err, ShouldBeNil) - So(writeConcern.W, ShouldEqual, 34) - writeConcern, err = BuildWriteConcern(`{w:"majority"}`, ReplSet, nil) - So(err, ShouldBeNil) - So(writeConcern.WMode, ShouldEqual, "majority") - writeConcern, err = BuildWriteConcern(`majority`, ReplSet, nil) - So(err, ShouldBeNil) - So(writeConcern.WMode, ShouldEqual, "majority") - writeConcern, err = BuildWriteConcern(`tagset`, ReplSet, nil) - So(err, ShouldBeNil) - So(writeConcern.WMode, ShouldEqual, "tagset") - }) - Convey("on replica sets, only a write concern of 1 or 0 should be returned", func() { - writeConcern, err := BuildWriteConcern(`{w:34}`, Standalone, nil) - So(err, ShouldBeNil) - So(writeConcern.W, ShouldEqual, 1) - writeConcern, err = BuildWriteConcern(`{w:"majority"}`, Standalone, nil) - So(err, ShouldBeNil) - So(writeConcern.W, ShouldEqual, 1) - writeConcern, err = BuildWriteConcern(`tagset`, Standalone, nil) - So(err, ShouldBeNil) - So(writeConcern.W, ShouldEqual, 1) - }) - Convey("with a w value of 0, without j set, a nil write concern should be returned", func() { - writeConcern, err := BuildWriteConcern(`{w:0}`, Standalone, nil) - So(err, ShouldBeNil) - So(writeConcern, ShouldBeNil) - }) - Convey("with a negative w value, an error should be returned", func() { - _, err := BuildWriteConcern(`{w:-1}`, ReplSet, nil) - So(err, ShouldNotBeNil) - _, err = BuildWriteConcern(`{w:-2}`, ReplSet, nil) - So(err, ShouldNotBeNil) - }) - Convey("with a w value of 0, with j set, a non-nil write concern should be returned", func() { - writeConcern, err := BuildWriteConcern(`{w:0, j:true}`, Standalone, nil) - So(err, ShouldBeNil) - So(writeConcern.J, ShouldBeTrue) - }) - // Regression test for TOOLS-1741 - Convey("When passing an empty writeConcern and empty URI"+ - "then write concern should default to being majority", func() { - writeConcern, err := BuildWriteConcern("", ReplSet, nil) - So(err, ShouldBeNil) - So(writeConcern.WMode, ShouldEqual, "majority") - }) - }) - Convey("and given a connection string", func() { - Convey("with a w value of 0, without j set, a nil write concern should be returned", func() { - writeConcern, err := BuildWriteConcern(``, Standalone, &connstring.ConnString{W: "0"}) - So(err, ShouldBeNil) - So(writeConcern, ShouldBeNil) - }) - Convey("with a negative w value, an error should be returned", func() { - _, err := BuildWriteConcern(``, ReplSet, &connstring.ConnString{W: "-1"}) - So(err, ShouldNotBeNil) - _, err = BuildWriteConcern(``, ReplSet, &connstring.ConnString{W: "-2"}) - So(err, ShouldNotBeNil) - }) - Convey("on replica sets, only a write concern of 1 or 0 should be returned", func() { - writeConcern, err := BuildWriteConcern(``, Standalone, &connstring.ConnString{W: "34"}) - So(err, ShouldBeNil) - So(writeConcern.W, ShouldEqual, 1) - writeConcern, err = BuildWriteConcern(``, Standalone, &connstring.ConnString{W: "majority"}) - So(err, ShouldBeNil) - So(writeConcern.W, ShouldEqual, 1) - writeConcern, err = BuildWriteConcern(``, Standalone, &connstring.ConnString{W: "tagset"}) - So(err, ShouldBeNil) - So(writeConcern.W, ShouldEqual, 1) - }) - }) - Convey("and given both, should error", func() { - _, err := BuildWriteConcern(`ab`, ReplSet, &connstring.ConnString{W: "-1"}) - So(err, ShouldNotBeNil) - }) - }) -} - -func TestConstructWCObject(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - Convey("Given a write concern string value, on calling constructWCObject...", t, func() { - - Convey("non-JSON string values should be assigned to the 'WMode' "+ - "field in their entirety", func() { - writeConcernString := "majority" - writeConcern, err := constructWCObject(writeConcernString) - So(err, ShouldBeNil) - So(writeConcern.WMode, ShouldEqual, writeConcernString) - }) - - Convey("non-JSON int values should be assigned to the 'w' field "+ - "in their entirety", func() { - writeConcernString := `{w: 4}` - writeConcern, err := constructWCObject(writeConcernString) - So(err, ShouldBeNil) - So(writeConcern.W, ShouldEqual, 4) - }) - - Convey("JSON strings with valid j, wtimeout, fsync and w, should be "+ - "assigned accordingly", func() { - writeConcernString := `{w: 3, j: true, fsync: false, wtimeout: 43}` - expectedW := 3 - expectedWTimeout := 43 - writeConcern, err := constructWCObject(writeConcernString) - So(err, ShouldBeNil) - So(writeConcern.W, ShouldEqual, expectedW) - So(writeConcern.J, ShouldBeTrue) - So(writeConcern.FSync, ShouldBeFalse) - So(writeConcern.WTimeout, ShouldEqual, expectedWTimeout) - }) - - Convey("JSON strings with an argument for j that is not false should set j true", func() { - writeConcernString := `{w: 3, j: "rue"}` - writeConcern, err := constructWCObject(writeConcernString) - So(err, ShouldBeNil) - So(writeConcern.W, ShouldEqual, 3) - So(writeConcern.J, ShouldBeTrue) - }) - - Convey("JSON strings with an argument for fsync that is not false should set fsync true", func() { - writeConcernString := `{w: 3, fsync: "rue"}` - writeConcern, err := constructWCObject(writeConcernString) - So(err, ShouldBeNil) - So(writeConcern.W, ShouldEqual, 3) - So(writeConcern.FSync, ShouldBeTrue) - }) - - Convey("JSON strings with an invalid wtimeout argument should error out", func() { - writeConcernString := `{w: 3, wtimeout: "rue"}` - _, err := constructWCObject(writeConcernString) - So(err, ShouldNotBeNil) - writeConcernString = `{w: 3, wtimeout: "43"}` - _, err = constructWCObject(writeConcernString) - So(err, ShouldNotBeNil) - }) - - Convey("JSON strings with any non-false j argument should not error out", func() { - writeConcernString := `{w: 3, j: "t"}` - writeConcern, err := constructWCObject(writeConcernString) - So(err, ShouldBeNil) - So(writeConcern.J, ShouldBeTrue) - writeConcernString = `{w: 3, j: "f"}` - writeConcern, err = constructWCObject(writeConcernString) - So(err, ShouldBeNil) - So(writeConcern.J, ShouldBeTrue) - writeConcernString = `{w: 3, j: false}` - writeConcern, err = constructWCObject(writeConcernString) - So(err, ShouldBeNil) - So(writeConcern.J, ShouldBeFalse) - writeConcernString = `{w: 3, j: 0}` - writeConcern, err = constructWCObject(writeConcernString) - So(err, ShouldBeNil) - So(writeConcern.J, ShouldBeFalse) - }) - - Convey("JSON strings with a shorthand fsync argument should not error out", func() { - writeConcernString := `{w: 3, fsync: "t"}` - writeConcern, err := constructWCObject(writeConcernString) - So(err, ShouldBeNil) - So(writeConcern.FSync, ShouldBeTrue) - writeConcernString = `{w: "3", fsync: "f"}` - writeConcern, err = constructWCObject(writeConcernString) - So(err, ShouldBeNil) - So(writeConcern.FSync, ShouldBeTrue) - writeConcernString = `{w: "3", fsync: false}` - writeConcern, err = constructWCObject(writeConcernString) - So(err, ShouldBeNil) - So(writeConcern.FSync, ShouldBeFalse) - writeConcernString = `{w: "3", fsync: 0}` - writeConcern, err = constructWCObject(writeConcernString) - So(err, ShouldBeNil) - So(writeConcern.FSync, ShouldBeFalse) - }) - - Convey("Unacknowledge write concern strings should return a nil object "+ - "if journaling is not required", func() { - writeConcernString := `{w: 0}` - writeConcern, err := constructWCObject(writeConcernString) - So(err, ShouldBeNil) - So(writeConcern, ShouldBeNil) - writeConcernString = `{w: 0}` - writeConcern, err = constructWCObject(writeConcernString) - So(err, ShouldBeNil) - So(writeConcern, ShouldBeNil) - writeConcernString = `0` - writeConcern, err = constructWCObject(writeConcernString) - So(err, ShouldBeNil) - So(writeConcern, ShouldBeNil) - }) - }) -} - -func TestConstructSafetyFromConnString(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - Convey("Given a parsed &connstring, on calling constructSafetyFromConnString...", t, func() { - - Convey("non string values should be assigned to the 'WMode' "+ - "field in their entirety", func() { - writeConcernString := "majority" - cs := &connstring.ConnString{ - W: writeConcernString, - } - writeConcern, err := constructSafetyFromConnString(cs) - So(err, ShouldBeNil) - So(writeConcern.WMode, ShouldEqual, writeConcernString) - }) - - Convey("Int values should be assigned to the 'w' field ", func() { - cs := &connstring.ConnString{ - W: "4", - } - writeConcern, err := constructSafetyFromConnString(cs) - So(err, ShouldBeNil) - So(writeConcern.W, ShouldEqual, 4) - }) - - Convey("&connstrings with valid j, wtimeout, fsync and w, should be "+ - "assigned accordingly", func() { - expectedW := 3 - expectedWTimeout := 43 - cs := &connstring.ConnString{ - W: "3", - Journal: true, - FSync: false, - WTimeout: time.Second * 43, - } - writeConcern, err := constructSafetyFromConnString(cs) - So(err, ShouldBeNil) - So(writeConcern.W, ShouldEqual, expectedW) - So(writeConcern.J, ShouldBeTrue) - So(writeConcern.FSync, ShouldBeFalse) - So(writeConcern.WTimeout, ShouldEqual, expectedWTimeout) - }) - - Convey("Unacknowledge write concern strings should return a nil object "+ - "if journaling is not required", func() { - cs := &connstring.ConnString{ - W: "0", - } - writeConcern, err := constructSafetyFromConnString(cs) - fmt.Println(writeConcern) - So(err, ShouldBeNil) - So(writeConcern, ShouldBeNil) - }) - }) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/intents/intent.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/intents/intent.go deleted file mode 100644 index 4b82567e612..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/intents/intent.go +++ /dev/null @@ -1,489 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -// Package intents provides utilities for performing dump/restore operations. -package intents - -import ( - "fmt" - "io" - - "github.com/mongodb/mongo-tools/common" - "github.com/mongodb/mongo-tools/common/log" - "github.com/mongodb/mongo-tools/common/util" - "gopkg.in/mgo.v2/bson" -) - -type file interface { - io.ReadWriteCloser - Open() error - Pos() int64 -} - -// DestinationConflictError occurs when multiple namespaces map to the same -// destination. -type DestinationConflictError struct { - Src, Dst string -} - -func (e DestinationConflictError) Error() string { - return fmt.Sprintf("destination conflict: %s (src) => %s (dst)", e.Src, e.Dst) -} - -// FileNeedsIOBuffer is an interface that denotes that a struct needs -// an IO buffer that is managed by an outside control. This interface -// is used to both hand off a buffer to a struct and signal that it should -// release its buffer. Added to reduce memory usage as outlined in TOOLS-1088. -type FileNeedsIOBuffer interface { - TakeIOBuffer([]byte) - ReleaseIOBuffer() -} - -// mongorestore first scans the directory to generate a list -// of all files to restore and what they map to. TODO comments -type Intent struct { - // Destination namespace info - DB string - C string - - // File locations as absolute paths - BSONFile file - BSONSize int64 - MetadataFile file - - // Indicates where the intent will be read from or written to - Location string - MetadataLocation string - - // Collection options - Options *bson.D - - // UUID (for MongoDB 3.6+) as a big-endian hex string - UUID string - - // File/collection size, for some prioritizer implementations. - // Units don't matter as long as they are consistent for a given use case. - Size int64 -} - -func (it *Intent) Namespace() string { - return it.DB + "." + it.C -} - -func (it *Intent) IsOplog() bool { - if it.DB == "" && it.C == "oplog" { - return true - } - return it.DB == "local" && (it.C == "oplog.rs" || it.C == "oplog.$main") -} - -func (it *Intent) IsUsers() bool { - if it.C == "$admin.system.users" { - return true - } - if it.DB == "admin" && it.C == "system.users" { - return true - } - return false -} - -func (it *Intent) IsRoles() bool { - if it.C == "$admin.system.roles" { - return true - } - if it.DB == "admin" && it.C == "system.roles" { - return true - } - return false -} - -func (it *Intent) IsAuthVersion() bool { - if it.C == "$admin.system.version" { - return true - } - if it.DB == "admin" && it.C == "system.version" { - return true - } - return false -} - -func (it *Intent) IsSystemIndexes() bool { - return it.C == "system.indexes" -} - -func (it *Intent) IsSystemProfile() bool { - return it.C == "system.profile" -} - -func (intent *Intent) IsSpecialCollection() bool { - // can't see oplog as special collection because when restore from archive it need to be a RegularCollectionReceiver - return intent.IsSystemIndexes() || intent.IsUsers() || intent.IsRoles() || intent.IsAuthVersion() || intent.IsSystemProfile() -} - -func (it *Intent) IsView() bool { - if it.Options == nil { - return false - } - _, isView := it.Options.Map()["viewOn"] - return isView -} - -func (existing *Intent) MergeIntent(intent *Intent) { - // merge new intent into old intent - if existing.BSONFile == nil { - existing.BSONFile = intent.BSONFile - } - if existing.Size == 0 { - existing.Size = intent.Size - } - if existing.Location == "" { - existing.Location = intent.Location - } - if existing.MetadataFile == nil { - existing.MetadataFile = intent.MetadataFile - } - if existing.MetadataLocation == "" { - existing.MetadataLocation = intent.MetadataLocation - } - -} - -type Manager struct { - // intents are for all of the regular user created collections - intents map[string]*Intent - // special intents are for all of the collections that are created by mongod - // and require special handling - specialIntents map[string]*Intent - - // legacy mongorestore works in the order that paths are discovered, - // so we need an ordered data structure to preserve this behavior. - intentsByDiscoveryOrder []*Intent - - // we need different scheduling order depending on the target - // mongod/mongos and whether or not we are multi threading; - // the IntentPrioritizer interface encapsulates this. - prioritizer IntentPrioritizer - - // special cases that should be saved but not be part of the queue. - // used to deal with oplog and user/roles restoration, which are - // handled outside of the basic logic of the tool - oplogIntent *Intent - usersIntent *Intent - rolesIntent *Intent - versionIntent *Intent - indexIntents map[string]*Intent - - // Tells the manager if it should choose a single oplog when multiple are provided. - smartPickOplog bool - - // Indicates if an the manager has seen two conflicting oplogs. - oplogConflict bool - - // prevent conflicting destinations by checking which sources map to the - // same namespace - destinations map[string][]string -} - -func NewIntentManager() *Manager { - return &Manager{ - intents: map[string]*Intent{}, - specialIntents: map[string]*Intent{}, - intentsByDiscoveryOrder: []*Intent{}, - indexIntents: map[string]*Intent{}, - smartPickOplog: false, - oplogConflict: false, - destinations: map[string][]string{}, - } -} - -func (mgr *Manager) SetSmartPickOplog(smartPick bool) { - mgr.smartPickOplog = smartPick -} - -// HasConfigDBIntent returns a bool indicating if any of the intents refer to the "config" database. -// This can be used to check for possible unwanted conflicts before restoring to a sharded system. -func (mgr *Manager) HasConfigDBIntent() bool { - for _, intent := range mgr.intentsByDiscoveryOrder { - if intent.DB == "config" { - return true - } - } - return false -} - -// PutOplogIntent takes an intent for an oplog and stores it in the intent manager with the -// provided key. If the manager has smartPickOplog enabled, then it uses a priority system -// to determine which oplog intent to maintain as the actual oplog. -func (manager *Manager) PutOplogIntent(intent *Intent, managerKey string) { - if manager.smartPickOplog { - if existing := manager.specialIntents[managerKey]; existing != nil { - existing.MergeIntent(intent) - return - } - if manager.oplogIntent == nil { - // If there is no oplog intent, make this one the oplog. - manager.oplogIntent = intent - manager.specialIntents[managerKey] = intent - } else if intent.DB == "" { - // We already have an oplog and this is a top priority oplog. - if manager.oplogIntent.DB == "" { - // If the manager's current oplog is also top priority, we have a - // conflict and ignore this oplog. - manager.oplogConflict = true - } else { - // If the manager's current oplog is lower priority, replace it and - // move that one to be a normal intent. - manager.putNormalIntent(manager.oplogIntent) - delete(manager.specialIntents, manager.oplogIntent.Namespace()) - manager.oplogIntent = intent - manager.specialIntents[managerKey] = intent - } - } else { - // We already have an oplog and this is a low priority oplog. - if manager.oplogIntent.DB != "" { - // If the manager's current oplog is also low priority, set a conflict. - manager.oplogConflict = true - } - // No matter what, set this lower priority oplog to be a normal intent. - manager.putNormalIntent(intent) - } - } else { - if intent.DB == "" && intent.C == "oplog" { - // If this is a normal oplog, then add it as an oplog intent. - if existing := manager.specialIntents[managerKey]; existing != nil { - existing.MergeIntent(intent) - return - } - manager.oplogIntent = intent - manager.specialIntents[managerKey] = intent - } else { - manager.putNormalIntent(intent) - } - } -} - -func (manager *Manager) putNormalIntent(intent *Intent) { - manager.putNormalIntentWithNamespace(intent.Namespace(), intent) -} - -func (manager *Manager) putNormalIntentWithNamespace(ns string, intent *Intent) { - // BSON and metadata files for the same collection are merged - // into the same intent. This is done to allow for simple - // pairing of BSON + metadata without keeping track of the - // state of the filepath walker - if existing := manager.intents[ns]; existing != nil { - if existing.Namespace() != intent.Namespace() { - // remove old destination, add new one - dst := existing.Namespace() - dsts := manager.destinations[dst] - i := util.StringSliceIndex(dsts, ns) - manager.destinations[dst] = append(dsts[:i], dsts[i+1:]...) - - dsts = manager.destinations[intent.Namespace()] - manager.destinations[intent.Namespace()] = append(dsts, ns) - } - existing.MergeIntent(intent) - return - } - - // if key doesn't already exist, add it to the manager - manager.intents[ns] = intent - manager.intentsByDiscoveryOrder = append(manager.intentsByDiscoveryOrder, intent) - - manager.destinations[intent.Namespace()] = append(manager.destinations[intent.Namespace()], ns) -} - -// Put inserts an intent into the manager with the same source namespace as -// its destinations. -func (manager *Manager) Put(intent *Intent) { - log.Logvf(log.DebugLow, "enqueued collection '%v'", intent.Namespace()) - manager.PutWithNamespace(intent.Namespace(), intent) -} - -// PutWithNamespace inserts an intent into the manager with the source set -// to the provided namespace. Intents for the same collection are merged -// together, so that BSON and metadata files for the same collection are -// returned in the same intent. -func (manager *Manager) PutWithNamespace(ns string, intent *Intent) { - if intent == nil { - panic("cannot insert nil *Intent into IntentManager") - } - db, _ := common.SplitNamespace(ns) - - // bucket special-case collections - if intent.IsOplog() { - manager.PutOplogIntent(intent, intent.Namespace()) - return - } - if intent.IsSystemIndexes() { - if intent.BSONFile != nil { - manager.indexIntents[db] = intent - manager.specialIntents[ns] = intent - } - return - } - if intent.IsUsers() { - if intent.BSONFile != nil { - manager.usersIntent = intent - manager.specialIntents[ns] = intent - } - return - } - if intent.IsRoles() { - if intent.BSONFile != nil { - manager.rolesIntent = intent - manager.specialIntents[ns] = intent - } - return - } - if intent.IsAuthVersion() { - if intent.BSONFile != nil { - manager.versionIntent = intent - manager.specialIntents[ns] = intent - } - return - } - - manager.putNormalIntentWithNamespace(ns, intent) -} - -func (manager *Manager) GetOplogConflict() bool { - return manager.oplogConflict -} - -func (manager *Manager) GetDestinationConflicts() (errs []DestinationConflictError) { - for dst, srcs := range manager.destinations { - if len(srcs) <= 1 { - continue - } - for _, src := range srcs { - errs = append(errs, DestinationConflictError{Dst: dst, Src: src}) - } - } - return -} - -// Intents returns a slice containing all of the intents in the manager. -// Intents is not thread safe -func (manager *Manager) Intents() []*Intent { - allIntents := []*Intent{} - for _, intent := range manager.intents { - allIntents = append(allIntents, intent) - } - for _, intent := range manager.indexIntents { - allIntents = append(allIntents, intent) - } - if manager.oplogIntent != nil { - allIntents = append(allIntents, manager.oplogIntent) - } - if manager.usersIntent != nil { - allIntents = append(allIntents, manager.usersIntent) - } - if manager.rolesIntent != nil { - allIntents = append(allIntents, manager.rolesIntent) - } - if manager.versionIntent != nil { - allIntents = append(allIntents, manager.versionIntent) - } - return allIntents -} - -func (manager *Manager) IntentForNamespace(ns string) *Intent { - intent := manager.intents[ns] - if intent != nil { - return intent - } - intent = manager.specialIntents[ns] - return intent -} - -// Pop returns the next available intent from the manager. If the manager is -// empty, it returns nil. Pop is thread safe. -func (manager *Manager) Pop() *Intent { - return manager.prioritizer.Get() -} - -// Peek returns a copy of a stored intent from the manager without removing -// the intent. This method is useful for edge cases that need to look ahead -// at what collections are in the manager before they are scheduled. -// -// NOTE: There are no guarantees that peek will return a usable -// intent after Finalize() is called. -func (manager *Manager) Peek() *Intent { - if len(manager.intentsByDiscoveryOrder) == 0 { - return nil - } - intentCopy := *manager.intentsByDiscoveryOrder[0] - return &intentCopy -} - -// Finish tells the prioritizer that mongorestore is done restoring -// the given collection intent. -func (manager *Manager) Finish(intent *Intent) { - manager.prioritizer.Finish(intent) -} - -// Oplog returns the intent representing the oplog, which isn't -// stored with the other intents, because it is dumped and restored in -// a very different way from other collections. -func (manager *Manager) Oplog() *Intent { - return manager.oplogIntent -} - -// SystemIndexes returns the system.indexes bson for a database -func (manager *Manager) SystemIndexes(dbName string) *Intent { - return manager.indexIntents[dbName] -} - -// SystemIndexes returns the databases for which there are system.indexes -func (manager *Manager) SystemIndexDBs() []string { - databases := []string{} - for dbname := range manager.indexIntents { - databases = append(databases, dbname) - } - return databases -} - -// Users returns the intent of the users collection to restore, a special case -func (manager *Manager) Users() *Intent { - return manager.usersIntent -} - -// Roles returns the intent of the user roles collection to restore, a special case -func (manager *Manager) Roles() *Intent { - return manager.rolesIntent -} - -// AuthVersion returns the intent of the version collection to restore, a special case -func (manager *Manager) AuthVersion() *Intent { - return manager.versionIntent -} - -// Finalize processes the intents for prioritization. Currently only two -// kinds of prioritizers are supported. No more "Put" operations may be done -// after finalize is called. -func (manager *Manager) Finalize(pType PriorityType) { - switch pType { - case Legacy: - log.Logv(log.DebugHigh, "finalizing intent manager with legacy prioritizer") - manager.prioritizer = NewLegacyPrioritizer(manager.intentsByDiscoveryOrder) - case LongestTaskFirst: - log.Logv(log.DebugHigh, "finalizing intent manager with longest task first prioritizer") - manager.prioritizer = NewLongestTaskFirstPrioritizer(manager.intentsByDiscoveryOrder) - case MultiDatabaseLTF: - log.Logv(log.DebugHigh, "finalizing intent manager with multi-database longest task first prioritizer") - manager.prioritizer = NewMultiDatabaseLTFPrioritizer(manager.intentsByDiscoveryOrder) - default: - panic("cannot initialize IntentPrioritizer with unknown type") - } - // release these for the garbage collector and to ensure code correctness - manager.intents = nil - manager.intentsByDiscoveryOrder = nil -} - -func (manager *Manager) UsePrioritizer(prioritizer IntentPrioritizer) { - manager.prioritizer = prioritizer -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/intents/intent_prioritizer.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/intents/intent_prioritizer.go deleted file mode 100644 index 280ee9a1e74..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/intents/intent_prioritizer.go +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package intents - -import ( - "container/heap" - "sort" - "sync" -) - -type PriorityType int - -const ( - Legacy PriorityType = iota - LongestTaskFirst - MultiDatabaseLTF -) - -// IntentPrioritizer encapsulates the logic of scheduling intents -// for restoration. It can know about which intents are in the -// process of being restored through the "Finish" hook. -// -// Oplog entries and auth entries are not handled by the prioritizer, -// as these are special cases handled by the regular mongorestore code. -type IntentPrioritizer interface { - Get() *Intent - Finish(*Intent) -} - -//===== Legacy ===== - -// legacyPrioritizer processes the intents in the order they were read off the -// file system, keeping with legacy mongorestore behavior. -type legacyPrioritizer struct { - sync.Mutex - queue []*Intent -} - -func NewLegacyPrioritizer(intentList []*Intent) *legacyPrioritizer { - return &legacyPrioritizer{queue: intentList} -} - -func (legacy *legacyPrioritizer) Get() *Intent { - legacy.Lock() - defer legacy.Unlock() - - if len(legacy.queue) == 0 { - return nil - } - - var intent *Intent - intent, legacy.queue = legacy.queue[0], legacy.queue[1:] - return intent -} - -func (legacy *legacyPrioritizer) Finish(*Intent) { - // no-op - return -} - -//===== Longest Task First ===== - -// longestTaskFirstPrioritizer returns intents in the order of largest -> smallest, -// with views at the front of the list, which is better at minimizing total -// runtime in parallel environments than other simple orderings. -type longestTaskFirstPrioritizer struct { - sync.Mutex - queue []*Intent -} - -// NewLongestTaskFirstPrioritizer returns an initialized LTP prioritizer -func NewLongestTaskFirstPrioritizer(intents []*Intent) *longestTaskFirstPrioritizer { - sort.Sort(BySizeAndView(intents)) - return &longestTaskFirstPrioritizer{ - queue: intents, - } -} - -func (ltf *longestTaskFirstPrioritizer) Get() *Intent { - ltf.Lock() - defer ltf.Unlock() - - if len(ltf.queue) == 0 { - return nil - } - - var intent *Intent - intent, ltf.queue = ltf.queue[0], ltf.queue[1:] - return intent -} - -func (ltf *longestTaskFirstPrioritizer) Finish(*Intent) { - // no-op - return -} - -// BySizeAndView attaches the methods for sort.Interface for sorting intents -// from largest to smallest size, taking into account if it's a view or not. -type BySizeAndView []*Intent - -func (s BySizeAndView) Len() int { return len(s) } -func (s BySizeAndView) Swap(i, j int) { s[i], s[j] = s[j], s[i] } -func (s BySizeAndView) Less(i, j int) bool { - if s[i].IsView() && !s[j].IsView() { - return true - } - if !s[i].IsView() && s[j].IsView() { - return false - } - return s[i].Size > s[j].Size -} - -// For sorting intents from largest to smallest size -type BySize []*Intent - -func (s BySize) Len() int { return len(s) } -func (s BySize) Swap(i, j int) { s[i], s[j] = s[j], s[i] } -func (s BySize) Less(i, j int) bool { return s[i].Size > s[j].Size } - -//===== Multi Database Longest Task First ===== - -// multiDatabaseLTF is designed to properly schedule intents with two constraints: -// 1. it is optimized to run in a multi-processor environment -// 2. it is optimized for parallelism against 2.6's db-level write lock -// These goals result in a design that attempts to have as many different -// database's intents being restored as possible and attempts to restore the -// largest collections first. -// -// If we can have a minimum number of collections in flight for a given db, -// we avoid lock contention in an optimal way on 2.6 systems. That is, -// it is better to have two restore jobs where -// job1 = "test.mycollection" -// job2 = "mydb2.othercollection" -// so that these collections do not compete for the db-level write lock. -// -// We also schedule the largest jobs first, in a greedy fashion, in order -// to minimize total restoration time. Each database's intents are sorted -// by decreasing file size at initialization, so that the largest jobs are -// run first. Admittedly, .bson file size is not a direct predictor of restore -// time, but there is certainly a strong correlation. Note that this attribute -// is secondary to the multi-db scheduling laid out above, since multi-db will -// get us bigger wins in terms of parallelism. -type multiDatabaseLTFPrioritizer struct { - sync.Mutex - dbHeap heap.Interface - counterMap map[string]*dbCounter -} - -// NewMultiDatabaseLTFPrioritizer takes in a list of intents and returns an -// initialized prioritizer. -func NewMultiDatabaseLTFPrioritizer(intents []*Intent) *multiDatabaseLTFPrioritizer { - prioritizer := &multiDatabaseLTFPrioritizer{ - counterMap: map[string]*dbCounter{}, - dbHeap: &DBHeap{}, - } - heap.Init(prioritizer.dbHeap) - // first, create all database counters - for _, intent := range intents { - counter, exists := prioritizer.counterMap[intent.DB] - if !exists { - // initialize a new counter if one doesn't exist for DB - counter = &dbCounter{} - prioritizer.counterMap[intent.DB] = counter - } - counter.collections = append(counter.collections, intent) - } - // then ensure that all the dbCounters have sorted intents - for _, counter := range prioritizer.counterMap { - counter.SortCollectionsBySize() - heap.Push(prioritizer.dbHeap, counter) - } - return prioritizer -} - -// Get returns the next prioritized intent and updates the count of active -// restores for the returned intent's DB. Get is not thread safe, and depends -// on the implementation of the intent manager to lock around it. -func (mdb *multiDatabaseLTFPrioritizer) Get() *Intent { - mdb.Lock() - defer mdb.Unlock() - - if mdb.dbHeap.Len() == 0 { - // we're out of things to return - return nil - } - optimalDB := heap.Pop(mdb.dbHeap).(*dbCounter) - optimalDB.active++ - nextIntent := optimalDB.PopIntent() - // only release the db counter if it's out of collections - if len(optimalDB.collections) > 0 { - heap.Push(mdb.dbHeap, optimalDB) - } - return nextIntent -} - -// Finish decreases the number of active restore jobs for the given intent's -// database, and reshuffles the heap accordingly. Finish is not thread safe, -// and depends on the implementation of the intent manager to lock around it. -func (mdb *multiDatabaseLTFPrioritizer) Finish(intent *Intent) { - mdb.Lock() - defer mdb.Unlock() - - counter := mdb.counterMap[intent.DB] - counter.active-- - // only fix up the heap if the counter is still in the heap - if len(counter.collections) > 0 { - // This is an O(n) operation on the heap. We could make all heap - // operations O(log(n)) if we set up dbCounters to track their own - // position in the heap, but in practice this overhead is likely negligible. - heap.Init(mdb.dbHeap) - } -} - -type dbCounter struct { - active int - collections []*Intent -} - -func (dbc *dbCounter) SortCollectionsBySize() { - sort.Sort(BySize(dbc.collections)) -} - -// PopIntent returns the largest intent remaining for the database -func (dbc *dbCounter) PopIntent() *Intent { - var intent *Intent - if len(dbc.collections) > 0 { - intent, dbc.collections = dbc.collections[0], dbc.collections[1:] - } - return intent -} - -// Returns the largest collection of the databases with the least active restores. -// Implements the container/heap interface. None of its methods are meant to be -// called directly. -type DBHeap []*dbCounter - -func (dbh DBHeap) Len() int { return len(dbh) } -func (dbh DBHeap) Swap(i, j int) { dbh[i], dbh[j] = dbh[j], dbh[i] } -func (dbh DBHeap) Less(i, j int) bool { - if dbh[i].active == dbh[j].active { - // prioritize the largest bson file if dbs have the same number - // of restorations in progress - return dbh[i].collections[0].Size > dbh[j].collections[0].Size - } - return dbh[i].active < dbh[j].active -} - -func (dbh *DBHeap) Push(x interface{}) { - *dbh = append(*dbh, x.(*dbCounter)) -} - -func (dbh *DBHeap) Pop() interface{} { - // for container/heap package: removes the top entry and resizes the heap array - old := *dbh - n := len(old) - toPop := old[n-1] - *dbh = old[0 : n-1] - return toPop -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/intents/intent_prioritizer_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/intents/intent_prioritizer_test.go deleted file mode 100644 index f11a96f6280..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/intents/intent_prioritizer_test.go +++ /dev/null @@ -1,213 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package intents - -import ( - "container/heap" - "testing" - - "github.com/mongodb/mongo-tools/common/testtype" - . "github.com/smartystreets/goconvey/convey" - "gopkg.in/mgo.v2/bson" -) - -func TestLegacyPrioritizer(t *testing.T) { - - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("With a legacyPrioritizer initialized with an ordered intent list", t, func() { - testList := []*Intent{ - {DB: "1"}, - {DB: "2"}, - {DB: "3"}, - } - legacy := NewLegacyPrioritizer(testList) - So(legacy, ShouldNotBeNil) - - Convey("the priority should be defined by 'first-in-first-out'", func() { - it0 := legacy.Get() - it1 := legacy.Get() - it2 := legacy.Get() - it3 := legacy.Get() - So(it3, ShouldBeNil) - So(it0.DB, ShouldBeLessThan, it1.DB) - So(it1.DB, ShouldBeLessThan, it2.DB) - }) - }) -} - -func TestBasicDBHeapBehavior(t *testing.T) { - var dbheap heap.Interface - - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("With an empty dbHeap", t, func() { - dbheap = &DBHeap{} - heap.Init(dbheap) - - Convey("when inserting unordered dbCounters with different active counts", func() { - heap.Push(dbheap, &dbCounter{75, nil}) - heap.Push(dbheap, &dbCounter{121, nil}) - heap.Push(dbheap, &dbCounter{76, nil}) - heap.Push(dbheap, &dbCounter{51, nil}) - heap.Push(dbheap, &dbCounter{82, nil}) - heap.Push(dbheap, &dbCounter{117, nil}) - heap.Push(dbheap, &dbCounter{49, nil}) - heap.Push(dbheap, &dbCounter{101, nil}) - heap.Push(dbheap, &dbCounter{122, nil}) - heap.Push(dbheap, &dbCounter{33, nil}) - heap.Push(dbheap, &dbCounter{0, nil}) - - Convey("they should pop in active order, least to greatest", func() { - prev := -1 - for dbheap.Len() > 0 { - popped := heap.Pop(dbheap).(*dbCounter) - So(popped.active, ShouldBeGreaterThan, prev) - prev = popped.active - } - }) - }) - - Convey("when inserting unordered dbCounters with different bson sizes", func() { - heap.Push(dbheap, &dbCounter{0, []*Intent{{Size: 70}}}) - heap.Push(dbheap, &dbCounter{0, []*Intent{{Size: 1024}}}) - heap.Push(dbheap, &dbCounter{0, []*Intent{{Size: 97}}}) - heap.Push(dbheap, &dbCounter{0, []*Intent{{Size: 3}}}) - heap.Push(dbheap, &dbCounter{0, []*Intent{{Size: 1024 * 1024}}}) - - Convey("they should pop in bson size order, greatest to least", func() { - prev := int64(1024*1024 + 1) // Maximum - for dbheap.Len() > 0 { - popped := heap.Pop(dbheap).(*dbCounter) - So(popped.collections[0].Size, ShouldBeLessThan, prev) - prev = popped.collections[0].Size - } - }) - }) - }) -} - -func TestDBCounterCollectionSorting(t *testing.T) { - - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("With a dbCounter and an unordered collection of intents", t, func() { - dbc := &dbCounter{ - collections: []*Intent{ - {Size: 100}, - {Size: 1000}, - {Size: 1}, - {Size: 10}, - }, - } - - Convey("popping the sorted intents should return in decreasing BSONSize", func() { - dbc.SortCollectionsBySize() - So(dbc.PopIntent().Size, ShouldEqual, 1000) - So(dbc.PopIntent().Size, ShouldEqual, 100) - So(dbc.PopIntent().Size, ShouldEqual, 10) - So(dbc.PopIntent().Size, ShouldEqual, 1) - So(dbc.PopIntent(), ShouldBeNil) - So(dbc.PopIntent(), ShouldBeNil) - }) - }) -} - -func TestBySizeAndView(t *testing.T) { - var prioritizer IntentPrioritizer - - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("With a prioritizer initialized with on a set of intents", t, func() { - intents := []*Intent{ - {C: "non-view2", Size: 32}, - {C: "view", Size: 0, - Options: &bson.D{{"viewOn", true}}, - }, - {C: "non-view1", Size: 1024}, - {C: "non-view3", Size: 2}, - {C: "view", Size: 0, - Options: &bson.D{{"viewOn", true}}, - }, - } - prioritizer = NewLongestTaskFirstPrioritizer(intents) - Convey("getting the sorted intents should produce views first, followed by largest to smallest", func() { - - So(prioritizer.Get().C, ShouldEqual, "view") - So(prioritizer.Get().C, ShouldEqual, "view") - So(prioritizer.Get().C, ShouldEqual, "non-view1") - So(prioritizer.Get().C, ShouldEqual, "non-view2") - So(prioritizer.Get().C, ShouldEqual, "non-view3") - }) - - }) - -} - -func TestSimulatedMultiDBJob(t *testing.T) { - var prioritizer IntentPrioritizer - - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("With a prioritizer initialized with a set of intents", t, func() { - intents := []*Intent{ - {C: "small", DB: "db2", Size: 32}, - {C: "medium", DB: "db2", Size: 128}, - {C: "giant", DB: "db1", Size: 1024}, - {C: "tiny", DB: "db1", Size: 2}, - } - prioritizer = NewMultiDatabaseLTFPrioritizer(intents) - So(prioritizer, ShouldNotBeNil) - - Convey("and a running simulation of two jobs threads:", func() { - Convey("the first two intents should be of different dbs", func() { - i0 := prioritizer.Get() - So(i0, ShouldNotBeNil) - i1 := prioritizer.Get() - So(i1, ShouldNotBeNil) - - Convey("the first intent should be the largest bson file", func() { - So(i0.C, ShouldEqual, "giant") - So(i0.DB, ShouldEqual, "db1") - }) - - Convey("the second intent should be the largest bson file of db2", func() { - So(i1.C, ShouldEqual, "medium") - So(i1.DB, ShouldEqual, "db2") - }) - - Convey("with the second job finishing the smaller intents", func() { - prioritizer.Finish(i1) - i2 := prioritizer.Get() - So(i2, ShouldNotBeNil) - prioritizer.Finish(i2) - i3 := prioritizer.Get() - So(i3, ShouldNotBeNil) - - Convey("the next job should be from db2", func() { - So(i2.C, ShouldEqual, "small") - So(i2.DB, ShouldEqual, "db2") - }) - - Convey("the final job should be from db1", func() { - So(i3.C, ShouldEqual, "tiny") - So(i3.DB, ShouldEqual, "db1") - - Convey("which means that there should be two active db1 jobs", func() { - counter := prioritizer.(*multiDatabaseLTFPrioritizer).counterMap["db1"] - So(counter.active, ShouldEqual, 2) - }) - }) - - Convey("the heap should now be empty", func() { - So(prioritizer.(*multiDatabaseLTFPrioritizer).dbHeap.Len(), ShouldEqual, 0) - }) - }) - }) - }) - }) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/intents/intent_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/intents/intent_test.go deleted file mode 100644 index 693994d1394..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/intents/intent_test.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package intents - -import ( - "testing" - - "github.com/mongodb/mongo-tools/common/testtype" - . "github.com/smartystreets/goconvey/convey" -) - -func TestIntentManager(t *testing.T) { - var manager *Manager - - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - Convey("With an empty IntentManager", t, func() { - manager = NewIntentManager() - So(manager, ShouldNotBeNil) - - Convey("putting a series of added bson intents", func() { - manager.Put(&Intent{DB: "1", C: "1", Location: "/b1/"}) - manager.Put(&Intent{DB: "1", C: "2", Location: "/b2/"}) - manager.Put(&Intent{DB: "1", C: "3", Location: "/b3/"}) - manager.Put(&Intent{DB: "2", C: "1", Location: "/b4/"}) - So(len(manager.intentsByDiscoveryOrder), ShouldEqual, 4) - So(len(manager.intents), ShouldEqual, 4) - - Convey("and then some matching metadata intents", func() { - manager.Put(&Intent{DB: "2", C: "1", MetadataLocation: "/4m/"}) - manager.Put(&Intent{DB: "1", C: "3", MetadataLocation: "/3m/"}) - manager.Put(&Intent{DB: "1", C: "1", MetadataLocation: "/1m/"}) - manager.Put(&Intent{DB: "1", C: "2", MetadataLocation: "/2m/"}) - - Convey("the size of the queue should be unchanged", func() { - So(len(manager.intentsByDiscoveryOrder), ShouldEqual, 4) - So(len(manager.intents), ShouldEqual, 4) - }) - - Convey("popping them from the IntentManager", func() { - manager.Finalize(Legacy) - it0 := manager.Pop() - it1 := manager.Pop() - it2 := manager.Pop() - it3 := manager.Pop() - it4 := manager.Pop() - So(it4, ShouldBeNil) - - Convey("should return them in insert order", func() { - So(*it0, ShouldResemble, - Intent{DB: "1", C: "1", Location: "/b1/", MetadataLocation: "/1m/"}) - So(*it1, ShouldResemble, - Intent{DB: "1", C: "2", Location: "/b2/", MetadataLocation: "/2m/"}) - So(*it2, ShouldResemble, - Intent{DB: "1", C: "3", Location: "/b3/", MetadataLocation: "/3m/"}) - So(*it3, ShouldResemble, - Intent{DB: "2", C: "1", Location: "/b4/", MetadataLocation: "/4m/"}) - }) - }) - }) - - Convey("but adding non-matching intents", func() { - manager.Put(&Intent{DB: "7", C: "49", MetadataLocation: "/5/"}) - manager.Put(&Intent{DB: "27", C: "B", MetadataLocation: "/6/"}) - - Convey("should increase the size, because they are not merged in", func() { - So(len(manager.intentsByDiscoveryOrder), ShouldEqual, 6) - So(len(manager.intents), ShouldEqual, 6) - }) - }) - - Convey("using the Peek() method", func() { - peeked := manager.Peek() - So(peeked, ShouldNotBeNil) - So(peeked, ShouldResemble, manager.intentsByDiscoveryOrder[0]) - - Convey("modifying the returned copy should not modify the original", func() { - peeked.DB = "SHINY NEW VALUE" - So(peeked, ShouldNotResemble, manager.intentsByDiscoveryOrder[0]) - }) - }) - }) - }) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/tlsgo/config.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/tlsgo/config.go deleted file mode 100644 index 8d3971b537b..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/tlsgo/config.go +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2018-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -// This file contains code adapted from the MongoDB Go Driver. - -// Package tlsgo provides a mgo connection using Go's native TLS library. -package tlsgo - -import ( - "crypto/tls" - "crypto/x509" - "encoding/asn1" - "encoding/hex" - "encoding/pem" - "fmt" - "io/ioutil" - "strings" -) - -// TLSConfig contains options for configuring an SSL connection to the server. -type TLSConfig struct { - caCert *x509.Certificate - clientCert *tls.Certificate - insecure bool -} - -// NewTLSConfig creates a new TLSConfig. -func NewTLSConfig() *TLSConfig { - cfg := &TLSConfig{} - - return cfg -} - -// SetInsecure sets whether the client should verify the server's certificate chain and hostnames. -func (c *TLSConfig) SetInsecure(allow bool) { - c.insecure = allow -} - -// AddClientCertFromFile adds a client certificate to the configuration given a path to the -// containing file and returns the certificate's subject name. -func (c *TLSConfig) AddClientCertFromFile(clientFile, password string) (string, error) { - data, err := ioutil.ReadFile(clientFile) - if err != nil { - return "", err - } - - certPEM, err := loadPEMBlock(data, "CERTIFICATE") - if err != nil { - return "", err - } - - keyPEM, err := loadPEMBlock(data, "PRIVATE KEY") - if err != nil { - return "", err - } - // This check only covers encrypted PEM data with a DEK-Info header. It - // does not detect unencrypted PEM containing PKCS#8 format data with an - // encrypted private key. - if x509.IsEncryptedPEMBlock(keyPEM) { - if password == "" { - return "", fmt.Errorf("No password provided to decrypt private key") - } - decrypted, err := x509.DecryptPEMBlock(keyPEM, []byte(password)) - if err != nil { - return "", err - } - keyPEM = &pem.Block{Bytes: decrypted, Type: keyPEM.Type} - } - - if strings.Contains(keyPEM.Type, "ENCRYPTED") { - return "", fmt.Errorf("PKCS#8 encrypted private keys are not supported") - } - - cert, err := tls.X509KeyPair(pem.EncodeToMemory(certPEM), pem.EncodeToMemory(keyPEM)) - if err != nil { - return "", err - } - - c.clientCert = &cert - - // The documentation for the tls.X509KeyPair indicates that the Leaf - // certificate is not retained. Because there isn't any way of creating a - // tls.Certificate from an x509.Certificate short of calling X509KeyPair - // on the raw bytes, we're forced to parse the certificate over again to - // get the subject name. - crt, err := x509.ParseCertificate(certPEM.Bytes) - if err != nil { - return "", err - } - - return x509CertSubject(crt), nil -} - -// AddCaCertFromFile adds a root CA certificate to the configuration given a path to the containing file. -func (c *TLSConfig) AddCaCertFromFile(caFile string) error { - data, err := ioutil.ReadFile(caFile) - if err != nil { - return err - } - - certBytes, err := loadCertBytes(data) - if err != nil { - return err - } - - cert, err := x509.ParseCertificate(certBytes) - if err != nil { - return err - } - - c.caCert = cert - - return nil -} - -// MakeConfig constructs a new tls.Config from the configuration specified. -func (c *TLSConfig) MakeConfig() (*tls.Config, error) { - cfg := &tls.Config{} - - if c.clientCert != nil { - cfg.Certificates = []tls.Certificate{*c.clientCert} - } - - if c.caCert == nil { - roots, err := loadSystemCAs() - if err != nil { - return nil, err - } - cfg.RootCAs = roots - } else { - cfg.RootCAs = x509.NewCertPool() - cfg.RootCAs.AddCert(c.caCert) - } - - cfg.InsecureSkipVerify = c.insecure - - return cfg, nil -} - -func loadCertBytes(data []byte) ([]byte, error) { - b, err := loadPEMBlock(data, "CERTIFICATE") - if err != nil { - return nil, err - } - return b.Bytes, nil -} - -func loadPEMBlock(data []byte, blocktype string) (*pem.Block, error) { - var b *pem.Block - - for b == nil { - if data == nil || len(data) == 0 { - return nil, fmt.Errorf("no block of type %s found in .pem file", blocktype) - } - - block, rest := pem.Decode(data) - if block == nil { - return nil, fmt.Errorf("invalid .pem file") - } - - if strings.Contains(block.Type, blocktype) { - if b != nil { - return nil, fmt.Errorf("multiple %s sections in .pem file", blocktype) - } - b = block - } - - data = rest - } - - return b, nil -} - -// Because the functionality to convert a pkix.Name to a string wasn't added until Go 1.10, we -// need to copy the implementation (along with the attributeTypeNames map below). -func x509CertSubject(cert *x509.Certificate) string { - r := cert.Subject.ToRDNSequence() - - s := "" - for i := 0; i < len(r); i++ { - rdn := r[len(r)-1-i] - if i > 0 { - s += "," - } - for j, tv := range rdn { - if j > 0 { - s += "+" - } - - oidString := tv.Type.String() - typeName, ok := attributeTypeNames[oidString] - if !ok { - derBytes, err := asn1.Marshal(tv.Value) - if err == nil { - s += oidString + "=#" + hex.EncodeToString(derBytes) - continue // No value escaping necessary. - } - - typeName = oidString - } - - valueString := fmt.Sprint(tv.Value) - escaped := make([]rune, 0, len(valueString)) - - for k, c := range valueString { - escape := false - - switch c { - case ',', '+', '"', '\\', '<', '>', ';': - escape = true - - case ' ': - escape = k == 0 || k == len(valueString)-1 - - case '#': - escape = k == 0 - } - - if escape { - escaped = append(escaped, '\\', c) - } else { - escaped = append(escaped, c) - } - } - - s += typeName + "=" + string(escaped) - } - } - - return s -} - -var attributeTypeNames = map[string]string{ - "2.5.4.6": "C", - "2.5.4.10": "O", - "2.5.4.11": "OU", - "2.5.4.3": "CN", - "2.5.4.5": "SERIALNUMBER", - "2.5.4.7": "L", - "2.5.4.8": "ST", - "2.5.4.9": "STREET", - "2.5.4.17": "POSTALCODE", -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/tlsgo/rootcerts.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/tlsgo/rootcerts.go deleted file mode 100644 index ee3ec3769f1..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/tlsgo/rootcerts.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2018-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -// -// Based on https://github.com/hashicorp/go-rootcerts by HashiCorp -// See THIRD-PARTY-NOTICES for original license terms. - -// +build !darwin - -package tlsgo - -import ( - "crypto/x509" -) - -// Stubbed for non-darwin systems. By returning nil, the Go library -// will use its own code for finding system certs. -func loadSystemCAs() (*x509.CertPool, error) { - return nil, nil -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/tlsgo/rootcerts_darwin.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/tlsgo/rootcerts_darwin.go deleted file mode 100644 index 72c7a9116ad..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/tlsgo/rootcerts_darwin.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2018-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -// -// Based on https://github.com/hashicorp/go-rootcerts by HashiCorp -// See THIRD-PARTY-NOTICES for original license terms. - -package tlsgo - -import ( - "crypto/x509" - "os/exec" - "os/user" - "path" -) - -// loadSystemCAs has special behavior on Darwin systems to work around -// bugs loading certs from keychains. See this GitHub issues query: -// https://github.com/golang/go/issues?utf8=%E2%9C%93&q=is%3Aissue+darwin+keychain -func loadSystemCAs() (*x509.CertPool, error) { - pool := x509.NewCertPool() - - for _, keychain := range certKeychains() { - err := addCertsFromKeychain(pool, keychain) - if err != nil { - return nil, err - } - } - - return pool, nil -} - -func addCertsFromKeychain(pool *x509.CertPool, keychain string) error { - cmd := exec.Command("/usr/bin/security", "find-certificate", "-a", "-p", keychain) - data, err := cmd.Output() - if err != nil { - return err - } - - pool.AppendCertsFromPEM(data) - - return nil -} - -func certKeychains() []string { - keychains := []string{ - "/System/Library/Keychains/SystemRootCertificates.keychain", - "/Library/Keychains/System.keychain", - } - user, err := user.Current() - if err == nil { - loginKeychain := path.Join(user.HomeDir, "Library", "Keychains", "login.keychain") - keychains = append(keychains, loginKeychain) - } - return keychains -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/progress/manager.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/progress/manager.go deleted file mode 100644 index 94d4c90e113..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/progress/manager.go +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package progress - -import ( - "fmt" - "io" - "sync" - "time" - - "github.com/mongodb/mongo-tools/common/text" -) - -// Manager is an interface which tools can use to registers progressors which -// track the progress of any arbitrary operation. -type Manager interface { - // Attach registers the progressor with the manager under the given name. - // Any call to Attach must have a matching call to Detach. - Attach(name string, progressor Progressor) - - // Detach removes the progressor with the given name from the manager - Detach(name string) -} - -const GridPadding = 2 - -// BarWriter implements Manager. It periodically prints the status of all of its -// progressors in the form of pretty progress bars. It handles thread-safe -// synchronized progress bar writing, so that its progressors are written in a -// group at a given interval. It maintains insertion order when printing, such -// that new bars appear at the bottom of the group. -type BarWriter struct { - sync.Mutex - - waitTime time.Duration - writer io.Writer - bars []*Bar - stopChan chan struct{} - barLength int - isBytes bool -} - -// NewBarWriter returns an initialized BarWriter with the given bar length and -// byte-formatting toggle, waiting the given duration between writes -func NewBarWriter(w io.Writer, waitTime time.Duration, barLength int, isBytes bool) *BarWriter { - return &BarWriter{ - waitTime: waitTime, - writer: w, - stopChan: make(chan struct{}), - barLength: barLength, - isBytes: isBytes, - } -} - -// Attach registers the given progressor with the manager -func (manager *BarWriter) Attach(name string, progressor Progressor) { - pb := &Bar{ - Name: name, - Watching: progressor, - BarLength: manager.barLength, - IsBytes: manager.isBytes, - } - pb.validate() - - manager.Lock() - defer manager.Unlock() - - // make sure we are not adding the same bar again - for _, bar := range manager.bars { - if bar.Name == name { - panic(fmt.Sprintf("progress bar with name '%s' already exists in manager", name)) - } - } - - manager.bars = append(manager.bars, pb) -} - -// Detach removes the progressor with the given name from the manager. Insert -// order is maintained for consistent ordering of the printed bars. -func (manager *BarWriter) Detach(name string) { - manager.Lock() - defer manager.Unlock() - var pb *Bar - for _, bar := range manager.bars { - if bar.Name == name { - pb = bar - break - } - } - if pb == nil { - panic("could not find progressor") - } - - grid := &text.GridWriter{ - ColumnPadding: GridPadding, - } - if pb.hasRendered { - // if we've rendered this bar at least once, render it one last time - pb.renderToGridRow(grid) - } - grid.FlushRows(manager.writer) - - updatedBars := make([]*Bar, 0, len(manager.bars)-1) - for _, bar := range manager.bars { - // move all bars to the updated list except for the bar we want to detach - if bar.Name != pb.Name { - updatedBars = append(updatedBars, bar) - } - } - - manager.bars = updatedBars -} - -// helper to render all bars in order -func (manager *BarWriter) renderAllBars() { - manager.Lock() - defer manager.Unlock() - grid := &text.GridWriter{ - ColumnPadding: GridPadding, - } - for _, bar := range manager.bars { - bar.renderToGridRow(grid) - } - grid.FlushRows(manager.writer) - // add padding of one row if we have more than one active bar - if len(manager.bars) > 1 { - // we just write an empty array here, since a write call of any - // length to our log.Writer will trigger a new logline. - manager.writer.Write([]byte{}) - } -} - -// Start kicks of the timed batch writing of progress bars. -func (manager *BarWriter) Start() { - if manager.writer == nil { - panic("Cannot use a progress.BarWriter with an unset Writer") - } - go manager.start() -} - -func (manager *BarWriter) start() { - if manager.waitTime <= 0 { - manager.waitTime = DefaultWaitTime - } - ticker := time.NewTicker(manager.waitTime) - defer ticker.Stop() - - for { - select { - case <-manager.stopChan: - return - case <-ticker.C: - manager.renderAllBars() - } - } -} - -// Stop ends the main manager goroutine, stopping the manager's bars -// from being rendered. -func (manager *BarWriter) Stop() { - manager.stopChan <- struct{}{} -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/progress/manager_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/progress/manager_test.go deleted file mode 100644 index 1fc4d7f7c1c..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/progress/manager_test.go +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package progress - -import ( - "bytes" - "github.com/mongodb/mongo-tools/common/testtype" - . "github.com/smartystreets/goconvey/convey" - "strconv" - "strings" - "sync" - "testing" - "time" -) - -type safeBuffer struct { - sync.Mutex - bytes.Buffer -} - -func (b *safeBuffer) Write(p []byte) (n int, err error) { - b.Lock() - defer b.Unlock() - return b.Buffer.Write(p) -} - -func (b *safeBuffer) String() string { - b.Lock() - defer b.Unlock() - return b.Buffer.String() -} - -func (b *safeBuffer) Reset() { - b.Lock() - defer b.Unlock() - b.Buffer.Reset() -} - -func TestManagerAttachAndDetach(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - writeBuffer := new(safeBuffer) - var manager *BarWriter - - Convey("With an empty progress.BarWriter", t, func() { - manager = NewBarWriter(writeBuffer, time.Second, 10, false) - So(manager, ShouldNotBeNil) - - Convey("adding 3 bars", func() { - progressor := NewCounter(10) - progressor.Inc(5) - manager.Attach("TEST1", progressor) - manager.Attach("TEST2", progressor) - manager.Attach("TEST3", progressor) - - So(len(manager.bars), ShouldEqual, 3) - - Convey("should write all three bars ar once", func() { - manager.renderAllBars() - writtenString := writeBuffer.String() - So(writtenString, ShouldContainSubstring, "TEST1") - So(writtenString, ShouldContainSubstring, "TEST2") - So(writtenString, ShouldContainSubstring, "TEST3") - }) - - Convey("detaching the second bar", func() { - manager.Detach("TEST2") - So(len(manager.bars), ShouldEqual, 2) - - Convey("should print 1,3", func() { - manager.renderAllBars() - writtenString := writeBuffer.String() - So(writtenString, ShouldContainSubstring, "TEST1") - So(writtenString, ShouldNotContainSubstring, "TEST2") - So(writtenString, ShouldContainSubstring, "TEST3") - So( - strings.Index(writtenString, "TEST1"), - ShouldBeLessThan, - strings.Index(writtenString, "TEST3"), - ) - }) - - Convey("but adding a new bar should print 1,2,4", func() { - manager.Attach("TEST4", progressor) - - So(len(manager.bars), ShouldEqual, 3) - manager.renderAllBars() - writtenString := writeBuffer.String() - So(writtenString, ShouldContainSubstring, "TEST1") - So(writtenString, ShouldNotContainSubstring, "TEST2") - So(writtenString, ShouldContainSubstring, "TEST3") - So(writtenString, ShouldContainSubstring, "TEST4") - So( - strings.Index(writtenString, "TEST1"), - ShouldBeLessThan, - strings.Index(writtenString, "TEST3"), - ) - So( - strings.Index(writtenString, "TEST3"), - ShouldBeLessThan, - strings.Index(writtenString, "TEST4"), - ) - }) - Reset(func() { writeBuffer.Reset() }) - - }) - Reset(func() { writeBuffer.Reset() }) - }) - }) -} - -func TestManagerStartAndStop(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - writeBuffer := new(safeBuffer) - var manager *BarWriter - - Convey("With a progress.BarWriter with a waitTime of 10 ms and one bar", t, func() { - manager = NewBarWriter(writeBuffer, time.Millisecond*10, 10, false) - So(manager, ShouldNotBeNil) - watching := NewCounter(10) - watching.Inc(5) - manager.Attach("TEST", watching) - - So(manager.waitTime, ShouldEqual, time.Millisecond*10) - So(len(manager.bars), ShouldEqual, 1) - - Convey("running the manager for 45 ms and stopping", func() { - manager.Start() - time.Sleep(time.Millisecond * 45) // enough time for the manager to write 4 times - manager.Stop() - - Convey("should generate 4 writes of the bar", func() { - output := writeBuffer.String() - So(strings.Count(output, "TEST"), ShouldEqual, 4) - }) - - Convey("starting and stopping the manager again should not panic", func() { - So(manager.Start, ShouldNotPanic) - So(manager.Stop, ShouldNotPanic) - }) - }) - }) -} - -func TestNumberOfWrites(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - var cw *CountWriter - var manager *BarWriter - Convey("With a test manager and counting writer", t, func() { - cw = new(CountWriter) - manager = NewBarWriter(cw, time.Millisecond*10, 10, false) - So(manager, ShouldNotBeNil) - - manager.Attach("1", NewCounter(10)) - - Convey("with one attached bar", func() { - So(len(manager.bars), ShouldEqual, 1) - - Convey("only one write should be made per render", func() { - manager.renderAllBars() - So(cw.Count(), ShouldEqual, 1) - }) - }) - - Convey("with two bars attached", func() { - manager.Attach("2", NewCounter(10)) - So(len(manager.bars), ShouldEqual, 2) - - Convey("three writes should be made per render, since an empty write is added", func() { - manager.renderAllBars() - So(cw.Count(), ShouldEqual, 3) - }) - }) - - Convey("with 57 bars attached", func() { - for i := 2; i <= 57; i++ { - manager.Attach(strconv.Itoa(i), NewCounter(10)) - } - So(len(manager.bars), ShouldEqual, 57) - - Convey("58 writes should be made per render, since an empty write is added", func() { - manager.renderAllBars() - So(cw.Count(), ShouldEqual, 58) - }) - }) - }) -} - -// helper type for counting calls to a writer -type CountWriter int - -func (cw CountWriter) Count() int { - return int(cw) -} - -func (cw *CountWriter) Write(b []byte) (int, error) { - *cw++ - return len(b), nil -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/progress/progress_bar.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/progress/progress_bar.go deleted file mode 100644 index 07059d4f18e..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/progress/progress_bar.go +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -// Package progress exposes utilities to asynchronously monitor and display processing progress. -package progress - -import ( - "bytes" - "fmt" - "io" - "time" - - "github.com/mongodb/mongo-tools/common/text" -) - -const ( - DefaultWaitTime = 3 * time.Second - BarFilling = "#" - BarEmpty = "." - BarLeft = "[" - BarRight = "]" -) - -// Bar is a tool for concurrently monitoring the progress -// of a task with a simple linear ASCII visualization -type Bar struct { - // Name is an identifier printed along with the bar - Name string - // BarLength is the number of characters used to print the bar - BarLength int - - // IsBytes denotes whether byte-specific formatting (kB, MB, GB) should - // be applied to the numeric output - IsBytes bool - - // Watching is the object that implements the Progressor to expose the - // values necessary for calculation - Watching Progressor - - // Writer is where the Bar is written out to - Writer io.Writer - // WaitTime is the time to wait between writing the bar - WaitTime time.Duration - - stopChan chan struct{} - stopChanSync chan struct{} - - // hasRendered indicates that the bar has been rendered at least once - // and implies that when detaching should be rendered one more time - hasRendered bool -} - -// Start starts the Bar goroutine. Once Start is called, a bar will -// be written to the given Writer at regular intervals. The goroutine -// can only be stopped manually using the Stop() method. The Bar -// must be set up before calling this. Panics if Start has already been called. -func (pb *Bar) Start() { - pb.validate() - // we only check for the writer if we're using a single bar without a manager - if pb.Writer == nil { - panic("Cannot use a Bar with an unset Writer") - } - pb.stopChan = make(chan struct{}) - pb.stopChanSync = make(chan struct{}) - - go pb.start() -} - -// validate does a set of sanity checks against the progress bar, and panics -// if the bar is unfit for use -func (pb *Bar) validate() { - if pb.Watching == nil { - panic("Cannot use a Bar with a nil Watching") - } - if pb.stopChan != nil { - panic("Cannot start a Bar more than once") - } -} - -// Stop kills the Bar goroutine, stopping it from writing. -// Generally called as -// myBar.Start() -// defer myBar.Stop() -// to stop leakage -// Stop() needs to be synchronous in order that when pb.Stop() is called -// all of the rendering has completed -func (pb *Bar) Stop() { - close(pb.stopChan) - <-pb.stopChanSync -} - -func (pb *Bar) formatCounts() (string, string) { - currentCount, maxCount := pb.Watching.Progress() - if pb.IsBytes { - return text.FormatByteAmount(maxCount), text.FormatByteAmount(currentCount) - } - return fmt.Sprintf("%v", maxCount), fmt.Sprintf("%v", currentCount) -} - -// computes all necessary values renders to the bar's Writer -func (pb *Bar) renderToWriter() { - pb.hasRendered = true - currentCount, maxCount := pb.Watching.Progress() - maxStr, currentStr := pb.formatCounts() - if maxCount == 0 { - // if we have no max amount, just print a count - fmt.Fprintf(pb.Writer, "%v\t%v", pb.Name, currentStr) - return - } - // otherwise, print a bar and percents - percent := float64(currentCount) / float64(maxCount) - fmt.Fprintf(pb.Writer, "%v %v\t%s/%s (%2.1f%%)", - drawBar(pb.BarLength, percent), - pb.Name, - currentStr, - maxStr, - percent*100, - ) -} - -func (pb *Bar) renderToGridRow(grid *text.GridWriter) { - pb.hasRendered = true - currentCount, maxCount := pb.Watching.Progress() - maxStr, currentStr := pb.formatCounts() - if maxCount == 0 { - // if we have no max amount, just print a count - grid.WriteCells(pb.Name, currentStr) - } else { - percent := float64(currentCount) / float64(maxCount) - grid.WriteCells( - drawBar(pb.BarLength, percent), - pb.Name, - fmt.Sprintf("%s/%s", currentStr, maxStr), - fmt.Sprintf("(%2.1f%%)", percent*100), - ) - } - grid.EndRow() -} - -// the main concurrent loop -func (pb *Bar) start() { - if pb.WaitTime <= 0 { - pb.WaitTime = DefaultWaitTime - } - ticker := time.NewTicker(pb.WaitTime) - defer ticker.Stop() - - for { - select { - case <-pb.stopChan: - if pb.hasRendered { - // if we've rendered this bar at least once, render it one last time - pb.renderToWriter() - } - close(pb.stopChanSync) - return - case <-ticker.C: - pb.renderToWriter() - } - } -} - -// drawBar returns a drawn progress bar of a given width and percentage -// as a string. Examples: -// [........................] -// [###########.............] -// [########################] -func drawBar(spaces int, percent float64) string { - if spaces <= 0 { - return "" - } - var strBuffer bytes.Buffer - strBuffer.WriteString(BarLeft) - - // the number of "#" to draw - fullSpaces := int(percent * float64(spaces)) - - // some bounds for ensuring a constant width, even with weird inputs - if fullSpaces > spaces { - fullSpaces = spaces - } - if fullSpaces < 0 { - fullSpaces = 0 - } - - // write the "#"s for the current percentage - for i := 0; i < fullSpaces; i++ { - strBuffer.WriteString(BarFilling) - } - // fill out the remainder of the bar - for i := 0; i < spaces-fullSpaces; i++ { - strBuffer.WriteString(BarEmpty) - } - strBuffer.WriteString(BarRight) - return strBuffer.String() -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/progress/progress_bar_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/progress/progress_bar_test.go deleted file mode 100644 index f5851196ada..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/progress/progress_bar_test.go +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -// +build !race - -// Disable race detector since these tests are inherently racy -package progress - -import ( - "bytes" - . "github.com/smartystreets/goconvey/convey" - "strings" - "testing" - "time" - - "github.com/mongodb/mongo-tools/common/testtype" -) - -func TestBasicProgressBar(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - - writeBuffer := &bytes.Buffer{} - - Convey("With a simple ProgressBar", t, func() { - watching := NewCounter(10) - pbar := &Bar{ - Name: "\nTEST", - Watching: watching, - WaitTime: 3 * time.Millisecond, - Writer: writeBuffer, - BarLength: 10, - } - - Convey("running it while incrementing its counter", func() { - pbar.Start() - // TODO make this test non-racy and reliable - time.Sleep(10 * time.Millisecond) - // iterate though each value 1-10, sleeping to make sure it is written - for localCounter := 0; localCounter < 10; localCounter++ { - watching.Inc(1) - time.Sleep(5 * time.Millisecond) - } - pbar.Stop() - - Convey("the messages written in the buffer should cover all states", func() { - results := writeBuffer.String() - So(results, ShouldContainSubstring, "TEST") - So(results, ShouldContainSubstring, BarLeft) - So(results, ShouldContainSubstring, BarRight) - So(results, ShouldContainSubstring, BarFilling) - So(results, ShouldContainSubstring, BarEmpty) - So(results, ShouldContainSubstring, "0/10") - So(results, ShouldContainSubstring, "1/10") - So(results, ShouldContainSubstring, "2/10") - So(results, ShouldContainSubstring, "3/10") - So(results, ShouldContainSubstring, "4/10") - So(results, ShouldContainSubstring, "5/10") - So(results, ShouldContainSubstring, "6/10") - So(results, ShouldContainSubstring, "7/10") - So(results, ShouldContainSubstring, "8/10") - So(results, ShouldContainSubstring, "9/10") - So(results, ShouldContainSubstring, "10.0%") - }) - }) - }) -} - -func TestProgressBarWithNoMax(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - writeBuffer := &bytes.Buffer{} - - Convey("With a simple ProgressBar with no max value", t, func() { - watching := NewCounter(0) - watching.Inc(5) - pbar := &Bar{ - Name: "test", - Watching: watching, - Writer: writeBuffer, - } - Convey("rendering the progress should still work, but not draw a bar", func() { - pbar.renderToWriter() - So(writeBuffer.String(), ShouldContainSubstring, "5") - So(writeBuffer.String(), ShouldContainSubstring, "test") - So(writeBuffer.String(), ShouldNotContainSubstring, "[") - So(writeBuffer.String(), ShouldNotContainSubstring, "]") - }) - }) -} - -func TestBarConcurrency(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - writeBuffer := &bytes.Buffer{} - - Convey("With a simple ProgressBar", t, func() { - watching := NewCounter(1000) - watching.Inc(777) - pbar := &Bar{ - Name: "\nTEST", - Watching: watching, - WaitTime: 10 * time.Millisecond, - Writer: writeBuffer, - } - - Convey("if it rendered only once", func() { - pbar.Start() - time.Sleep(15 * time.Millisecond) - watching.Inc(1) - results := writeBuffer.String() - So(results, ShouldContainSubstring, "777") - So(results, ShouldNotContainSubstring, "778") - - Convey("it will render a second time on stop", func() { - pbar.Stop() - results := writeBuffer.String() - So(results, ShouldContainSubstring, "777") - So(results, ShouldContainSubstring, "778") - - Convey("and trying to start or stop the bar again should panic", func() { - So(func() { pbar.Start() }, ShouldPanic) - So(func() { pbar.Stop() }, ShouldPanic) - }) - }) - }) - }) -} - -func TestBarDrawing(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - Convey("Drawing some test bars and checking their character counts", t, func() { - Convey("20 wide @ 50%", func() { - b := drawBar(20, .5) - So(strings.Count(b, BarFilling), ShouldEqual, 10) - So(strings.Count(b, BarEmpty), ShouldEqual, 10) - So(b, ShouldContainSubstring, BarLeft) - So(b, ShouldContainSubstring, BarRight) - }) - Convey("100 wide @ 50%", func() { - b := drawBar(100, .5) - So(strings.Count(b, BarFilling), ShouldEqual, 50) - So(strings.Count(b, BarEmpty), ShouldEqual, 50) - }) - Convey("100 wide @ 99.9999%", func() { - b := drawBar(100, .999999) - So(strings.Count(b, BarFilling), ShouldEqual, 99) - So(strings.Count(b, BarEmpty), ShouldEqual, 1) - }) - Convey("9 wide @ 72%", func() { - b := drawBar(9, .72) - So(strings.Count(b, BarFilling), ShouldEqual, 6) - So(strings.Count(b, BarEmpty), ShouldEqual, 3) - }) - Convey("10 wide @ 0%", func() { - b := drawBar(10, 0) - So(strings.Count(b, BarFilling), ShouldEqual, 0) - So(strings.Count(b, BarEmpty), ShouldEqual, 10) - }) - Convey("10 wide @ 100%", func() { - b := drawBar(10, 1) - So(strings.Count(b, BarFilling), ShouldEqual, 10) - So(strings.Count(b, BarEmpty), ShouldEqual, 0) - }) - Convey("10 wide @ -60%", func() { - b := drawBar(10, -0.6) - So(strings.Count(b, BarFilling), ShouldEqual, 0) - So(strings.Count(b, BarEmpty), ShouldEqual, 10) - }) - Convey("10 wide @ 160%", func() { - b := drawBar(10, 1.6) - So(strings.Count(b, BarFilling), ShouldEqual, 10) - So(strings.Count(b, BarEmpty), ShouldEqual, 0) - }) - }) -} - -func TestBarUnits(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - writeBuffer := &bytes.Buffer{} - - Convey("With a simple ProgressBar with IsBytes==true", t, func() { - watching := NewCounter(1024 * 1024) - watching.Inc(777) - pbar := &Bar{ - Name: "\nTEST", - Watching: watching, - WaitTime: 10 * time.Millisecond, - Writer: writeBuffer, - IsBytes: true, - } - - Convey("the written output should contain units", func() { - pbar.renderToWriter() - So(writeBuffer.String(), ShouldContainSubstring, "B") - So(writeBuffer.String(), ShouldContainSubstring, "MB") - }) - }) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/progress/progressor.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/progress/progressor.go deleted file mode 100644 index 606ff3dd6c8..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/progress/progressor.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -// Package progress exposes utilities to asynchronously monitor and display processing progress. -package progress - -import ( - "sync/atomic" -) - -// Progressor can be implemented to allow an object to hook up to a progress.Bar. -type Progressor interface { - // Progress returns a pair of integers: the amount completed and the total - // amount to reach 100%. This method is called by progress.Bar to determine - // what percentage to display. - Progress() (current, max int64) -} - -// Updateable is a Progressor which also exposes the ability for the progressing -// value to be updated. -type Updateable interface { - Progressor - - // Inc increments the current progress counter by the given amount. - Inc(amount int64) - - // Set sets the progress counter to the given amount. - Set(amount int64) -} - -// countProgressor is an implementation of Progressor that uses -type countProgressor struct { - max, current int64 -} - -func (c *countProgressor) Progress() (int64, int64) { - current := atomic.LoadInt64(&c.current) - return current, c.max -} - -func (c *countProgressor) Inc(amount int64) { - atomic.AddInt64(&c.current, amount) -} - -func (c *countProgressor) Set(amount int64) { - atomic.StoreInt64(&c.current, amount) -} - -func NewCounter(max int64) *countProgressor { - return &countProgressor{max, 0} -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/signals/signals.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/signals/signals.go deleted file mode 100644 index 4ee7f5f90ec..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/signals/signals.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package signals - -import ( - "github.com/mongodb/mongo-tools/common/log" - "github.com/mongodb/mongo-tools/common/util" - - "os" - "os/signal" - "syscall" -) - -// Handle is like HandleWithInterrupt but it doesn't take a finalizer and will -// exit immediately after the first signal is received. -func Handle() chan struct{} { - return HandleWithInterrupt(nil) -} - -// HandleWithInterrupt starts a goroutine which listens for SIGTERM, SIGINT, and -// SIGKILL and explicitly ignores SIGPIPE. It calls the finalizer function when -// the first signal is received and forcibly terminates the program after the -// second. If a nil function is provided, the program will exit after the first -// signal. -func HandleWithInterrupt(finalizer func()) chan struct{} { - finishedChan := make(chan struct{}) - go handleSignals(finalizer, finishedChan) - return finishedChan -} - -func handleSignals(finalizer func(), finishedChan chan struct{}) { - // explicitly ignore SIGPIPE; the tools should deal with write errors - noopChan := make(chan os.Signal) - signal.Notify(noopChan, syscall.SIGPIPE) - - log.Logv(log.DebugLow, "will listen for SIGTERM, SIGINT, and SIGKILL") - sigChan := make(chan os.Signal, 2) - signal.Notify(sigChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGKILL) - defer signal.Stop(sigChan) - if finalizer != nil { - select { - case sig := <-sigChan: - // first signal use finalizer to terminate cleanly - log.Logvf(log.Always, "signal '%s' received; attempting to shut down", sig) - finalizer() - case <-finishedChan: - return - } - } - select { - case sig := <-sigChan: - // second signal exits immediately - log.Logvf(log.Always, "signal '%s' received; forcefully terminating", sig) - os.Exit(util.ExitKill) - case <-finishedChan: - return - } -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/testutil/auth.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/testutil/auth.go deleted file mode 100644 index 37aa3beda4f..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/testutil/auth.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package testutil - -import ( - "fmt" - "github.com/mongodb/mongo-tools/common/options" - "github.com/mongodb/mongo-tools/common/testtype" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" - "os" -) - -var ( - UserAdmin = "uAdmin" - UserAdminPassword = "password" - CreatedUserNameEnv = "TOOLS_TESTING_AUTH_USERNAME" - CreatedUserPasswordEnv = "TOOLS_TESTING_AUTH_PASSWORD" -) - -// Initialize a user admin, using the already-connected session passed in. -// Assumes that there are no existing users, otherwise will fail with a -// permissions issue. -func CreateUserAdmin(session *mgo.Session) error { - err := CreateUserWithRole(session, UserAdmin, UserAdminPassword, - mgo.RoleUserAdminAny, false) - return err -} - -// Create a user with the specified password and role, using the -// already-connected session passed in. If needsLogin is true, then the -// default user admin and password will be used to log in to the admin -// db before creating the user. -func CreateUserWithRole(session *mgo.Session, user, - password string, role mgo.Role, needsLogin bool) error { - - adminDB := session.DB("admin") - if needsLogin { - err := adminDB.Login( - UserAdmin, - UserAdminPassword, - ) - if err != nil { - return fmt.Errorf("error logging in: %v", err) - } - } - - err := adminDB.Run( - bson.D{ - {"createUser", user}, - {"pwd", password}, - {"roles", []bson.M{ - { - "role": role, - "db": "admin", - }, - }}, - }, - &bson.M{}, - ) - - if err != nil { - return fmt.Errorf("error adding user %v with role %v: %v", user, role, err) - } - - return nil -} - -func GetAuthOptions() options.Auth { - if testtype.HasTestType(testtype.AuthTestType) { - return options.Auth{ - Username: os.Getenv(CreatedUserNameEnv), - Password: os.Getenv(CreatedUserPasswordEnv), - Source: "admin", - } - } - - return options.Auth{} -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/testutil/kerberos.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/testutil/kerberos.go deleted file mode 100644 index d52128aef7b..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/testutil/kerberos.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package testutil - -import ( - "fmt" - "github.com/mongodb/mongo-tools/common/options" - "os" - "runtime" -) - -var ( - WinKerberosPwdEnv = "MONGODB_KERBEROS_PASSWORD" -) - -func GetKerberosOptions() (*options.ToolOptions, error) { - opts := &options.ToolOptions{ - Namespace: &options.Namespace{ - DB: "kerberos", - Collection: "test", - }, - SSL: &options.SSL{}, - Auth: &options.Auth{ - Username: "drivers@LDAPTEST.10GEN.CC", - Source: "$external", - Mechanism: "GSSAPI", - }, - Kerberos: &options.Kerberos{}, - Connection: &options.Connection{ - Host: "ldaptest.10gen.cc", - Port: "27017", - }, - } - - if runtime.GOOS == "windows" { - opts.Auth.Password = os.Getenv(WinKerberosPwdEnv) - if opts.Auth.Password == "" { - return nil, fmt.Errorf("Need to set %v environment variable to run "+ - "kerberos tests on windows", WinKerberosPwdEnv) - } - } - - return opts, nil -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/testutil/ssl_integration.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/testutil/ssl_integration.go deleted file mode 100644 index 8fc9293bd17..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/testutil/ssl_integration.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package testutil - -import ( - commonOpts "github.com/mongodb/mongo-tools/common/options" - "github.com/mongodb/mongo-tools/common/testtype" -) - -func GetSSLOptions() commonOpts.SSL { - if testtype.HasTestType(testtype.SSLTestType) { - return commonOpts.SSL{ - UseSSL: true, - SSLCAFile: "../common/db/openssl/testdata/ca.pem", - SSLPEMKeyFile: "../common/db/openssl/testdata/server.pem", - } - } - - return commonOpts.SSL{ - UseSSL: false, - } -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/testutil/testutil.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/testutil/testutil.go deleted file mode 100644 index 4efedd9e149..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/testutil/testutil.go +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -// Package testutil implements functions for filtering and configuring tests. -package testutil - -import ( - "os" - "strconv" - "strings" - - "github.com/mongodb/mongo-tools/common/db" - "github.com/mongodb/mongo-tools/common/options" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" -) - -// GetBareSession returns an mgo.Session from the environment or -// from a default host and port. -func GetBareSession() (*mgo.Session, error) { - sessionProvider, _, err := GetBareSessionProvider() - if err != nil { - return nil, err - } - session, err := sessionProvider.GetSession() - if err != nil { - return nil, err - } - return session, nil -} - -// GetBareSessionProvider returns a session provider from the environment or -// from a default host and port. -func GetBareSessionProvider() (*db.SessionProvider, *options.ToolOptions, error) { - var toolOptions *options.ToolOptions - - // get ToolOptions from URI or defaults - if uri := os.Getenv("TOOLS_TESTING_MONGOD"); uri != "" { - fakeArgs := []string{"--uri=" + uri} - toolOptions = options.New("mongodump", "", options.EnabledOptions{URI: true}) - toolOptions.URI.AddKnownURIParameters(options.KnownURIOptionsReadPreference) - _, err := toolOptions.ParseArgs(fakeArgs) - if err != nil { - panic("Could not parse MONGOD environment variable") - } - } else { - ssl := GetSSLOptions() - auth := GetAuthOptions() - connection := &options.Connection{ - Host: "localhost", - Port: db.DefaultTestPort, - } - toolOptions = &options.ToolOptions{ - SSL: &ssl, - Connection: connection, - Auth: &auth, - Verbosity: &options.Verbosity{}, - URI: &options.URI{}, - } - } - sessionProvider, err := db.NewSessionProvider(*toolOptions) - if err != nil { - return nil, nil, err - } - return sessionProvider, toolOptions, nil -} - -// GetFCV returns the featureCompatibilityVersion string for an mgo Session -// or the empty string if it can't be found. -func GetFCV(s *mgo.Session) string { - coll := s.DB("admin").C("system.version") - var result struct { - Version string - } - _ = coll.Find(bson.M{"_id": "featureCompatibilityVersion"}).One(&result) - return result.Version -} - -// CompareFCV compares two strings as dot-delimited tuples of integers -func CompareFCV(x, y string) (int, error) { - left, err := dottedStringToSlice(x) - if err != nil { - return 0, err - } - right, err := dottedStringToSlice(y) - if err != nil { - return 0, err - } - - // Ensure left is the shorter one, flip logic if necessary - inverter := 1 - if len(right) < len(left) { - inverter = -1 - left, right = right, left - } - - for i := range left { - switch { - case left[i] < right[i]: - return -1 * inverter, nil - case left[i] > right[i]: - return 1 * inverter, nil - } - } - - // compared equal to length of left. If right is longer, then left is less - // than right (-1) (modulo the inverter) - if len(left) < len(right) { - return -1 * inverter, nil - } - - return 0, nil -} - -func dottedStringToSlice(s string) ([]int, error) { - parts := make([]int, 0, 2) - for _, v := range strings.Split(s, ".") { - i, err := strconv.Atoi(v) - if err != nil { - return parts, err - } - parts = append(parts, i) - } - return parts, nil -} - -func IsReplicaSet(s *mgo.Session) bool { - db := s.DB("admin") - var result bson.M - cmd := bson.M{"isMaster": 1} - db.Run(cmd, &result) - _, ok := result["setName"] - return ok -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/text/grid.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/text/grid.go deleted file mode 100644 index c1b5cebed9b..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/text/grid.go +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -// Package text provides utilities for formatting text data. -package text - -import ( - "bufio" - "bytes" - "fmt" - "io" - "strings" -) - -type Cell struct { - contents string - feed bool -} - -type GridWriter struct { - ColumnPadding int - MinWidth int - Grid [][]Cell - CurrentRow int - colWidths []int -} - -func max(a, b int) int { - if a > b { - return a - } - return b -} - -// init() makes the initial row if this is the first time any data is being written. -// otherwise, no-op. -func (gw *GridWriter) init() { - if len(gw.Grid) <= gw.CurrentRow { - gw.Grid = append(gw.Grid, []Cell{}) - } -} - -// WriteCell writes the given string into the next cell in the current row. -func (gw *GridWriter) WriteCell(data string) { - gw.init() - gw.Grid[gw.CurrentRow] = append(gw.Grid[gw.CurrentRow], Cell{data, false}) -} - -// WriteCells writes multiple cells by calling WriteCell for each argument. -func (gw *GridWriter) WriteCells(data ...string) { - for _, s := range data { - gw.WriteCell(s) - } -} - -// Feed writes the given string into the current cell but allowing the cell contents -// to extend past the width of the current column, and ends the row. -func (gw *GridWriter) Feed(data string) { - gw.init() - gw.Grid[gw.CurrentRow] = append(gw.Grid[gw.CurrentRow], Cell{data, true}) - gw.EndRow() -} - -// EndRow terminates the row of cells and begins a new row in the grid. -func (gw *GridWriter) EndRow() { - gw.CurrentRow++ - if len(gw.Grid) <= gw.CurrentRow { - gw.Grid = append(gw.Grid, []Cell{}) - } -} - -// Reset discards any grid data and resets the current row. -func (gw *GridWriter) Reset() { - gw.CurrentRow = 0 - gw.Grid = [][]Cell{} -} - -// updateWidths sets the column widths in the Grid. For each column in the Grid, -// it updates the cached width if its value is less than the current width. -func (gw *GridWriter) updateWidths(colWidths []int) { - if gw.colWidths == nil { - gw.colWidths = make([]int, len(colWidths)) - copy(gw.colWidths, colWidths) - } - for i, cw := range colWidths { - if gw.colWidths[i] < cw { - gw.colWidths[i] = cw - } - } -} - -// calculateWidths returns an array containing the correct padded size for -// each column in the grid. -func (gw *GridWriter) calculateWidths() []int { - colWidths := []int{} - - // Loop over each column - for j := 0; ; j++ { - found := false - - // Examine all the rows at column 'j' - for i := range gw.Grid { - if len(gw.Grid[i]) <= j { - continue - } - found = true - - if len(colWidths) <= j { - colWidths = append(colWidths, 0) - } - - if gw.Grid[i][j].feed { - // we're at a row-terminating cell - skip over the rest of this row - continue - } - // Set the size for the row to be the largest - // of all the cells in the column - newMin := max(gw.MinWidth, len(gw.Grid[i][j].contents)) - if newMin > colWidths[j] { - colWidths[j] = newMin - } - } - // This column did not have any data in it at all, so we've hit the - // end of the grid - stop. - if !found { - break - } - } - return colWidths -} - -// Flush writes the fully-formatted grid to the given io.Writer. -func (gw *GridWriter) Flush(w io.Writer) { - colWidths := gw.calculateWidths() - - // invalidate all cached widths if new cells are added/removed - if len(gw.colWidths) != len(colWidths) { - gw.colWidths = make([]int, len(colWidths)) - copy(gw.colWidths, colWidths) - } else { - gw.updateWidths(colWidths) - } - - for i, row := range gw.Grid { - lastRow := i == (len(gw.Grid) - 1) - for j, cell := range row { - lastCol := (j == len(row)-1) - fmt.Fprintf(w, fmt.Sprintf("%%%vs", gw.colWidths[j]), cell.contents) - if gw.ColumnPadding > 0 && !lastCol { - fmt.Fprint(w, strings.Repeat(" ", gw.ColumnPadding)) - } - } - if !lastRow { - fmt.Fprint(w, "\n") - } - } -} - -// FlushRows writes the fully-formatted grid to the given io.Writer, but -// gives each row its own Write() call instead of using newlines. -func (gw *GridWriter) FlushRows(w io.Writer) { - gridBuff := &bytes.Buffer{} - gw.Flush(gridBuff) - lineScanner := bufio.NewScanner(gridBuff) - for lineScanner.Scan() { - w.Write(lineScanner.Bytes()) - } -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/text/grid_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/text/grid_test.go deleted file mode 100644 index d55000959e9..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/text/grid_test.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package text - -import ( - "bytes" - "fmt" - "github.com/mongodb/mongo-tools/common/testtype" - . "github.com/smartystreets/goconvey/convey" - "testing" -) - -func TestUpdateWidths(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - Convey("Using a grid writer, the cached column width", t, func() { - gw := GridWriter{} - defaultWidths := []int{1, 2, 3, 4} - Convey("should be updated when one or more new cell widths are greater", func() { - // the first time, the grid's widths are nil - So(gw.colWidths, ShouldBeNil) - gw.updateWidths(defaultWidths) - So(gw.colWidths, ShouldResemble, defaultWidths) - - // the grid's widths should not be updated if all the new cell widths are less than or equal - newWidths := []int{1, 2, 1, 2} - So(gw.colWidths, ShouldNotBeNil) - gw.updateWidths(newWidths) - So(gw.colWidths, ShouldResemble, defaultWidths) - So(gw.colWidths, ShouldNotResemble, newWidths) - - // the grid's widths should be updated if any of the new cell widths are greater - newWidths = []int{1, 2, 3, 5} - So(gw.colWidths, ShouldNotBeNil) - gw.updateWidths(newWidths) - So(gw.colWidths, ShouldResemble, newWidths) - So(gw.colWidths, ShouldNotResemble, defaultWidths) - }) - }) -} - -func writeData(gw *GridWriter) { - gw.Reset() - for i := 0; i < 3; i++ { - for j := 0; j < 3; j++ { - gw.WriteCell(fmt.Sprintf("(%v,%v)", i, j)) - } - gw.EndRow() - } -} - -func TestWriteGrid(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - Convey("Test grid writer with no min width", t, func() { - gw := GridWriter{} - writeData(&gw) - buf := bytes.Buffer{} - gw.Flush(&buf) - So(buf.String(), ShouldEqual, - "(0,0)(0,1)(0,2)\n(1,0)(1,1)(1,2)\n(2,0)(2,1)(2,2)\n") - - writeData(&gw) - gw.MinWidth = 7 - buf = bytes.Buffer{} - gw.Flush(&buf) - So(buf.String(), ShouldStartWith, - " (0,0) (0,1) (0,2)\n (1,0) (1,1)") - - writeData(&gw) - gw.colWidths = []int{} - gw.MinWidth = 0 - gw.ColumnPadding = 1 - buf = bytes.Buffer{} - gw.Flush(&buf) - So(buf.String(), ShouldStartWith, - "(0,0) (0,1) (0,2)\n(1,0) (1,1)") - - writeData(&gw) - buf = bytes.Buffer{} - gw.FlushRows(&buf) - So(buf.String(), ShouldStartWith, - "(0,0) (0,1) (0,2)(1,0) (1,1)") - }) - - Convey("Test grid writer width calculation", t, func() { - gw := GridWriter{} - gw.WriteCell("bbbb") - gw.WriteCell("aa") - gw.WriteCell("c") - gw.EndRow() - gw.WriteCell("bb") - gw.WriteCell("a") - gw.WriteCell("") - gw.EndRow() - So(gw.calculateWidths(), ShouldResemble, []int{4, 2, 1}) - - gw.WriteCell("bbbbbbb") - gw.WriteCell("a") - gw.WriteCell("cccc") - gw.EndRow() - So(gw.calculateWidths(), ShouldResemble, []int{7, 2, 4}) - - gw.WriteCell("bbbbbbb") - gw.WriteCell("a") - gw.WriteCell("cccc") - gw.WriteCell("ddddddddd") - gw.EndRow() - So(gw.calculateWidths(), ShouldResemble, []int{7, 2, 4, 9}) - }) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/text/units.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/text/units.go deleted file mode 100644 index e3fd32feca4..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/text/units.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package text - -import ( - "fmt" - "math" -) - -const ( - decimal = 1000 - binary = 1024 -) - -var ( - longByteUnits = []string{"B", "KB", "MB", "GB"} - shortByteUnits = []string{"B", "K", "M", "G"} - shortBitUnits = []string{"b", "k", "m", "g"} -) - -// FormatByteAmount takes an int64 representing a size in bytes and -// returns a formatted string of a minimum amount of significant figures. -// e.g. 12.4 GB, 0.0 B, 124.5 KB -func FormatByteAmount(size int64) string { - return formatUnitAmount(binary, size, 3, longByteUnits) -} - -// FormatMegabyteAmount is equivalent to FormatByteAmount but expects -// an amount of MB instead of bytes. -func FormatMegabyteAmount(size int64) string { - return formatUnitAmount(binary, size*1024*1024, 3, shortByteUnits) -} - -// FormatBits takes in a bit (not byte) count and returns a formatted string -// including units with three total digits (except if it is less than 1k) -// e.g. 12.0g, 0b, 124k -func FormatBits(size int64) string { - return formatUnitAmount(decimal, size, 3, shortBitUnits) -} - -// formatUnitAmount formats the size using the units and at least minDigits -// numbers, unless the number is already less than the base, where no decimal -// will be added -func formatUnitAmount(base, size int64, minDigits int, units []string) string { - result := float64(size) - divisor := float64(base) - var shifts int - // keep dividing by base and incrementing our unit until - // we hit the right unit or run out of unit strings - for ; result >= divisor && shifts < len(units)-1; shifts++ { - result /= divisor - } - result = round(result, minDigits) - - var precision int // Number of digits to show after the decimal - len := 1 + int(math.Log10(result)) // Number of pre-decimal digits in result - if shifts != 0 && len < minDigits { - // Add as many decimal digits as we can - precision = minDigits - len - } - format := fmt.Sprintf("%%.%df%%s", precision) - return fmt.Sprintf(format, result, units[shifts]) -} - -// round applies the gradeschool method to round to the nth place -func round(result float64, precision int) float64 { - divisor := float64(math.Pow(10.0, float64(precision-1))) - // round(x) == floor(x + 0.5) - return math.Floor(result*divisor+0.5) / divisor -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/text/units_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/text/units_test.go deleted file mode 100644 index 90011240cbc..00000000000 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/text/units_test.go +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2014-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package text - -import ( - "github.com/mongodb/mongo-tools/common/testtype" - "testing" - - . "github.com/smartystreets/goconvey/convey" -) - -func TestFormatByteCount(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - Convey("With some sample byte amounts", t, func() { - Convey("0 Bytes -> 0B", func() { - So(FormatByteAmount(0), ShouldEqual, "0B") - }) - Convey("1024 Bytes -> 1.00KB", func() { - So(FormatByteAmount(1024), ShouldEqual, "1.00KB") - }) - Convey("2500 Bytes -> 2.44KB", func() { - So(FormatByteAmount(2500), ShouldEqual, "2.44KB") - }) - Convey("2*1024*1024 Bytes -> 2.00MB", func() { - So(FormatByteAmount(2*1024*1024), ShouldEqual, "2.00MB") - }) - Convey("5*1024*1024*1024 Bytes -> 5.00GB", func() { - So(FormatByteAmount(5*1024*1024*1024), ShouldEqual, "5.00GB") - }) - Convey("5*1024*1024*1024*1024 Bytes -> 5120GB", func() { - So(FormatByteAmount(5*1024*1024*1024*1024), ShouldEqual, "5120GB") - }) - }) -} - -func TestOtherByteFormats(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - Convey("With some sample byte amounts", t, func() { - Convey("with '10'", func() { - Convey("FormatMegabyteAmount -> 10.0M", func() { - So(FormatMegabyteAmount(10), ShouldEqual, "10.0M") - }) - Convey("FormatByteAmount -> 10B", func() { - So(FormatByteAmount(10), ShouldEqual, "10B") - }) - Convey("FormatBitsWithLowPrecision -> 10b", func() { - So(FormatBits(10), ShouldEqual, "10b") - }) - }) - Convey("with '1024 * 2.5'", func() { - val := int64(2.5 * 1024) - Convey("FormatMegabyteAmount -> 2.50G", func() { - So(FormatMegabyteAmount(val), ShouldEqual, "2.50G") - }) - Convey("FormatByteAmount -> 2.50KB", func() { - So(FormatByteAmount(val), ShouldEqual, "2.50KB") - }) - Convey("FormatBits -> 2.56k", func() { - So(FormatBits(val), ShouldEqual, "2.56k") - }) - }) - }) -} - -func TestBitFormatPrecision(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - Convey("With values less than 1k", t, func() { - Convey("with '999'", func() { - Convey("FormatBits -> 999b", func() { - So(FormatBits(999), ShouldEqual, "999b") - }) - }) - Convey("with '99'", func() { - Convey("FormatBits -> 99b", func() { - So(FormatBits(99), ShouldEqual, "99b") - }) - }) - Convey("with '9'", func() { - Convey("FormatBits -> 9b", func() { - So(FormatBits(9), ShouldEqual, "9b") - }) - }) - }) - Convey("With values less than 1m", t, func() { - Convey("with '9999'", func() { - Convey("FormatBits -> 10.0k", func() { - So(FormatBits(9999), ShouldEqual, "10.0k") - }) - }) - Convey("with '9990'", func() { - Convey("FormatBits -> 9.99k", func() { - So(FormatBits(9990), ShouldEqual, "9.99k") - }) - }) - }) - Convey("With big numbers", t, func() { - Convey("with '999000000'", func() { - Convey("FormatBits -> 999m", func() { - So(FormatBits(999000000), ShouldEqual, "999m") - }) - }) - Convey("with '9990000000'", func() { - Convey("FormatBits -> 9.99g", func() { - So(FormatBits(9990000000), ShouldEqual, "9.99g") - }) - }) - }) -} diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/import.data b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/import.data index 515c18c0332..b9ade24c1d4 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/import.data +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/import.data @@ -1,5 +1,5 @@ { - "commit": "e4705b54b0699ec5cbc1bedab90f9d26a9fbb816", + "commit": "d91cc9fc08dd8f6dc5d71cdfcdefafd70f35149b", "github": "mongodb/mongo-tools.git", "vendor": "tools", "branch": "v4.1" diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/connstring/connstring.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/connstring/connstring.go index 2437ff32406..2437ff32406 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/connstring/connstring.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/connstring/connstring.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/failpoint/failpoint.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/failpoint/failpoint.go index e6cdd59b28f..e6cdd59b28f 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/failpoint/failpoint.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/failpoint/failpoint.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/failpoint/failpoint_disabled.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/failpoint/failpoint_disabled.go index 4b5cf9ab955..4b5cf9ab955 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/failpoint/failpoint_disabled.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/failpoint/failpoint_disabled.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/failpoint/failpoint_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/failpoint/failpoint_test.go index 713b0172717..fcd6bcaad3d 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/failpoint/failpoint_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/failpoint/failpoint_test.go @@ -11,7 +11,7 @@ package failpoint import ( "testing" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/failpoint/failpoints.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/failpoint/failpoints.go index 0258ca1d2a0..0258ca1d2a0 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/failpoint/failpoints.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/failpoint/failpoints.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/bench_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/bench_test.go index ba34a4015fc..7825983c25a 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/bench_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/bench_test.go @@ -18,7 +18,7 @@ package json import ( "bytes" "compress/gzip" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" "io/ioutil" "os" "testing" diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/bindata.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/bindata.go index c508ba904ed..c508ba904ed 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/bindata.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/bindata.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/bindata_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/bindata_test.go index cf38992217a..866376c3abd 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/bindata_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/bindata_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "testing" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/boolean.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/boolean.go index 2cae19de10d..2cae19de10d 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/boolean.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/boolean.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/boolean_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/boolean_test.go index 6e41b09918d..da487ab2ca7 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/boolean_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/boolean_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "testing" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/constructor.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/constructor.go index 5383662dcd6..5383662dcd6 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/constructor.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/constructor.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/consts.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/consts.go index 1992a911761..1992a911761 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/consts.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/consts.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/csv_format.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/csv_format.go index ab6c30cc853..ab6c30cc853 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/csv_format.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/csv_format.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/date.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/date.go index 925952b46d7..bbf2f522ccb 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/date.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/date.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/util" + "github.com/mongodb/mongo-tools/legacy/util" "reflect" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/date_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/date_test.go index 79d9a84994c..d7218d57399 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/date_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/date_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "testing" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/dbpointer.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/dbpointer.go index b9e3de89de4..b9e3de89de4 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/dbpointer.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/dbpointer.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/dbpointer_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/dbpointer_test.go index 130321ddf4e..efdeb82c0b4 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/dbpointer_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/dbpointer_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "gopkg.in/mgo.v2/bson" "testing" diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/dbref.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/dbref.go index 15fba1fbcb4..15fba1fbcb4 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/dbref.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/dbref.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/dbref_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/dbref_test.go index c27522f6ec0..f86f6832d6f 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/dbref_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/dbref_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "math" "testing" diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/decode.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/decode.go index 1e23aa69815..1e23aa69815 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/decode.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/decode.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/decode_d_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/decode_d_test.go index d08d9c2ef9f..cd359c0754d 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/decode_d_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/decode_d_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "gopkg.in/mgo.v2/bson" "testing" diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/decode_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/decode_test.go index 330792c0e3f..b4a5e919d2b 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/decode_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/decode_test.go @@ -13,7 +13,7 @@ import ( "bytes" "encoding" "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" "image" "reflect" "strings" diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/encode.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/encode.go index ed123925cc6..ed123925cc6 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/encode.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/encode.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/encode_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/encode_test.go index e821c4583bb..d154c182694 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/encode_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/encode_test.go @@ -16,7 +16,7 @@ import ( "testing" "unicode" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" ) type Optionals struct { diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/example_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/example_test.go index 9e1e2bfbf4e..9e1e2bfbf4e 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/example_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/example_test.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/float_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/float_test.go index 787a2b7e137..bfa5cf7d2f2 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/float_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/float_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "testing" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/fold.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/fold.go index bedb1d5d0cf..bedb1d5d0cf 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/fold.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/fold.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/fold_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/fold_test.go index 9024ee08c80..9eabd399785 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/fold_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/fold_test.go @@ -11,7 +11,7 @@ package json import ( "bytes" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" "strings" "testing" "unicode/utf8" diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/frac_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/frac_test.go index 1a7a7b72bc4..c7ff6249346 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/frac_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/frac_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "testing" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/helpers.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/helpers.go index b88191b4cff..b88191b4cff 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/helpers.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/helpers.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/hex.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/hex.go index 66dd5ee2695..66dd5ee2695 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/hex.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/hex.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/hex_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/hex_test.go index 7ea44bc351c..86fdaaa2ac4 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/hex_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/hex_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "testing" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/indent.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/indent.go index 70ea7ed8e23..70ea7ed8e23 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/indent.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/indent.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/infinity.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/infinity.go index 40ee607beae..40ee607beae 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/infinity.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/infinity.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/infinity_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/infinity_test.go index e01e7f621f9..84ee3ddf646 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/infinity_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/infinity_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "math" "testing" diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/iso_date.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/iso_date.go index 48ae58a887c..48ae58a887c 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/iso_date.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/iso_date.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/iso_date_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/iso_date_test.go index 95238dadfc6..056bcb2c44e 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/iso_date_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/iso_date_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "testing" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/json_format.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/json_format.go index 3bed8dd05d4..3bed8dd05d4 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/json_format.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/json_format.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/maxkey.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/maxkey.go index 538ff0af6f1..538ff0af6f1 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/maxkey.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/maxkey.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/maxkey_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/maxkey_test.go index 6914ba420d3..fb4d2753759 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/maxkey_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/maxkey_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "testing" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/minkey.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/minkey.go index bb513c036f6..bb513c036f6 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/minkey.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/minkey.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/minkey_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/minkey_test.go index 3fb469ea397..bb29bc218ea 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/minkey_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/minkey_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "testing" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/mongo_extjson.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/mongo_extjson.go index 355975bbdb9..355975bbdb9 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/mongo_extjson.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/mongo_extjson.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/nan.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/nan.go index bc17a33af4e..bc17a33af4e 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/nan.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/nan.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/nan_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/nan_test.go index 9cd4a920a11..9cc6bf61f88 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/nan_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/nan_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "math" "testing" diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/new.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/new.go index 96b48ac215b..96b48ac215b 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/new.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/new.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/new_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/new_test.go index 936327b1dc4..4ff7e2124af 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/new_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/new_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "testing" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/number.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/number.go index ae888143cb7..ae888143cb7 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/number.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/number.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/number_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/number_test.go index 3edadd93745..8c9ab8a29f7 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/number_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/number_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "testing" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/objectid.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/objectid.go index f7d7a017a6d..f7d7a017a6d 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/objectid.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/objectid.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/objectid_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/objectid_test.go index 1f20af30170..0525ad62a0f 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/objectid_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/objectid_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "testing" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/regexp.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/regexp.go index 21f6ddab52d..21f6ddab52d 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/regexp.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/regexp.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/regexp_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/regexp_test.go index 1eae1ddf389..8a5fc10fdab 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/regexp_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/regexp_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "testing" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/scanner.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/scanner.go index f98abb562bb..f98abb562bb 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/scanner.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/scanner.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/scanner_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/scanner_test.go index 51d842c1fdf..7e1a43f68bb 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/scanner_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/scanner_test.go @@ -11,7 +11,7 @@ package json import ( "bytes" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" "math" "math/rand" "reflect" diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/single_quoted.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/single_quoted.go index f4299f17440..f4299f17440 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/single_quoted.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/single_quoted.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/single_quoted_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/single_quoted_test.go index 596b0d0b08a..e31c482cc56 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/single_quoted_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/single_quoted_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "testing" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/stream.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/stream.go index dda4351eea2..dda4351eea2 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/stream.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/stream.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/stream_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/stream_test.go index a45ac3d2ddf..dbb4e291ae8 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/stream_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/stream_test.go @@ -11,7 +11,7 @@ package json import ( "bytes" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" "io/ioutil" "net" "reflect" diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/tagkey_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/tagkey_test.go index 67c2a852689..d414b34de30 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/tagkey_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/tagkey_test.go @@ -10,7 +10,7 @@ package json import ( - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" "testing" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/tags.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/tags.go index 763085f9fae..763085f9fae 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/tags.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/tags.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/tags_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/tags_test.go index bf2a782c4ab..be19cd70e8c 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/tags_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/tags_test.go @@ -10,7 +10,7 @@ package json import ( - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" "testing" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/testdata/code.json.gz b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/testdata/code.json.gz Binary files differindex 0e2895b53ac..0e2895b53ac 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/testdata/code.json.gz +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/testdata/code.json.gz diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/timestamp.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/timestamp.go index d7e9e1ba77b..d7e9e1ba77b 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/timestamp.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/timestamp.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/timestamp_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/timestamp_test.go index f522f923c70..b7535471b88 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/timestamp_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/timestamp_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "testing" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/undefined.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/undefined.go index 145516ecc42..145516ecc42 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/undefined.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/undefined.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/undefined_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/undefined_test.go index 29df73bc9ae..609f2670eba 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/undefined_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/undefined_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "testing" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/unquoted.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/unquoted.go index 15906343787..15906343787 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/unquoted.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/unquoted.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/unquoted_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/unquoted_test.go index 7a4a99a584b..f0dc39ea950 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/json/unquoted_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/json/unquoted_test.go @@ -8,7 +8,7 @@ package json import ( "fmt" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "testing" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/connector.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/connector.go index b1623243528..44d7e0deadf 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/connector.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/connector.go @@ -11,9 +11,9 @@ import ( "time" mgo "github.com/10gen/llmgo" - "github.com/mongodb/mongo-tools/common/lldb/kerberos" - "github.com/mongodb/mongo-tools/common/options" - "github.com/mongodb/mongo-tools/common/util" + "github.com/mongodb/mongo-tools/legacy/lldb/kerberos" + "github.com/mongodb/mongo-tools/legacy/options" + "github.com/mongodb/mongo-tools/legacy/util" ) // Interface type for connecting to the database. diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/db.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/db.go index f7f60c758f4..a92a1168bc8 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/db.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/db.go @@ -11,8 +11,8 @@ package lldb import ( mgo "github.com/10gen/llmgo" "github.com/10gen/llmgo/bson" - "github.com/mongodb/mongo-tools/common/options" - "github.com/mongodb/mongo-tools/common/password" + "github.com/mongodb/mongo-tools/legacy/options" + "github.com/mongodb/mongo-tools/legacy/password" "fmt" "io" diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/db_openssl.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/db_openssl.go index 4a313972060..73eff3b39c9 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/db_openssl.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/db_openssl.go @@ -9,8 +9,8 @@ package lldb import ( - "github.com/mongodb/mongo-tools/common/lldb/openssl" - "github.com/mongodb/mongo-tools/common/options" + "github.com/mongodb/mongo-tools/legacy/lldb/openssl" + "github.com/mongodb/mongo-tools/legacy/options" ) func init() { diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/db_tlsgo.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/db_tlsgo.go index e1cfcf18e17..eab88ff585d 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/db_tlsgo.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/db_tlsgo.go @@ -9,8 +9,8 @@ package lldb import ( - "github.com/mongodb/mongo-tools/common/lldb/tlsgo" - "github.com/mongodb/mongo-tools/common/options" + "github.com/mongodb/mongo-tools/legacy/lldb/tlsgo" + "github.com/mongodb/mongo-tools/legacy/options" ) func init() { diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/kerberos/gssapi.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/kerberos/gssapi.go index b0255c281ed..698d94f7da2 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/kerberos/gssapi.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/kerberos/gssapi.go @@ -12,7 +12,7 @@ package kerberos import ( mgo "github.com/10gen/llmgo" - "github.com/mongodb/mongo-tools/common/options" + "github.com/mongodb/mongo-tools/legacy/options" ) const authMechanism = "GSSAPI" diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/openssl/openssl.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/openssl/openssl.go index b07019a7956..3afe62bca9a 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/openssl/openssl.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/openssl/openssl.go @@ -16,10 +16,10 @@ import ( mgo "github.com/10gen/llmgo" "github.com/10gen/openssl" - "github.com/mongodb/mongo-tools/common/lldb/kerberos" - "github.com/mongodb/mongo-tools/common/log" - "github.com/mongodb/mongo-tools/common/options" - "github.com/mongodb/mongo-tools/common/util" + "github.com/mongodb/mongo-tools/legacy/lldb/kerberos" + "github.com/mongodb/mongo-tools/legacy/log" + "github.com/mongodb/mongo-tools/legacy/options" + "github.com/mongodb/mongo-tools/legacy/util" ) // For connecting to the database over ssl diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/openssl/openssl_fips.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/openssl/openssl_fips.go index 68bf0a92d3d..b706f1e7ff0 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/openssl/openssl_fips.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/openssl/openssl_fips.go @@ -12,7 +12,7 @@ import ( "fmt" "github.com/10gen/openssl" - "github.com/mongodb/mongo-tools/common/options" + "github.com/mongodb/mongo-tools/legacy/options" ) func init() { diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/config.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/tlsgo/config.go index 8d3971b537b..8d3971b537b 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/config.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/tlsgo/config.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/rootcerts.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/tlsgo/rootcerts.go index ee3ec3769f1..ee3ec3769f1 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/rootcerts.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/tlsgo/rootcerts.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/rootcerts_darwin.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/tlsgo/rootcerts_darwin.go index 72c7a9116ad..72c7a9116ad 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/tlsgo/rootcerts_darwin.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/tlsgo/rootcerts_darwin.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/tlsgo/tlsgo.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/tlsgo/tlsgo.go index 2b11e991ca1..bcf501aa69c 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/lldb/tlsgo/tlsgo.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/lldb/tlsgo/tlsgo.go @@ -15,10 +15,10 @@ import ( "time" mgo "github.com/10gen/llmgo" - "github.com/mongodb/mongo-tools/common/lldb/kerberos" - "github.com/mongodb/mongo-tools/common/log" - "github.com/mongodb/mongo-tools/common/options" - "github.com/mongodb/mongo-tools/common/util" + "github.com/mongodb/mongo-tools/legacy/lldb/kerberos" + "github.com/mongodb/mongo-tools/legacy/log" + "github.com/mongodb/mongo-tools/legacy/options" + "github.com/mongodb/mongo-tools/legacy/util" ) // TLSDBConnector makes a connection to the database with Go native TLS. diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/log/tool_logger.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/log/tool_logger.go index bde3d871348..bde3d871348 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/log/tool_logger.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/log/tool_logger.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/log/tool_logger_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/log/tool_logger_test.go index d258b225980..3c4a1e98abc 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/log/tool_logger_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/log/tool_logger_test.go @@ -13,7 +13,7 @@ import ( "testing" "time" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/options/options.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/options/options.go index 02e286a0a90..bbd11b4c035 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/options/options.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/options/options.go @@ -18,10 +18,10 @@ import ( "time" "github.com/jessevdk/go-flags" - "github.com/mongodb/mongo-tools/common/connstring" - "github.com/mongodb/mongo-tools/common/failpoint" - "github.com/mongodb/mongo-tools/common/log" - "github.com/mongodb/mongo-tools/common/util" + "github.com/mongodb/mongo-tools/legacy/connstring" + "github.com/mongodb/mongo-tools/legacy/failpoint" + "github.com/mongodb/mongo-tools/legacy/log" + "github.com/mongodb/mongo-tools/legacy/util" ) // Gitspec that the tool was built with. Needs to be set using -ldflags diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/options/options_fp.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/options/options_fp.go index f9a103a3dbb..f9a103a3dbb 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/options/options_fp.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/options/options_fp.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/options/options_fp_disabled.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/options/options_fp_disabled.go index fc73c63d34f..fc73c63d34f 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/options/options_fp_disabled.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/options/options_fp_disabled.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/options/options_gssapi.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/options/options_gssapi.go index 0446e7c01e1..0446e7c01e1 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/options/options_gssapi.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/options/options_gssapi.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/options/options_openssl.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/options/options_openssl.go index afb18ab8eb2..afb18ab8eb2 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/options/options_openssl.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/options/options_openssl.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/options/options_ssl.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/options/options_ssl.go index 4220c6d909e..4220c6d909e 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/options/options_ssl.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/options/options_ssl.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/options/options_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/options/options_test.go index cda13773d75..559f5672794 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/options/options_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/options/options_test.go @@ -7,10 +7,10 @@ package options import ( - "github.com/mongodb/mongo-tools/common/connstring" + "github.com/mongodb/mongo-tools/legacy/connstring" . "github.com/smartystreets/goconvey/convey" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" "runtime" "testing" "time" diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/password/pass_util.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/password/pass_util.go index 4d55a1453de..4d55a1453de 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/password/pass_util.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/password/pass_util.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/password/pass_util_solaris.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/password/pass_util_solaris.go index 58ad45c0b76..58ad45c0b76 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/password/pass_util_solaris.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/password/pass_util_solaris.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/password/password.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/password/password.go index bf8e9a0450b..5558d82982d 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/password/password.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/password/password.go @@ -10,7 +10,7 @@ package password import ( "fmt" - "github.com/mongodb/mongo-tools/common/log" + "github.com/mongodb/mongo-tools/legacy/log" "os" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/testtype/types.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/testtype/types.go index 0993f9b4cbc..0993f9b4cbc 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/testtype/types.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/testtype/types.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/bool.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/bool.go index fced8f28014..fced8f28014 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/bool.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/bool.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/bool_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/bool_test.go index 0068defee3a..6675fbcab04 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/bool_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/bool_test.go @@ -7,7 +7,7 @@ package util import ( - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "gopkg.in/mgo.v2/bson" "math" diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/exit_code.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/exit_code.go index d894ee1b428..d894ee1b428 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/exit_code.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/exit_code.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/file.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/file.go index ed7b6d758c0..ed7b6d758c0 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/file.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/file.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/format_date.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/format_date.go index 2b3a06b247e..2b3a06b247e 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/format_date.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/format_date.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/format_date_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/format_date_test.go index 73a4aca64de..f7fa6177b2a 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/format_date_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/format_date_test.go @@ -7,7 +7,7 @@ package util import ( - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" "testing" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/math.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/math.go index 1bd0ff57053..1bd0ff57053 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/math.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/math.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/math_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/math_test.go index 43349931237..ef46caa5f2d 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/math_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/math_test.go @@ -10,7 +10,7 @@ import ( "reflect" "testing" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/mongo.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/mongo.go index cf3168d2b68..cf3168d2b68 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/mongo.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/mongo.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/mongo_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/mongo_test.go index 0bff112d8e5..7130c1e733e 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/mongo_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/mongo_test.go @@ -9,7 +9,7 @@ package util import ( "testing" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" . "github.com/smartystreets/goconvey/convey" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/net.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/net.go index b8a2ccf8220..b8a2ccf8220 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/net.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/net.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/slice.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/slice.go index fdcb7f774aa..fdcb7f774aa 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/slice.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/slice.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/strings.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/strings.go index b64f8de18da..b64f8de18da 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/strings.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/strings.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/util.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/util.go index f33344bb9b9..f33344bb9b9 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/util/util.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/legacy/util/util.go diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoexport/mongoexport.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoexport/mongoexport.go index 3123f8b551f..6994fb1701e 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoexport/mongoexport.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoexport/mongoexport.go @@ -123,6 +123,8 @@ func New(opts Options) (*MongoExport, error) { } } + log.Logvf(log.Always, "connected to: %v", opts.URI.ConnectionString) + isMongos, err := provider.IsMongos() if err != nil { provider.Close() @@ -421,15 +423,6 @@ func (exp *MongoExport) exportInternal(out io.Writer) (int64, error) { } defer cursor.Close(nil) - connURL := exp.ToolOptions.Host - if connURL == "" { - connURL = util.DefaultHost - } - if exp.ToolOptions.Port != "" { - connURL = connURL + ":" + exp.ToolOptions.Port - } - log.Logvf(log.Always, "connected to: %v", connURL) - // Write headers err = exportOutput.WriteHeader() if err != nil { diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongofiles/mongofiles.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongofiles/mongofiles.go index df66aba6b86..adf002f683c 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongofiles/mongofiles.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongofiles/mongofiles.go @@ -407,14 +407,6 @@ func (mf *MongoFiles) handlePut() error { func (mf *MongoFiles) Run(displayHost bool) (output string, finalErr error) { var err error - connURL := mf.ToolOptions.Host - if connURL == "" { - connURL = util.DefaultHost - } - if mf.ToolOptions.Port != "" { - connURL = fmt.Sprintf("%s:%s", connURL, mf.ToolOptions.Port) - } - // check type of node we're connected to, and fall back to w=1 if standalone (for <= 2.4) nodeType, err := mf.SessionProvider.GetNodeType() if err != nil { @@ -440,7 +432,7 @@ func (mf *MongoFiles) Run(displayHost bool) (output string, finalErr error) { } if displayHost { - log.Logvf(log.Always, "connected to: %v", connURL) + log.Logvf(log.Always, "connected to: %v", mf.ToolOptions.URI.ConnectionString) } // first validate the namespaces we'll be using: <db>.<prefix>.files and <db>.<prefix>.chunks diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongofiles/options_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongofiles/options_test.go index 55d9e96f7c9..bfe1d7cc779 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongofiles/options_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongofiles/options_test.go @@ -9,7 +9,7 @@ package mongofiles import ( "testing" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools-common/testtype" . "github.com/smartystreets/goconvey/convey" "go.mongodb.org/mongo-driver/mongo/writeconcern" ) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoimport/mongoimport.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoimport/mongoimport.go index 0497f85a0f3..9747310be1f 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoimport/mongoimport.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoimport/mongoimport.go @@ -365,26 +365,6 @@ func (imp *MongoImport) ImportDocuments() (uint64, error) { return imp.importDocuments(inputReader) } -func (imp *MongoImport) getTargetConnection() (connURL string, err error) { - connURL = util.DefaultHost - if imp.ToolOptions.Host != "" { - connURL = imp.ToolOptions.Host - } else if imp.ToolOptions.URI.ConnectionString != "" { - connURL = strings.Join(imp.ToolOptions.URI.GetConnectionAddrs(), ",") - if connURL == "" { - return connURL, fmt.Errorf("error parsing connection string from URI") - } - if imp.ToolOptions.URI.ParsedConnString().ReplicaSet != "" { - connURL = imp.ToolOptions.URI.ParsedConnString().ReplicaSet + - "/" + connURL - } - } - if imp.ToolOptions.Port != "" { - connURL = connURL + ":" + imp.ToolOptions.Port - } - return connURL, nil -} - // importDocuments is a helper to ImportDocuments and does all the ingestion // work by taking data from the inputReader source and writing it to the // appropriate namespace @@ -394,12 +374,7 @@ func (imp *MongoImport) importDocuments(inputReader InputReader) (numImported ui return 0, err } - connURL, err := imp.getTargetConnection() - if err != nil { - return 0, err - } - - log.Logvf(log.Always, "connected to: %v", connURL) + log.Logvf(log.Always, "connected to: %v", imp.ToolOptions.URI.ConnectionString) log.Logvf(log.Info, "ns: %v.%v", imp.ToolOptions.Namespace.DB, diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoimport/mongoimport_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoimport/mongoimport_test.go index ef7b4f992c6..cd05adca1fa 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoimport/mongoimport_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoimport/mongoimport_test.go @@ -830,86 +830,6 @@ func TestImportDocuments(t *testing.T) { }) } -func TestGetTargetDestination(t *testing.T) { - testtype.SkipUnlessTestType(t, testtype.UnitTestType) - Convey("Given mongoimport instance, on calling getTargetConnection", t, func() { - imp := NewMockMongoImport() - imp.ToolOptions = options.New("", "", "", "", options.EnabledOptions{URI: true}) - Convey("no error should be thrown with undefined --host and --port", func() { - imp.ToolOptions.Host = "" - imp.ToolOptions.Port = "" - url, err := imp.getTargetConnection() - So(err, ShouldBeNil) - So(url, ShouldEqual, "localhost") - }) - Convey("no error should be thrown with undefined --host and defined --port", func() { - imp.ToolOptions.Host = "" - imp.ToolOptions.Port = "30000" - url, err := imp.getTargetConnection() - So(err, ShouldBeNil) - So(url, ShouldEqual, "localhost:30000") - }) - Convey("no error should be thrown with defined --host and undefined --port", func() { - imp.ToolOptions.Host = "foobar" - imp.ToolOptions.Port = "" - url, err := imp.getTargetConnection() - So(err, ShouldBeNil) - So(url, ShouldEqual, "foobar") - }) - Convey("no error should be thrown with defined standalone --host containing port", func() { - imp.ToolOptions.Host = "foobar:30000" - imp.ToolOptions.Port = "" - url, err := imp.getTargetConnection() - So(err, ShouldBeNil) - So(url, ShouldEqual, "foobar:30000") - }) - Convey("no error should be thrown with defined replica set --host", func() { - imp.ToolOptions.Host = "replica/foo1:30000,foo2:30000,foo3:30000" - imp.ToolOptions.Port = "" - url, err := imp.getTargetConnection() - So(err, ShouldBeNil) - So(url, ShouldEqual, "replica/foo1:30000,foo2:30000,foo3:30000") - }) - Convey("no error should be thrown with defined standalone --uri", func() { - _, err := imp.ToolOptions.ParseArgs([]string{"--uri=mongodb://foo1:30000"}) - So(err, ShouldBeNil) - url, err := imp.getTargetConnection() - So(err, ShouldBeNil) - So(url, ShouldEqual, "foo1:30000") - }) - Convey("no error should be thrown with defined replica set --uri", func() { - _, err := imp.ToolOptions.ParseArgs([]string{"--uri=mongodb://foo1,foo2,foo3/dbx?replicaSet=replica"}) - So(err, ShouldBeNil) - url, err := imp.getTargetConnection() - So(err, ShouldBeNil) - So(url, ShouldEqual, "replica/foo1,foo2,foo3") - }) - Convey("no error should be thrown with defined replica set --uri containing credentials", func() { - _, err := imp.ToolOptions.ParseArgs([]string{"--uri=mongodb://username:password@foo1,foo2,foo3/dbx?replicaSet=replica"}) - So(err, ShouldBeNil) - url, err := imp.getTargetConnection() - So(err, ShouldBeNil) - So(url, ShouldEqual, "replica/foo1,foo2,foo3") - }) - Convey("no error should be thrown with defined SRV --uri", func() { - // NOTE: this requires SRV and TXT records in DNS as specified here: - // https://github.com/mongodb/specifications/tree/master/source/initial-dns-seedlist-discovery - _, err := imp.ToolOptions.ParseArgs([]string{"--uri=mongodb+srv://test5.test.build.10gen.cc/dbx?replicaSet=rs0"}) - url, err := imp.getTargetConnection() - So(err, ShouldBeNil) - So(url, ShouldEqual, "rs0/localhost.test.build.10gen.cc:27017") - }) - Convey("an error should be thrown with defined SRV --uri but no record found", func() { - // NOTE: this requires SRV and TXT records in DNS as specified here: - // https://github.com/mongodb/specifications/tree/master/source/initial-dns-seedlist-discovery - _, err := imp.ToolOptions.ParseArgs([]string{"--uri=mongodb+srv://test4.test.build.10gen.cc/dbx?replicaSet=rs0"}) - url, err := imp.getTargetConnection() - So(err, ShouldNotBeNil) - So(url, ShouldEqual, "") - }) - }) -} - // Regression test for TOOLS-1694 to prevent issue from TOOLS-1115 func TestHiddenOptionsDefaults(t *testing.T) { testtype.SkipUnlessTestType(t, testtype.UnitTestType) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoimport/options.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoimport/options.go index a3e634b884a..0d571a865a7 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoimport/options.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoimport/options.go @@ -10,8 +10,8 @@ import ( "fmt" "github.com/mongodb/mongo-tools-common/db" + "github.com/mongodb/mongo-tools-common/log" "github.com/mongodb/mongo-tools-common/options" - "github.com/mongodb/mongo-tools/common/log" ) var Usage = `<options> <file> diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/auth_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/auth_test.go index dd044ce990a..781bf1ab18a 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/auth_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/auth_test.go @@ -11,7 +11,7 @@ import ( mgo "github.com/10gen/llmgo" "github.com/10gen/llmgo/bson" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" ) // TestCommandsAgainstAuthedDBWhenAuthed tests some basic commands against a diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/cursors_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/cursors_test.go index ff943b29431..7ab26ff2062 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/cursors_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/cursors_test.go @@ -7,7 +7,7 @@ package mongoreplay import ( - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" "sync" "testing" "time" diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/execute_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/execute_test.go index 1346876aa82..2a56f679c77 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/execute_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/execute_test.go @@ -10,7 +10,7 @@ import ( "testing" mgo "github.com/10gen/llmgo" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" ) func TestCompleteReply(t *testing.T) { diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/filter_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/filter_test.go index 3e89f0792ff..ec96d7ec2b2 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/filter_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/filter_test.go @@ -13,7 +13,7 @@ import ( "time" "github.com/10gen/llmgo/bson" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" ) func TestRemoveDriverOpsFromFile(t *testing.T) { diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/main/mongoreplay.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/main/mongoreplay.go index 82a1b355c51..9e416ce18f2 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/main/mongoreplay.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/main/mongoreplay.go @@ -8,7 +8,7 @@ package main import ( "github.com/jessevdk/go-flags" - "github.com/mongodb/mongo-tools/common/options" + "github.com/mongodb/mongo-tools/legacy/options" "github.com/mongodb/mongo-tools/mongoreplay" "fmt" diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/mongoreplay_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/mongoreplay_test.go index d5fa5a4ef71..a249bb4e42e 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/mongoreplay_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/mongoreplay_test.go @@ -15,7 +15,7 @@ import ( mgo "github.com/10gen/llmgo" "github.com/10gen/llmgo/bson" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" ) type testDoc struct { diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/pcap_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/pcap_test.go index dc7a5ef82e2..063960b6cc9 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/pcap_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/pcap_test.go @@ -14,7 +14,7 @@ import ( mgo "github.com/10gen/llmgo" "github.com/10gen/llmgo/bson" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" ) type verifyFunc func(*testing.T, *mgo.Session, *BufferedStatRecorder, *preprocessCursorManager) diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/play.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/play.go index 237127d38ed..8f94bd0b0e2 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/play.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/play.go @@ -12,8 +12,8 @@ import ( "strings" "time" - "github.com/mongodb/mongo-tools/common/lldb" - "github.com/mongodb/mongo-tools/common/options" + "github.com/mongodb/mongo-tools/legacy/lldb" + "github.com/mongodb/mongo-tools/legacy/options" ) // PlayCommand stores settings for the mongoreplay 'play' subcommand diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/play_livedb_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/play_livedb_test.go index ba3f367c0ac..381c53f34cb 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/play_livedb_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/play_livedb_test.go @@ -15,7 +15,7 @@ import ( mgo "github.com/10gen/llmgo" "github.com/10gen/llmgo/bson" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" ) const ( diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/play_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/play_test.go index 5fd3e3a04e6..a31fd7253d3 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/play_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/play_test.go @@ -12,7 +12,7 @@ import ( "testing" "time" - "github.com/mongodb/mongo-tools/common/testtype" + "github.com/mongodb/mongo-tools/legacy/testtype" ) func TestRepeatGeneration(t *testing.T) { diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/playbackfile.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/playbackfile.go index a5224d27f56..fd97da5d2c5 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/playbackfile.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/playbackfile.go @@ -15,7 +15,7 @@ import ( "time" "github.com/10gen/llmgo/bson" - "github.com/mongodb/mongo-tools/common/util" + "github.com/mongodb/mongo-tools/legacy/util" ) const PlaybackFileVersion = 1 diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/query_op.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/query_op.go index 46755eeb103..bcbcb8f290c 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/query_op.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/query_op.go @@ -14,7 +14,7 @@ import ( mgo "github.com/10gen/llmgo" "github.com/10gen/llmgo/bson" - "github.com/mongodb/mongo-tools/common/json" + "github.com/mongodb/mongo-tools/legacy/json" ) // QueryOp is used to query the database for documents in a collection. diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/util.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/util.go index 9851e5479bd..7b0d0a627e0 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/util.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongoreplay/util.go @@ -18,7 +18,7 @@ import ( mgo "github.com/10gen/llmgo" bson "github.com/10gen/llmgo/bson" - "github.com/mongodb/mongo-tools/common/json" + "github.com/mongodb/mongo-tools/legacy/json" ) var ( diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongorestore/mongorestore_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongorestore/mongorestore_test.go index 1ba417f7ee5..15726308f74 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongorestore/mongorestore_test.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongorestore/mongorestore_test.go @@ -13,9 +13,9 @@ import ( "github.com/mongodb/mongo-tools-common/db" "github.com/mongodb/mongo-tools-common/log" + "github.com/mongodb/mongo-tools-common/options" "github.com/mongodb/mongo-tools-common/testtype" "github.com/mongodb/mongo-tools-common/testutil" - "github.com/mongodb/mongo-tools/common/options" . "github.com/smartystreets/goconvey/convey" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo/writeconcern" diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongotop/mongotop.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongotop/mongotop.go index c45d08d80bf..49e68fb731f 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongotop/mongotop.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongotop/mongotop.go @@ -103,15 +103,6 @@ func (mt *MongoTop) runServerStatusDiff() (outDiff FormattableDiff, err error) { // Run executes the mongotop program. func (mt *MongoTop) Run() error { - - connURL := mt.Options.Host - if connURL == "" { - connURL = "127.0.0.1" - } - if mt.Options.Port != "" { - connURL = connURL + ":" + mt.Options.Port - } - hasData := false numPrinted := 0 @@ -135,7 +126,7 @@ func (mt *MongoTop) Run() error { // if this is the first time and the connection is successful, print // the connection message if !hasData && !mt.OutputOptions.Json { - log.Logvf(log.Always, "connected to: %v\n", connURL) + log.Logvf(log.Always, "connected to: %v\n", mt.Options.URI.ConnectionString) } hasData = true diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/runTests.sh b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/runTests.sh index 84b45ffc93b..5c3ef93d8e5 100755 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/runTests.sh +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/runTests.sh @@ -24,7 +24,7 @@ ec=0 # Run all tests depending on what flags are set in the environment # TODO: mongotop needs a test # Note: Does not test mongoreplay -for i in common/db common/archive common/bsonutil common/db/tlsgo common/failpoint common/intents common/json common/log common/options common/progress common/text common/util mongostat mongofiles mongoexport mongoimport mongorestore mongodump mongotop bsondump ; do +for i in legacy/failpoint legacy/json legacy/log legacy/options legacy/util mongostat mongofiles mongoexport mongoimport mongorestore mongodump mongotop bsondump ; do echo "Testing ${i}..." COMMON_SUBPKG=$(basename $i) COVERAGE_ARGS=""; diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/openssl/testdata/ca.pem b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/testdata/certs/ca.pem index d1a5689cf0f..d1a5689cf0f 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/openssl/testdata/ca.pem +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/testdata/certs/ca.pem diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/openssl/testdata/server.pem b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/testdata/certs/server.pem index df2b49163d6..df2b49163d6 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common/db/openssl/testdata/server.pem +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/testdata/certs/server.pem diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/vendor/github.com/mongodb/mongo-tools-common/options/options.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/vendor/github.com/mongodb/mongo-tools-common/options/options.go index 6215dd95630..c4199202e59 100644 --- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/vendor/github.com/mongodb/mongo-tools-common/options/options.go +++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/vendor/github.com/mongodb/mongo-tools-common/options/options.go @@ -359,6 +359,14 @@ func (auth *Auth) ShouldAskForPassword() bool { !(auth.Mechanism == "MONGODB-X509" || auth.Mechanism == "GSSAPI") } +func NewURI(unparsed string) (*URI, error) { + cs, err := connstring.Parse(unparsed) + if err != nil { + return nil, fmt.Errorf("error parsing URI from %v: %v", unparsed, err) + } + return &URI{ConnectionString: cs.String(), connString: cs}, nil +} + func (uri *URI) GetConnectionAddrs() []string { return uri.connString.Hosts } @@ -463,7 +471,11 @@ func (opts *ToolOptions) NormalizeHostPortURI() error { } else { // If URI not provided, get replica set name and generate connection string _, opts.ReplicaSetName = util.SplitHostArg(opts.Host) - opts.URI = &URI{ConnectionString: util.BuildURI(opts.Host, opts.Port)} + uri, err := NewURI(util.BuildURI(opts.Host, opts.Port)) + if err != nil { + return err + } + opts.URI = uri } // connect directly, unless a replica set name is explicitly specified |