summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorThomas Schubert <thomas.schubert@mongodb.com>2017-07-31 23:03:23 -0400
committerThomas Schubert <thomas.schubert@mongodb.com>2017-07-31 23:03:23 -0400
commitcf38c1b8a0a8dca4a11737581beafef4fe120bcd (patch)
tree548b58b5bbf6672b7423cf30f646c40cfea4f240
parent4463700eb94a5ee95c706d69cbb7bf44e67a50d2 (diff)
downloadmongo-cf38c1b8a0a8dca4a11737581beafef4fe120bcd.tar.gz
Import tools: 4f093ae71cdb4c6a6e9de7cd1dc67ea4405f0013 from branch v3.4r3.4.7-rc0r3.4.7
ref: 29b8883c56..4f093ae71c for: 3.4.7 TOOLS-1109 failes to build on arm64 (syscall.Dup2 not supported) TOOLS-1542 dump and export shouldn't count views before running TOOLS-1563 windows tests are failing after `use mongodb 3.4 "current" tests` TOOLS-1577 update the readme with information about mongoreplay TOOLS-1713 Move mongoreplay evergreen config .evergreen.yml into common.yml TOOLS-1741 mongoimport --uri throws errors when passed Atlas Connection String URI TOOLS-1743 legacy24 and legacy26 dumprestore tests failing on master
-rw-r--r--src/mongo/gotools/.evergreen.yml367
-rw-r--r--src/mongo/gotools/Godeps2
-rw-r--r--src/mongo/gotools/README.md2
-rw-r--r--src/mongo/gotools/common.yml15
-rw-r--r--src/mongo/gotools/common/db/namespaces.go61
-rw-r--r--src/mongo/gotools/common/db/namespaces_test.go52
-rw-r--r--src/mongo/gotools/common/db/write_concern.go8
-rw-r--r--src/mongo/gotools/common/db/write_concern_test.go7
-rw-r--r--src/mongo/gotools/common/intents/intent.go1
-rw-r--r--src/mongo/gotools/common/intents/intent_prioritizer.go22
-rw-r--r--src/mongo/gotools/common/intents/intent_prioritizer_test.go35
-rw-r--r--src/mongo/gotools/import.data2
-rw-r--r--src/mongo/gotools/mongodump/mongodump.go10
-rw-r--r--src/mongo/gotools/mongodump/mongodump_test.go2
-rw-r--r--src/mongo/gotools/mongodump/prepare.go191
-rw-r--r--src/mongo/gotools/mongoexport/mongoexport.go24
-rw-r--r--src/mongo/gotools/mongofiles/options.go3
-rw-r--r--src/mongo/gotools/mongofiles/options_test.go74
-rw-r--r--src/mongo/gotools/mongoimport/options.go4
-rw-r--r--src/mongo/gotools/mongoimport/options_test.go74
-rw-r--r--src/mongo/gotools/mongorestore/options.go9
-rw-r--r--src/mongo/gotools/mongorestore/options_test.go74
-rw-r--r--src/mongo/gotools/vendor/src/github.com/spacemonkeygo/spacelog/capture_linux.go33
-rw-r--r--src/mongo/gotools/vendor/src/github.com/spacemonkeygo/spacelog/capture_other.go1
24 files changed, 534 insertions, 539 deletions
diff --git a/src/mongo/gotools/.evergreen.yml b/src/mongo/gotools/.evergreen.yml
deleted file mode 100644
index 43535b8124e..00000000000
--- a/src/mongo/gotools/.evergreen.yml
+++ /dev/null
@@ -1,367 +0,0 @@
-functions:
- "fetch source" :
- - command: git.get_project
- params:
- directory: src
- - command: git.apply_patch
- params:
- directory: src
-
- "build tool" :
- command: shell.exec
- params:
- working_dir: src
- script: |
- echo "Building ${tool}..."
- . ./set_gopath.sh
- go build ${args} -o bin/${tool} cmd/${tool}/main.go
-
- "upload archive" :
- command: s3.put
- params:
- aws_key: ${aws_key}
- aws_secret: ${aws_secret}
- local_file: src/${filename}
- remote_file: mongotape/archive/${build_id}/${filename}
- bucket: mciuploads
- permissions: public-read
- content_type: application/gzip
- display_name: ${filename}
-
- "create timeseries" :
- command: shell.exec
- params:
- working_dir: src
- script: |
- git clone git@github.com:10gen/support-tools
- sudo pip install argparse python-dateutil pytz
- python support-tools/timeseries/timeseries.py /data/mongotape/diagnostic.data --html timeseries.html
-
- "upload timeseries" :
- command: s3.put
- params:
- aws_key: ${aws_key}
- aws_secret: ${aws_secret}
- local_file: src/timeseries.html
- remote_file: mongotape/timeseries/${build_id}/timeseries.html
- bucket: mciuploads
- permissions: public-read
- content_type: text/html
- display_name: timeseries.html
-
- "upload tool" :
- command: s3.put
- params:
- aws_key: ${aws_key}
- aws_secret: ${aws_secret}
- local_file: src/bin/${tool}
- remote_file: mongotape/binaries/${build_id}/${tool}
- bucket: mciuploads
- permissions: public-read
- content_type: application/octet-stream
- display_name: ${tool} binary
-
- "fetch tool" :
- command: s3.get
- params:
- bucket: mciuploads
- aws_key: ${aws_key}
- aws_secret: ${aws_secret}
- local_file: src/${tool}
- remote_file: mongotape/binaries/${build_id}/${tool}
-
- "fetch pcap" :
- command: s3.get
- params:
- bucket: boxes.10gen.com
- aws_key: ${aws_key}
- aws_secret: ${aws_secret}
- local_file: src/testPcap/${pcapFname}
- remote_file: build/mongotape/${pcapFname}
-
- "fetch ftdc" :
- - command: shell.exec
- type: test
- params:
- working_dir: src
- script: |
- . ./set_gopath.sh
- go get github.com/10gen/ftdc-utils/cmd/ftdc
-
- "start mongod" :
- - command: shell.exec
- params:
- background: true
- script: |
- set -e
- set -o verbose
- cd mongodb
- echo "starting mongodb"
- mkdir -p ./mongotape_test
- ./mongod --dbpath ./mongotape_test --port=${mongod_port} ${additional_args} &
- - command: shell.exec
- params:
- script: |
- set -e
- set -o verbose
- cd mongodb
- ./mongo --nodb --eval 'assert.soon(function(x){try{var d = new Mongo("localhost:${mongod_port}"); return true}catch(e){return false}}, "timed out connecting")'
-
- "fetch mongodb" :
- - command: shell.exec
- params:
- script: |
- rm -rf mongodb
- mkdir mongodb
- cd mongodb
- curl ${mongo_url} -o mongodb.tgz
- ${decompress} mongodb.tgz
- chmod +x ./mongodb-*/bin/*
- mv ./mongodb-*/bin/* .
- rm -rf db_files
- rm -rf db_logs
- mkdir db_files
- mkdir db_logs
-
- "create auth_user" :
- - command: shell.exec
- params:
- script: |
- set -e
- set -o verbose
- cd mongodb
- ./mongo --port ${mongod_port} admin --eval "db.createUser({user:\"authorizedUser\", pwd: \"authorizedPwd\", roles:[\"readWriteAnyDatabase\", \"clusterManager\"]});"
-
- "create sharded_cluster" :
- - command: shell.exec
- params:
- background: true
- script: |
- set -e
- set -o verbose
- cd mongodb
- echo "starting mongodb"
- mkdir -p /data/db/
- ./mongo --nodb --eval 'var d = new ShardingTest({shards:3, mongos:[{port:${mongod_port}}]}); while(true){sleep(1000)}'
- - command: shell.exec
- params:
- script: |
- set -e
- set -o verbose
- cd mongodb
- ./mongo --nodb --eval 'var d; assert.soon(function(x){try{d = new Mongo("localhost:${mongod_port}"); return true} catch(e){return false}}, "timed out connection");d.setLogLevel(5, "write");'
-
- "create repl_set" :
- - command: shell.exec
- params:
- background: true
- script: |
- set -e
- set -o verbose
- cd mongodb
- echo "starting mongodb"
- mkdir -p /data/db/
- ./mongo --nodb --eval 'var repl = new ReplSetTest({nodes:3, startPort:${mongod_port}});repl.startSet();repl.initiate();repl.awaitSecondaryNodes();while(true){sleep(1000);}'
- - command: shell.exec
- params:
- script: |
- set -e
- set -o verbose
- cd mongodb
- ./mongo --nodb --eval 'assert.soon(function(x){try{var d = new Mongo("localhost:${mongod_port}"); return true} catch(e){return false}}, "timed out connection")'
-
- "run go_test" :
- - command: shell.exec
- type: test
- params:
- working_dir: src
- script: |
- . ./set_gopath.sh
- ${environment_vars} go test ${additional_args} -v > ${filename}.suite
-
-pre:
- - command: shell.track
-
-post:
- - command: shell.exec
- - command: gotest.parse_files
- params:
- files: ["src/*.suite"]
- - command: shell.cleanup
-
-tasks:
-- name: replay-dist
- commands:
- - func: "fetch source"
- - func: "build tool"
- vars:
- tool: mongotape
- - func: "upload tool"
- vars:
- tool: mongotape
- - func: "build tool"
-
-- name: replay-sanity_check
- depends_on:
- - name: replay-dist
- commands:
- - func: "fetch source"
- - func: "fetch tool"
- vars:
- tool: mongotape
- - func: "fetch mongodb"
- - func: "start mongod"
- - command: shell.exec
- params:
- working_dir: src
- script: |
- set -e
- chmod +x mongotape
- PATH=$PATH:`pwd`:`pwd`/../mongodb
- echo "Running sanity check"
- ./sanity_check.sh -p ${mongod_port}
-
-- name: replay-go_test
- depends_on:
- - name: replay-dist
- commands:
- - func: "fetch source"
- - func: "fetch tool"
- vars:
- tool: mongotape
- - func: "fetch pcap"
- vars:
- pcapFname: getmore_multi_channel.pcap
- - func: "fetch pcap"
- vars:
- pcapFname: getmore_single_channel.pcap
- - func: "fetch mongodb"
- - func: "start mongod"
- vars:
- mongod_port: 20000
- - func: "run go_test"
- vars:
- filename: playtest
-
-- name: replay-sharded_test
- depends_on:
- - name: replay-dist
- commands:
- - func: "fetch source"
- - func: "fetch tool"
- vars:
- tool: mongotape
- - func: "fetch pcap"
- vars:
- pcapFname: getmore_multi_channel.pcap
- - func: "fetch pcap"
- vars:
- pcapFname: getmore_single_channel.pcap
- - func: "fetch mongodb"
- - func: "create sharded_cluster"
- vars:
- mongod_port: 20010
- - func: "run go_test"
- vars:
- filename: sharded
- environment_vars: DB_PORT=20010
- additional_args: --run "LiveDB"
-
-- name: replay-auth_test
- depends_on:
- - name: replay-dist
- commands:
- - func: "fetch source"
- - func: "fetch tool"
- vars:
- tool: mongotape
- - func: "fetch pcap"
- vars:
- pcapFname: getmore_multi_channel.pcap
- - func: "fetch pcap"
- vars:
- pcapFname: getmore_single_channel.pcap
- - func: "fetch mongodb"
- - func: "start mongod"
- vars:
- mongod_port: 20000
- additional_args: --auth
- - func: "create auth_user"
- vars:
- mongod_port: 20000
- - func: "run go_test"
- vars:
- environment_vars: AUTH=1
- filename: authtest
- additional_args: --run "(LiveDB)|(Authed)"
-
-- name: replay-repl_test
- depends_on:
- - name: replay-dist
- commands:
- - func: "fetch source"
- - func: "fetch tool"
- vars:
- tool: mongotape
- - func: "fetch pcap"
- vars:
- pcapFname: getmore_multi_channel.pcap
- - func: "fetch pcap"
- vars:
- pcapFname: getmore_single_channel.pcap
- - func: "fetch mongodb"
- - func: "create repl_set"
- vars:
- mongod_port: 20000
- - func: "run go_test"
- vars:
- filename: repltest
- additional_args: --run "LiveDB"
-
-- name: replay-replay_test
- depends_on:
- - name: replay-dist
- commands:
- - func: "fetch source"
- - func: "fetch tool"
- vars:
- tool: mongotape
- - func: "fetch mongodb"
- - func: "fetch ftdc"
- - command: shell.exec
- params:
- working_dir: src
- script: |
- set -e
- . ./set_gopath.sh
- PATH=$PATH:`pwd`:`pwd`/vendor/bin:`pwd`/../mongodb
- chmod +x mongotape
- echo "Running replay test"
- ./replay_test.sh --verbose --explicit --keep
- - command: shell.exec
- params:
- working_dir: src
- script: |
- tar czf replay.tar.gz tmp.*
- - func: "upload archive"
- vars:
- filename: replay.tar.gz
- - func: "create timeseries"
- - func: "upload timeseries"
-
-buildvariants:
-- name: ubuntu
- display_name: Ubuntu
- expansions:
- mongod_port: 27017
- mongo_url: http://fastdl.mongodb.org/linux/mongodb-linux-x86_64-3.2.4.tgz
- run_on:
- - ubuntu1404-test
- tasks:
- - name: replay-dist
- - name: replay-sanity_check
- - name: replay-go_test
- - name: replay-auth_test
- - name: replay-sharded_test
- - name: replay-repl_test
- - name: replay-replay_test
diff --git a/src/mongo/gotools/Godeps b/src/mongo/gotools/Godeps
index d3b757426d3..8a0702bafda 100644
--- a/src/mongo/gotools/Godeps
+++ b/src/mongo/gotools/Godeps
@@ -7,7 +7,7 @@ github.com/smartystreets/goconvey bf58a9a1291224109919756b4dcc469c670cc7e4
github.com/jessevdk/go-flags 97448c91aac742cbca3d020b3e769013a420a06f
github.com/3rf/mongo-lint 3550fdcf1f43b89aaeabaa4559eaae6dc4407e42
github.com/spacemonkeygo/openssl 2869e8ca1a6eb35fb727f41611fd52b55cd0f49c github.com/10gen/openssl
-github.com/spacemonkeygo/spacelog ae95ccc1eb0c8ce2496c43177430efd61930f7e4
+github.com/spacemonkeygo/spacelog f936fb050dc6b5fe4a96b485a6f069e8bdc59aeb
github.com/howeyc/gopass 44476384cd4721b68705e72f19e95d1a3a504370
github.com/nsf/termbox-go 0723e7c3d0a317dea811f0fbe4d6edd81908c971
github.com/mattn/go-runewidth d6bea18f789704b5f83375793155289da36a3c7f
diff --git a/src/mongo/gotools/README.md b/src/mongo/gotools/README.md
index c3e6f670c82..589e6041db9 100644
--- a/src/mongo/gotools/README.md
+++ b/src/mongo/gotools/README.md
@@ -9,6 +9,8 @@ MongoDB Tools
- **mongofiles** - _Read, write, delete, or update files in [GridFS](http://docs.mongodb.org/manual/core/gridfs/)_
- **mongooplog** - _Replay oplog entries between MongoDB servers_
- **mongotop** - _Monitor read/write activity on a mongo server_
+ - **mongoreplay** - _Capture, observe, and replay traffic for MongoDB_
+
Report any bugs, improvements, or new feature requests at https://jira.mongodb.org/browse/TOOLS
diff --git a/src/mongo/gotools/common.yml b/src/mongo/gotools/common.yml
index 79b68563d0d..2fb7cb57dc8 100644
--- a/src/mongo/gotools/common.yml
+++ b/src/mongo/gotools/common.yml
@@ -1785,21 +1785,6 @@ buildvariants:
#######################################
# Windows Buildvariants #
#######################################
-- name: windows-32
- display_name: Windows 32-bit
- run_on:
- - windows-32
- expansions:
- <<: *mongod_win32_startup_args
- <<: *mongo_default_startup_args
- mongo_target: "windows_i686"
- mongo_arch: "i386"
- extension: .exe
- preproc_gpm: "perl -pi -e 's/\\r\\n/\\n/g' "
- excludes: requires_large_ram
- integration_test_args: "integration"
- tasks: *windows_32_tasks
-
- name: windows-64
display_name: Windows 64-bit
run_on:
diff --git a/src/mongo/gotools/common/db/namespaces.go b/src/mongo/gotools/common/db/namespaces.go
index 908687b1c56..8530b16aa5e 100644
--- a/src/mongo/gotools/common/db/namespaces.go
+++ b/src/mongo/gotools/common/db/namespaces.go
@@ -2,13 +2,23 @@ package db
import (
"fmt"
- "github.com/mongodb/mongo-tools/common/bsonutil"
+ "strings"
+
"github.com/mongodb/mongo-tools/common/log"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
- "strings"
)
+type CollectionInfo struct {
+ Name string `bson:"name"`
+ Type string `bson:"type"`
+ Options *bson.D `bson:"options"`
+}
+
+func (ci *CollectionInfo) IsView() bool {
+ return ci.Type == "view"
+}
+
// IsNoCmd reeturns true if err indicates a query command is not supported,
// otherwise, returns false.
func IsNoCmd(err error) bool {
@@ -124,7 +134,7 @@ func getCollectionsPre28(database *mgo.Database, name string) (*mgo.Iter, error)
return iter, nil
}
-func GetCollectionOptions(coll *mgo.Collection) (*bson.D, error) {
+func GetCollectionInfo(coll *mgo.Collection) (*CollectionInfo, error) {
iter, useFullName, err := GetCollections(coll.Database, coll.Name)
if err != nil {
return nil, err
@@ -134,33 +144,32 @@ func GetCollectionOptions(coll *mgo.Collection) (*bson.D, error) {
if useFullName {
comparisonName = coll.FullName
}
- collInfo := &bson.D{}
+
+ collInfo := &CollectionInfo{}
for iter.Next(collInfo) {
- name, err := bsonutil.FindValueByKey("name", collInfo)
- if err != nil {
- collInfo = nil
- continue
- }
- if nameStr, ok := name.(string); ok {
- if nameStr == comparisonName {
- // we've found the collection we're looking for
- break
+ if collInfo.Name == comparisonName {
+ if useFullName {
+ collName, err := StripDBFromNamespace(collInfo.Name, coll.Database.Name)
+ if err != nil {
+ return nil, err
+ }
+ collInfo.Name = collName
}
- } else {
- collInfo = nil
- continue
+ break
}
}
+ if err := iter.Err(); err != nil {
+ return nil, err
+ }
+ return collInfo, nil
+}
- if collInfo != nil {
- optsInterface, _ := bsonutil.FindValueByKey("options", collInfo)
- if optsInterface != nil {
- optsD, ok := optsInterface.(bson.D)
- if !ok {
- return nil, fmt.Errorf("Cannot unmarshal collection options for collection %v.%v", coll.Database, coll.Name)
- }
- return &optsD, nil
- }
+func StripDBFromNamespace(namespace string, dbName string) (string, error) {
+ namespacePrefix := dbName + "."
+ // if the collection info came from querying system.indexes (2.6 or earlier) then the
+ // "name" we get includes the db name as well, so we must remove it
+ if strings.HasPrefix(namespace, namespacePrefix) {
+ return namespace[len(namespacePrefix):], nil
}
- return nil, iter.Err()
+ return "", fmt.Errorf("namespace '%v' format is invalid - expected to start with '%v'", namespace, namespacePrefix)
}
diff --git a/src/mongo/gotools/common/db/namespaces_test.go b/src/mongo/gotools/common/db/namespaces_test.go
new file mode 100644
index 00000000000..0e390a2f43c
--- /dev/null
+++ b/src/mongo/gotools/common/db/namespaces_test.go
@@ -0,0 +1,52 @@
+package db
+
+import (
+ "fmt"
+ "testing"
+
+ . "github.com/smartystreets/goconvey/convey"
+)
+
+type stripDBFromNamespaceTestCase struct {
+ inputNamespace string
+ inputDBName string
+
+ outputNamespace string
+ outputError error
+}
+
+func TestStripDBFromNamespace(t *testing.T) {
+ Convey("When testing StripDBFromNamespace with cases", t, func() {
+ testCases := []stripDBFromNamespaceTestCase{
+ {
+ inputNamespace: "database.col",
+ inputDBName: "database",
+
+ outputNamespace: "col",
+ outputError: nil,
+ },
+ {
+ inputNamespace: "database2.col",
+ inputDBName: "database",
+
+ outputNamespace: "",
+ outputError: fmt.Errorf("namespace 'database2.col' format is invalid - expected to start with 'database.'"),
+ },
+ {
+ inputNamespace: "database.col",
+ inputDBName: "notAPrefix",
+
+ outputNamespace: "",
+ outputError: fmt.Errorf("namespace 'database.col' format is invalid - expected to start with 'notAPrefix.'"),
+ },
+ }
+ Convey("cases should match expected", func() {
+ for _, tc := range testCases {
+ resultNamespace, resultError := StripDBFromNamespace(tc.inputNamespace, tc.inputDBName)
+ So(resultError, ShouldResemble, tc.outputError)
+ So(resultNamespace, ShouldEqual, tc.outputNamespace)
+ }
+ })
+ })
+
+}
diff --git a/src/mongo/gotools/common/db/write_concern.go b/src/mongo/gotools/common/db/write_concern.go
index cf7a8e98d56..8759fb59ad1 100644
--- a/src/mongo/gotools/common/db/write_concern.go
+++ b/src/mongo/gotools/common/db/write_concern.go
@@ -127,11 +127,17 @@ func BuildWriteConcern(writeConcern string, nodeType NodeType, cs *connstring.Co
}
if cs != nil {
+ if cs.W == "" {
+ cs.W = "majority"
+ }
sessionSafety, err = constructSafetyFromConnString(cs)
if err != nil {
return nil, err
}
- } else if writeConcern != "" {
+ } else {
+ if writeConcern == "" {
+ writeConcern = "majority"
+ }
sessionSafety, err = constructWCObject(writeConcern)
if err != nil {
return nil, err
diff --git a/src/mongo/gotools/common/db/write_concern_test.go b/src/mongo/gotools/common/db/write_concern_test.go
index bdc6ef69018..31a5baf1bfb 100644
--- a/src/mongo/gotools/common/db/write_concern_test.go
+++ b/src/mongo/gotools/common/db/write_concern_test.go
@@ -54,6 +54,13 @@ func TestBuildWriteConcern(t *testing.T) {
So(err, ShouldBeNil)
So(writeConcern.J, ShouldBeTrue)
})
+ // Regression test for TOOLS-1741
+ Convey("When passing an empty writeConcern and empty URI"+
+ "then write concern should default to being majority", func() {
+ writeConcern, err := BuildWriteConcern("", ReplSet, nil)
+ So(err, ShouldBeNil)
+ So(writeConcern.WMode, ShouldEqual, "majority")
+ })
})
Convey("and given a connection string", func() {
Convey("with a w value of 0, without j set, a nil write concern should be returned", func() {
diff --git a/src/mongo/gotools/common/intents/intent.go b/src/mongo/gotools/common/intents/intent.go
index 8f317a3716f..db7eb78a772 100644
--- a/src/mongo/gotools/common/intents/intent.go
+++ b/src/mongo/gotools/common/intents/intent.go
@@ -285,6 +285,7 @@ func (manager *Manager) putNormalIntentWithNamespace(ns string, intent *Intent)
// Put inserts an intent into the manager with the same source namespace as
// its destinations.
func (manager *Manager) Put(intent *Intent) {
+ log.Logvf(log.DebugLow, "enqueued collection '%v'", intent.Namespace())
manager.PutWithNamespace(intent.Namespace(), intent)
}
diff --git a/src/mongo/gotools/common/intents/intent_prioritizer.go b/src/mongo/gotools/common/intents/intent_prioritizer.go
index 290a7c83d1e..bea15bb1ab9 100644
--- a/src/mongo/gotools/common/intents/intent_prioritizer.go
+++ b/src/mongo/gotools/common/intents/intent_prioritizer.go
@@ -59,8 +59,8 @@ func (legacy *legacyPrioritizer) Finish(*Intent) {
//===== Longest Task First =====
// longestTaskFirstPrioritizer returns intents in the order of largest -> smallest,
-// which is better at minimizing total runtime in parallel environments than
-// other simple orderings.
+// with views at the front of the list, which is better at minimizing total
+// runtime in parallel environments than other simple orderings.
type longestTaskFirstPrioritizer struct {
sync.Mutex
queue []*Intent
@@ -68,7 +68,7 @@ type longestTaskFirstPrioritizer struct {
// NewLongestTaskFirstPrioritizer returns an initialized LTP prioritizer
func NewLongestTaskFirstPrioritizer(intents []*Intent) *longestTaskFirstPrioritizer {
- sort.Sort(BySize(intents))
+ sort.Sort(BySizeAndView(intents))
return &longestTaskFirstPrioritizer{
queue: intents,
}
@@ -92,6 +92,22 @@ func (ltf *longestTaskFirstPrioritizer) Finish(*Intent) {
return
}
+// BySizeAndView attaches the methods for sort.Interface for sorting intents
+// from largest to smallest size, taking into account if it's a view or not.
+type BySizeAndView []*Intent
+
+func (s BySizeAndView) Len() int { return len(s) }
+func (s BySizeAndView) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s BySizeAndView) Less(i, j int) bool {
+ if s[i].IsView() && !s[j].IsView() {
+ return true
+ }
+ if !s[i].IsView() && s[j].IsView() {
+ return false
+ }
+ return s[i].Size > s[j].Size
+}
+
// For sorting intents from largest to smallest size
type BySize []*Intent
diff --git a/src/mongo/gotools/common/intents/intent_prioritizer_test.go b/src/mongo/gotools/common/intents/intent_prioritizer_test.go
index 2abd79e5641..2e153d3f9ea 100644
--- a/src/mongo/gotools/common/intents/intent_prioritizer_test.go
+++ b/src/mongo/gotools/common/intents/intent_prioritizer_test.go
@@ -2,9 +2,11 @@ package intents
import (
"container/heap"
+ "testing"
+
"github.com/mongodb/mongo-tools/common/testutil"
. "github.com/smartystreets/goconvey/convey"
- "testing"
+ "gopkg.in/mgo.v2/bson"
)
func TestLegacyPrioritizer(t *testing.T) {
@@ -109,6 +111,37 @@ func TestDBCounterCollectionSorting(t *testing.T) {
})
}
+func TestBySizeAndView(t *testing.T) {
+ var prioritizer IntentPrioritizer
+
+ testutil.VerifyTestType(t, testutil.UnitTestType)
+
+ Convey("With a prioritizer initialized with on a set of intents", t, func() {
+ intents := []*Intent{
+ &Intent{C: "non-view2", Size: 32},
+ &Intent{C: "view", Size: 0,
+ Options: &bson.D{{"viewOn", true}},
+ },
+ &Intent{C: "non-view1", Size: 1024},
+ &Intent{C: "non-view3", Size: 2},
+ &Intent{C: "view", Size: 0,
+ Options: &bson.D{{"viewOn", true}},
+ },
+ }
+ prioritizer = NewLongestTaskFirstPrioritizer(intents)
+ Convey("getting the sorted intents should produce views first, followed by largest to smallest", func() {
+
+ So(prioritizer.Get().C, ShouldEqual, "view")
+ So(prioritizer.Get().C, ShouldEqual, "view")
+ So(prioritizer.Get().C, ShouldEqual, "non-view1")
+ So(prioritizer.Get().C, ShouldEqual, "non-view2")
+ So(prioritizer.Get().C, ShouldEqual, "non-view3")
+ })
+
+ })
+
+}
+
func TestSimulatedMultiDBJob(t *testing.T) {
var prioritizer IntentPrioritizer
diff --git a/src/mongo/gotools/import.data b/src/mongo/gotools/import.data
index 4c780b2ed0b..f0eee069180 100644
--- a/src/mongo/gotools/import.data
+++ b/src/mongo/gotools/import.data
@@ -1,5 +1,5 @@
{
- "commit": "29b8883c560319b016f8bd4927807fa36f1a682f",
+ "commit": "4f093ae71cdb4c6a6e9de7cd1dc67ea4405f0013",
"github": "mongodb/mongo-tools.git",
"vendor": "tools",
"branch": "v3.4"
diff --git a/src/mongo/gotools/mongodump/mongodump.go b/src/mongo/gotools/mongodump/mongodump.go
index 079ec3c737a..30cb0b2a2d3 100644
--- a/src/mongo/gotools/mongodump/mongodump.go
+++ b/src/mongo/gotools/mongodump/mongodump.go
@@ -51,8 +51,8 @@ type MongoDump struct {
// as well as the signal handler, and allows them to notify
// the intent dumpers that they should shutdown
shutdownIntentsNotifier *notifier
- // the value of stdout gets initizlied to os.Stdout if it's unset
- stdout io.Writer
+ // the value of OutputWriter gets initizlied to os.Stdout if it's unset
+ OutputWriter io.Writer
readPrefMode mgo.Mode
readPrefTags []bson.D
}
@@ -115,8 +115,8 @@ func (dump *MongoDump) Init() error {
if err != nil {
return fmt.Errorf("bad option: %v", err)
}
- if dump.stdout == nil {
- dump.stdout = os.Stdout
+ if dump.OutputWriter == nil {
+ dump.OutputWriter = os.Stdout
}
dump.sessionProvider, err = db.NewSessionProvider(*dump.ToolOptions)
if err != nil {
@@ -782,7 +782,7 @@ func (*nopCloseWriter) Close() error {
func (dump *MongoDump) getArchiveOut() (out io.WriteCloser, err error) {
if dump.OutputOptions.Archive == "-" {
- out = &nopCloseWriter{dump.stdout}
+ out = &nopCloseWriter{dump.OutputWriter}
} else {
targetStat, err := os.Stat(dump.OutputOptions.Archive)
if err == nil && targetStat.IsDir() {
diff --git a/src/mongo/gotools/mongodump/mongodump_test.go b/src/mongo/gotools/mongodump/mongodump_test.go
index 92a209ca5f8..a856a3d6a56 100644
--- a/src/mongo/gotools/mongodump/mongodump_test.go
+++ b/src/mongo/gotools/mongodump/mongodump_test.go
@@ -434,7 +434,7 @@ func TestMongoDumpBSON(t *testing.T) {
Convey("it dumps to standard output", func() {
md.OutputOptions.Out = "-"
stdoutBuf := &bytes.Buffer{}
- md.stdout = stdoutBuf
+ md.OutputWriter = stdoutBuf
err = md.Dump()
So(err, ShouldBeNil)
var count int
diff --git a/src/mongo/gotools/mongodump/prepare.go b/src/mongo/gotools/mongodump/prepare.go
index 6c22748a810..b7f562b5799 100644
--- a/src/mongo/gotools/mongodump/prepare.go
+++ b/src/mongo/gotools/mongodump/prepare.go
@@ -3,15 +3,15 @@ package mongodump
import (
"bytes"
"fmt"
- "github.com/mongodb/mongo-tools/common/archive"
- "github.com/mongodb/mongo-tools/common/db"
- "github.com/mongodb/mongo-tools/common/intents"
- "github.com/mongodb/mongo-tools/common/log"
- "gopkg.in/mgo.v2/bson"
"io"
"os"
"path/filepath"
"strings"
+
+ "github.com/mongodb/mongo-tools/common/archive"
+ "github.com/mongodb/mongo-tools/common/db"
+ "github.com/mongodb/mongo-tools/common/intents"
+ "github.com/mongodb/mongo-tools/common/log"
)
type NilPos struct{}
@@ -20,16 +20,6 @@ func (NilPos) Pos() int64 {
return -1
}
-type collectionInfo struct {
- Name string `bson:"name"`
- Type string `bson:"type"`
- Options *bson.D `bson:"options"`
-}
-
-func (ci *collectionInfo) IsView() bool {
- return ci.Type == "view"
-}
-
// writeFlusher wraps an io.Writer and adds a Flush function.
type writeFlusher interface {
Flush() error
@@ -178,55 +168,6 @@ func checkStringForPathSeparator(s string, c *rune) bool {
return false
}
-// NewIntent creates a bare intent without populating the options.
-func (dump *MongoDump) NewIntent(dbName, colName string) (*intents.Intent, error) {
- intent := &intents.Intent{
- DB: dbName,
- C: colName,
- }
- if dump.OutputOptions.Out == "-" {
- intent.BSONFile = &stdoutFile{Writer: dump.stdout}
- } else {
- if dump.OutputOptions.Archive != "" {
- intent.BSONFile = &archive.MuxIn{Intent: intent, Mux: dump.archive.Mux}
- } else {
- var c rune
- if checkStringForPathSeparator(colName, &c) || checkStringForPathSeparator(dbName, &c) {
- return nil, fmt.Errorf(`"%v.%v" contains a path separator '%c' `+
- `and can't be dumped to the filesystem`, dbName, colName, c)
- }
- path := nameGz(dump.OutputOptions.Gzip, dump.outputPath(dbName, colName)+".bson")
- intent.BSONFile = &realBSONFile{path: path, intent: intent}
- }
- if !intent.IsSystemIndexes() {
- if dump.OutputOptions.Archive != "" {
- intent.MetadataFile = &archive.MetadataFile{
- Intent: intent,
- Buffer: &bytes.Buffer{},
- }
- } else {
- path := nameGz(dump.OutputOptions.Gzip, dump.outputPath(dbName, colName+".metadata.json"))
- intent.MetadataFile = &realMetadataFile{path: path, intent: intent}
- }
- }
- }
-
- // get a document count for scheduling purposes
- session, err := dump.sessionProvider.GetSession()
- if err != nil {
- return nil, err
- }
- defer session.Close()
-
- count, err := session.DB(dbName).C(colName).Count()
- if err != nil {
- return nil, fmt.Errorf("error counting %v: %v", intent.Namespace(), err)
- }
- intent.Size = int64(count)
-
- return intent, nil
-}
-
// CreateOplogIntents creates an intents.Intent for the oplog and adds it to the manager
func (dump *MongoDump) CreateOplogIntents() error {
err := dump.determineOplogCollectionName()
@@ -289,57 +230,94 @@ func (dump *MongoDump) CreateCollectionIntent(dbName, colName string) error {
return nil
}
- intent, err := dump.NewIntent(dbName, colName)
- if err != nil {
- return err
- }
-
session, err := dump.sessionProvider.GetSession()
if err != nil {
return err
}
defer session.Close()
- intent.Options, err = db.GetCollectionOptions(session.DB(dbName).C(colName))
+ collOptions, err := db.GetCollectionInfo(session.DB(dbName).C(colName))
if err != nil {
return fmt.Errorf("error getting collection options: %v", err)
}
- dump.manager.Put(intent)
+ intent, err := dump.NewIntentFromOptions(dbName, collOptions)
+ if err != nil {
+ return err
+ }
- log.Logvf(log.DebugLow, "enqueued collection '%v'", intent.Namespace())
+ dump.manager.Put(intent)
return nil
}
-func (dump *MongoDump) createIntentFromOptions(dbName string, ci *collectionInfo) error {
- if dump.shouldSkipCollection(ci.Name) {
- log.Logvf(log.DebugLow, "skipping dump of %v.%v, it is excluded", dbName, ci.Name)
- return nil
+func (dump *MongoDump) NewIntentFromOptions(dbName string, ci *db.CollectionInfo) (*intents.Intent, error) {
+ intent := &intents.Intent{
+ DB: dbName,
+ C: ci.Name,
+ Options: ci.Options,
}
- if dump.OutputOptions.ViewsAsCollections && !ci.IsView() {
- log.Logvf(log.DebugLow, "skipping dump of %v.%v because it is not a view", dbName, ci.Name)
- return nil
+ // Setup output location
+ if dump.OutputOptions.Out == "-" { // regular standard output
+ intent.BSONFile = &stdoutFile{Writer: dump.OutputWriter}
+ } else {
+ // Set the BSONFile path.
+ if dump.OutputOptions.Archive != "" {
+ // if archive mode, then the output should be written using an output
+ // muxer.
+ intent.BSONFile = &archive.MuxIn{Intent: intent, Mux: dump.archive.Mux}
+ } else if dump.OutputOptions.ViewsAsCollections || !ci.IsView() {
+ // otherwise, if it's either not a view or we're treating views as collections
+ // then create a standard filesystem path for this collection.
+ var c rune
+ if checkStringForPathSeparator(ci.Name, &c) || checkStringForPathSeparator(dbName, &c) {
+ return nil, fmt.Errorf(`"%v.%v" contains a path separator '%c' `+
+ `and can't be dumped to the filesystem`, dbName, ci.Name, c)
+ }
+ path := nameGz(dump.OutputOptions.Gzip, dump.outputPath(dbName, ci.Name)+".bson")
+ intent.BSONFile = &realBSONFile{path: path, intent: intent}
+ } else {
+ // otherwise, it's a view and the options specify not dumping a view
+ // so don't dump it.
+ log.Logvf(log.DebugLow, "not dumping data for %v.%v because it is a view", dbName, ci.Name)
+ }
+ //Set the MetadataFile path.
+ if dump.OutputOptions.ViewsAsCollections && ci.IsView() {
+ log.Logvf(log.DebugLow, "not dumping metadata for %v.%v because it is a view", dbName, ci.Name)
+ } else {
+ if !intent.IsSystemIndexes() {
+ if dump.OutputOptions.Archive != "" {
+ intent.MetadataFile = &archive.MetadataFile{
+ Intent: intent,
+ Buffer: &bytes.Buffer{},
+ }
+ } else {
+ path := nameGz(dump.OutputOptions.Gzip, dump.outputPath(dbName, ci.Name+".metadata.json"))
+ intent.MetadataFile = &realMetadataFile{path: path, intent: intent}
+ }
+ }
+ }
+ }
+
+ // get a document count for scheduling purposes.
+ // skips this if it is a view, as it may be incredibly slow if the
+ // view is based on a slow query.
+
+ if ci.IsView() {
+ return intent, nil
}
- intent, err := dump.NewIntent(dbName, ci.Name)
+ session, err := dump.sessionProvider.GetSession()
if err != nil {
- return err
+ return nil, err
}
- if dump.OutputOptions.ViewsAsCollections {
- log.Logvf(log.DebugLow, "not dumping metadata for %v.%v because it is a view", dbName, ci.Name)
- intent.MetadataFile = nil
- } else if ci.IsView() {
- log.Logvf(log.DebugLow, "not dumping data for %v.%v because it is a view", dbName, ci.Name)
- // only write a bson file if using archive
- if dump.OutputOptions.Archive == "" {
- intent.BSONFile = nil
- }
+ defer session.Close()
+ count, err := session.DB(dbName).C(ci.Name).Count()
+ if err != nil {
+ return nil, fmt.Errorf("error counting %v: %v", intent.Namespace(), err)
}
- intent.Options = ci.Options
- dump.manager.Put(intent)
- log.Logvf(log.DebugLow, "enqueued collection '%v'", intent.Namespace())
- return nil
+ intent.Size = int64(count)
+ return intent, nil
}
// CreateIntentsForDatabase iterates through collections in a db
@@ -358,7 +336,7 @@ func (dump *MongoDump) CreateIntentsForDatabase(dbName string) error {
return fmt.Errorf("error getting collections for database `%v`: %v", dbName, err)
}
- collInfo := &collectionInfo{}
+ collInfo := &db.CollectionInfo{}
for colsIter.Next(collInfo) {
// ignore <db>.system.* except for admin
if dbName != "admin" && strings.HasPrefix(collInfo.Name, "system.") {
@@ -370,19 +348,26 @@ func (dump *MongoDump) CreateIntentsForDatabase(dbName string) error {
continue
}
if fullName {
- namespacePrefix := dbName + "."
- // if the collection info came from querying system.indexes (2.6 or earlier) then the
- // "name" we get includes the db name as well, so we must remove it
- if strings.HasPrefix(collInfo.Name, namespacePrefix) {
- collInfo.Name = collInfo.Name[len(namespacePrefix):]
- } else {
- return fmt.Errorf("namespace '%v' format is invalid - expected to start with '%v'", collInfo.Name, namespacePrefix)
+ collName, err := db.StripDBFromNamespace(collInfo.Name, dbName)
+ if err != nil {
+ return err
}
+ collInfo.Name = collName
+ }
+ if dump.shouldSkipCollection(collInfo.Name) {
+ log.Logvf(log.DebugLow, "skipping dump of %v.%v, it is excluded", dbName, collInfo.Name)
+ continue
+ }
+
+ if dump.OutputOptions.ViewsAsCollections && !collInfo.IsView() {
+ log.Logvf(log.DebugLow, "skipping dump of %v.%v because it is not a view", dbName, collInfo.Name)
+ continue
}
- err := dump.createIntentFromOptions(dbName, collInfo)
+ intent, err := dump.NewIntentFromOptions(dbName, collInfo)
if err != nil {
return err
}
+ dump.manager.Put(intent)
}
return colsIter.Err()
}
diff --git a/src/mongo/gotools/mongoexport/mongoexport.go b/src/mongo/gotools/mongoexport/mongoexport.go
index 4feab4894d2..1508800790b 100644
--- a/src/mongo/gotools/mongoexport/mongoexport.go
+++ b/src/mongo/gotools/mongoexport/mongoexport.go
@@ -174,6 +174,7 @@ func makeFieldSelector(fields string) bson.M {
// It always returns Limit if there is a limit, assuming that in general
// limits will less then the total possible.
// If there is a query and no limit then it returns 0, because it's too expensive to count the query.
+// If the collection is a view then it returns 0, because it is too expensive to count the view.
// Otherwise it returns the count minus the skip
func (exp *MongoExport) getCount() (c int, err error) {
session, err := exp.SessionProvider.GetSession()
@@ -187,7 +188,17 @@ func (exp *MongoExport) getCount() (c int, err error) {
if exp.InputOpts != nil && exp.InputOpts.Query != "" {
return 0, nil
}
- q := session.DB(exp.ToolOptions.Namespace.DB).C(exp.ToolOptions.Namespace.Collection).Find(nil)
+ mgoCollection := session.DB(exp.ToolOptions.Namespace.DB).C(exp.ToolOptions.Namespace.Collection)
+
+ collInfo, err := db.GetCollectionInfo(mgoCollection)
+ if err != nil {
+ return 0, err
+ }
+ if collInfo.IsView() {
+ return 0, nil
+ }
+
+ q := mgoCollection.Find(nil)
c, err = q.Count()
if err != nil {
return 0, err
@@ -240,22 +251,15 @@ func (exp *MongoExport) getCursor() (*mgo.Iter, *mgo.Session, error) {
collection := session.DB(exp.ToolOptions.Namespace.DB).C(exp.ToolOptions.Namespace.Collection)
// figure out if we're exporting a view
- isView := false
- opts, err := db.GetCollectionOptions(collection)
+ collInfo, err := db.GetCollectionInfo(collection)
if err != nil {
return nil, nil, err
}
- if opts != nil {
- viewOn, _ := bsonutil.FindValueByKey("viewOn", opts)
- if viewOn != nil {
- isView = true
- }
- }
flags := 0
// don't snapshot if we've been asked not to,
// or if we cannot because we are querying, sorting, or if the collection is a view
- if !exp.InputOpts.ForceTableScan && len(query) == 0 && exp.InputOpts != nil && exp.InputOpts.Sort == "" && !isView {
+ if !exp.InputOpts.ForceTableScan && len(query) == 0 && exp.InputOpts != nil && exp.InputOpts.Sort == "" && !collInfo.IsView() {
flags = flags | db.Snapshot
}
diff --git a/src/mongo/gotools/mongofiles/options.go b/src/mongo/gotools/mongofiles/options.go
index 2df4774ebbc..a5b3271d7e9 100644
--- a/src/mongo/gotools/mongofiles/options.go
+++ b/src/mongo/gotools/mongofiles/options.go
@@ -34,7 +34,8 @@ type StorageOptions struct {
// Specifies the write concern for each write operation that mongofiles writes to the target database.
// By default, mongofiles waits for a majority of members from the replica set to respond before returning.
- WriteConcern string `long:"writeConcern" value-name:"<write-concern>" default:"majority" default-mask:"-" description:"write concern options e.g. --writeConcern majority, --writeConcern '{w: 3, wtimeout: 500, fsync: true, j: true}' (defaults to 'majority')"`
+ // Cannot be used simultaneously with write concern options in a URI.
+ WriteConcern string `long:"writeConcern" value-name:"<write-concern>" default-mask:"-" description:"write concern options e.g. --writeConcern majority, --writeConcern '{w: 3, wtimeout: 500, fsync: true, j: true}'"`
}
// Name returns a human-readable group name for storage options.
diff --git a/src/mongo/gotools/mongofiles/options_test.go b/src/mongo/gotools/mongofiles/options_test.go
new file mode 100644
index 00000000000..97fb48f51f6
--- /dev/null
+++ b/src/mongo/gotools/mongofiles/options_test.go
@@ -0,0 +1,74 @@
+package mongofiles
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/mongodb/mongo-tools/common/db"
+ "github.com/mongodb/mongo-tools/common/options"
+ . "github.com/smartystreets/goconvey/convey"
+)
+
+// Regression test for TOOLS-1741
+func TestWriteConcernWithURIParsing(t *testing.T) {
+ Convey("With an IngestOptions and ToolsOptions", t, func() {
+
+ // create an 'EnabledOptions' to determine what options should be able to be
+ // parsed and set form the input.
+ enabled := options.EnabledOptions{URI: true}
+
+ // create a new tools options to hold the parsed options
+ opts := options.New("", "", enabled)
+
+ // create a 'StorageOptions', which holds the value of the write concern
+ // for mongofiles.
+ storageOpts := &StorageOptions{}
+ opts.AddOptions(storageOpts)
+
+ // Specify that a write concern set on the URI is not an error and is a known
+ // possible option.
+ opts.URI.AddKnownURIParameters(options.KnownURIOptionsWriteConcern)
+
+ Convey("Parsing with no value should leave write concern empty", func() {
+ _, err := opts.ParseArgs([]string{})
+ So(err, ShouldBeNil)
+ So(storageOpts.WriteConcern, ShouldEqual, "")
+ Convey("and building write concern object, WMode should be majority", func() {
+ sessionSafety, err := db.BuildWriteConcern(storageOpts.WriteConcern, "",
+ opts.ParsedConnString())
+ So(err, ShouldBeNil)
+ So(sessionSafety.WMode, ShouldEqual, "majority")
+ })
+ })
+
+ Convey("Parsing with no writeconcern in URI should not error", func() {
+ args := []string{
+ "--uri", "mongodb://localhost:27017/test",
+ }
+ _, err := opts.ParseArgs(args)
+ So(err, ShouldBeNil)
+ So(storageOpts.WriteConcern, ShouldEqual, "")
+ Convey("and parsing write concern, WMode should be majority", func() {
+ sessionSafety, err := db.BuildWriteConcern(storageOpts.WriteConcern, "",
+ opts.ParsedConnString())
+ So(err, ShouldBeNil)
+ So(sessionSafety, ShouldNotBeNil)
+ So(sessionSafety.WMode, ShouldEqual, "majority")
+ })
+ })
+ Convey("Parsing with both writeconcern in URI and command line should error", func() {
+ args := []string{
+ "--uri", "mongodb://localhost:27017/test",
+ "--writeConcern", "majority",
+ }
+ _, err := opts.ParseArgs(args)
+ So(err, ShouldBeNil)
+ So(storageOpts.WriteConcern, ShouldEqual, "majority")
+ Convey("and parsing write concern, WMode should be majority", func() {
+ _, err := db.BuildWriteConcern(storageOpts.WriteConcern, "",
+ opts.ParsedConnString())
+ So(err, ShouldResemble, fmt.Errorf("cannot specify writeConcern string and connectionString object"))
+ })
+ })
+ })
+}
diff --git a/src/mongo/gotools/mongoimport/options.go b/src/mongo/gotools/mongoimport/options.go
index 8ab9f9c9d47..1e59ad43f52 100644
--- a/src/mongo/gotools/mongoimport/options.go
+++ b/src/mongo/gotools/mongoimport/options.go
@@ -69,7 +69,9 @@ type IngestOptions struct {
UpsertFields string `long:"upsertFields" value-name:"<field>[,<field>]*" description:"comma-separated fields for the query part when --mode is set to upsert or merge"`
// Sets write concern level for write operations.
- WriteConcern string `long:"writeConcern" default:"majority" value-name:"<write-concern-specifier>" default-mask:"-" description:"write concern options e.g. --writeConcern majority, --writeConcern '{w: 3, wtimeout: 500, fsync: true, j: true}' (defaults to 'majority')"`
+ // By default mongoimport uses a write concern of 'majority'.
+ // Cannot be used simultaneously with write concern options in a URI.
+ WriteConcern string `long:"writeConcern" value-name:"<write-concern-specifier>" default-mask:"-" description:"write concern options e.g. --writeConcern majority, --writeConcern '{w: 3, wtimeout: 500, fsync: true, j: true}'"`
// Indicates that the server should bypass document validation on import.
BypassDocumentValidation bool `long:"bypassDocumentValidation" description:"bypass document validation"`
diff --git a/src/mongo/gotools/mongoimport/options_test.go b/src/mongo/gotools/mongoimport/options_test.go
new file mode 100644
index 00000000000..55950a5d5c7
--- /dev/null
+++ b/src/mongo/gotools/mongoimport/options_test.go
@@ -0,0 +1,74 @@
+package mongoimport
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/mongodb/mongo-tools/common/db"
+ "github.com/mongodb/mongo-tools/common/options"
+ . "github.com/smartystreets/goconvey/convey"
+)
+
+// Regression test for TOOLS-1741
+func TestWriteConcernWithURIParsing(t *testing.T) {
+ Convey("With an IngestOptions and ToolsOptions", t, func() {
+
+ // create an 'EnabledOptions' to determine what options should be able to be
+ // parsed and set form the input.
+ enabled := options.EnabledOptions{URI: true}
+
+ // create a new tools options to hold the parsed options
+ opts := options.New("", "", enabled)
+
+ // create an 'IngestOptions', which holds the value of the write concern
+ // for mongoimport.
+ ingestOpts := &IngestOptions{}
+ opts.AddOptions(ingestOpts)
+
+ // Specify that a write concern set on the URI is not an error and is a known
+ // possible option.
+ opts.URI.AddKnownURIParameters(options.KnownURIOptionsWriteConcern)
+
+ Convey("Parsing with no value should leave write concern empty", func() {
+ _, err := opts.ParseArgs([]string{})
+ So(err, ShouldBeNil)
+ So(ingestOpts.WriteConcern, ShouldEqual, "")
+ Convey("and building write concern object, WMode should be majority", func() {
+ sessionSafety, err := db.BuildWriteConcern(ingestOpts.WriteConcern, "",
+ opts.ParsedConnString())
+ So(err, ShouldBeNil)
+ So(sessionSafety.WMode, ShouldEqual, "majority")
+ })
+ })
+
+ Convey("Parsing with no writeconcern in URI should not error", func() {
+ args := []string{
+ "--uri", "mongodb://localhost:27017/test",
+ }
+ _, err := opts.ParseArgs(args)
+ So(err, ShouldBeNil)
+ So(ingestOpts.WriteConcern, ShouldEqual, "")
+ Convey("and parsing write concern, WMode should be majority", func() {
+ sessionSafety, err := db.BuildWriteConcern(ingestOpts.WriteConcern, "",
+ opts.ParsedConnString())
+ So(err, ShouldBeNil)
+ So(sessionSafety, ShouldNotBeNil)
+ So(sessionSafety.WMode, ShouldEqual, "majority")
+ })
+ })
+ Convey("Parsing with both writeconcern in URI and command line should error", func() {
+ args := []string{
+ "--uri", "mongodb://localhost:27017/test",
+ "--writeConcern", "majority",
+ }
+ _, err := opts.ParseArgs(args)
+ So(err, ShouldBeNil)
+ So(ingestOpts.WriteConcern, ShouldEqual, "majority")
+ Convey("and parsing write concern, WMode should be majority", func() {
+ _, err := db.BuildWriteConcern(ingestOpts.WriteConcern, "",
+ opts.ParsedConnString())
+ So(err, ShouldResemble, fmt.Errorf("cannot specify writeConcern string and connectionString object"))
+ })
+ })
+ })
+}
diff --git a/src/mongo/gotools/mongorestore/options.go b/src/mongo/gotools/mongorestore/options.go
index 93965cc3e2b..0913cf52e21 100644
--- a/src/mongo/gotools/mongorestore/options.go
+++ b/src/mongo/gotools/mongorestore/options.go
@@ -29,9 +29,12 @@ func (*InputOptions) Name() string {
// OutputOptions defines the set of options for restoring dump data.
type OutputOptions struct {
- Drop bool `long:"drop" description:"drop each collection before import"`
- DryRun bool `long:"dryRun" description:"view summary without importing anything. recommended with verbosity"`
- WriteConcern string `long:"writeConcern" value-name:"<write-concern>" default:"majority" default-mask:"-" description:"write concern options e.g. --writeConcern majority, --writeConcern '{w: 3, wtimeout: 500, fsync: true, j: true}' (defaults to 'majority')"`
+ Drop bool `long:"drop" description:"drop each collection before import"`
+ DryRun bool `long:"dryRun" description:"view summary without importing anything. recommended with verbosity"`
+
+ // By default mongorestore uses a write concern of 'majority'.
+ // Cannot be used simultaneously with write concern options in a URI.
+ WriteConcern string `long:"writeConcern" value-name:"<write-concern>" default-mask:"-" description:"write concern options e.g. --writeConcern majority, --writeConcern '{w: 3, wtimeout: 500, fsync: true, j: true}'"`
NoIndexRestore bool `long:"noIndexRestore" description:"don't restore indexes"`
NoOptionsRestore bool `long:"noOptionsRestore" description:"don't restore collection options"`
KeepIndexVersion bool `long:"keepIndexVersion" description:"don't update index version"`
diff --git a/src/mongo/gotools/mongorestore/options_test.go b/src/mongo/gotools/mongorestore/options_test.go
new file mode 100644
index 00000000000..5513ecf2f02
--- /dev/null
+++ b/src/mongo/gotools/mongorestore/options_test.go
@@ -0,0 +1,74 @@
+package mongorestore
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/mongodb/mongo-tools/common/db"
+ "github.com/mongodb/mongo-tools/common/options"
+ . "github.com/smartystreets/goconvey/convey"
+)
+
+// Regression test for TOOLS-1741
+func TestWriteConcernWithURIParsing(t *testing.T) {
+ Convey("With an IngestOptions and ToolsOptions", t, func() {
+
+ // create an 'EnabledOptions' to determine what options should be able to be
+ // parsed and set form the input.
+ enabled := options.EnabledOptions{URI: true}
+
+ // create a new tools options to hold the parsed options
+ opts := options.New("", "", enabled)
+
+ // create a 'OutputOptions', which holds the value of the write concern
+ // for mongorestore.
+ outputOpts := &OutputOptions{}
+ opts.AddOptions(outputOpts)
+
+ // Specify that a write concern set on the URI is not an error and is a known
+ // possible option.
+ opts.URI.AddKnownURIParameters(options.KnownURIOptionsWriteConcern)
+
+ Convey("Parsing with no value should leave write concern empty", func() {
+ _, err := opts.ParseArgs([]string{})
+ So(err, ShouldBeNil)
+ So(outputOpts.WriteConcern, ShouldEqual, "")
+ Convey("and building write concern object, WMode should be majority", func() {
+ sessionSafety, err := db.BuildWriteConcern(outputOpts.WriteConcern, "",
+ opts.ParsedConnString())
+ So(err, ShouldBeNil)
+ So(sessionSafety.WMode, ShouldEqual, "majority")
+ })
+ })
+
+ Convey("Parsing with no writeconcern in URI should not error", func() {
+ args := []string{
+ "--uri", "mongodb://localhost:27017/test",
+ }
+ _, err := opts.ParseArgs(args)
+ So(err, ShouldBeNil)
+ So(outputOpts.WriteConcern, ShouldEqual, "")
+ Convey("and parsing write concern, WMode should be majority", func() {
+ sessionSafety, err := db.BuildWriteConcern(outputOpts.WriteConcern, "",
+ opts.ParsedConnString())
+ So(err, ShouldBeNil)
+ So(sessionSafety, ShouldNotBeNil)
+ So(sessionSafety.WMode, ShouldEqual, "majority")
+ })
+ })
+ Convey("Parsing with both writeconcern in URI and command line should error", func() {
+ args := []string{
+ "--uri", "mongodb://localhost:27017/test",
+ "--writeConcern", "majority",
+ }
+ _, err := opts.ParseArgs(args)
+ So(err, ShouldBeNil)
+ So(outputOpts.WriteConcern, ShouldEqual, "majority")
+ Convey("and parsing write concern, WMode should be majority", func() {
+ _, err := db.BuildWriteConcern(outputOpts.WriteConcern, "",
+ opts.ParsedConnString())
+ So(err, ShouldResemble, fmt.Errorf("cannot specify writeConcern string and connectionString object"))
+ })
+ })
+ })
+}
diff --git a/src/mongo/gotools/vendor/src/github.com/spacemonkeygo/spacelog/capture_linux.go b/src/mongo/gotools/vendor/src/github.com/spacemonkeygo/spacelog/capture_linux.go
new file mode 100644
index 00000000000..160a3b3c1a9
--- /dev/null
+++ b/src/mongo/gotools/vendor/src/github.com/spacemonkeygo/spacelog/capture_linux.go
@@ -0,0 +1,33 @@
+// Copyright (C) 2014 Space Monkey, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package spacelog
+
+import (
+ "syscall"
+)
+
+// CaptureOutputToFd redirects the current process' stdout and stderr file
+// descriptors to the given file descriptor, using the dup3 syscall.
+func CaptureOutputToFd(fd int) error {
+ err := syscall.Dup3(fd, syscall.Stdout, 0)
+ if err != nil {
+ return err
+ }
+ err = syscall.Dup3(fd, syscall.Stderr, 0)
+ if err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/src/mongo/gotools/vendor/src/github.com/spacemonkeygo/spacelog/capture_other.go b/src/mongo/gotools/vendor/src/github.com/spacemonkeygo/spacelog/capture_other.go
index 5a62a2accaf..5fb22093798 100644
--- a/src/mongo/gotools/vendor/src/github.com/spacemonkeygo/spacelog/capture_other.go
+++ b/src/mongo/gotools/vendor/src/github.com/spacemonkeygo/spacelog/capture_other.go
@@ -13,6 +13,7 @@
// limitations under the License.
// +build !windows
+// +build !linux
package spacelog