summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authormike o'brien <mpobrien005@gmail.com>2014-11-18 19:09:14 -0500
committermike o'brien <mpobrien005@gmail.com>2014-11-20 14:51:25 -0500
commite4fa10d01cf26f6808c5b52fff351a16b66b67bd (patch)
tree15fd28ebfc6159cc0bdffeac2b2bf0e1a821f4b6
parent47c44d0a4ca508729510b57a8557b9ed75f55cc3 (diff)
downloadmongo-e4fa10d01cf26f6808c5b52fff351a16b66b67bd.tar.gz
TOOLS-295 make certain flags hidden, make --dbpath trigger error message
Former-commit-id: 17ddf0fa38c75b4886969f8de0fcdc8a12745874
-rw-r--r--Godeps2
-rw-r--r--common/log/tool_logger.go26
-rw-r--r--common/options/options.go101
-rw-r--r--mongodump/main/mongodump.go6
-rw-r--r--mongodump/mongodump.go10
-rw-r--r--mongodump/mongodump_kerberos_test.go3
-rw-r--r--mongodump/mongodump_test.go15
-rw-r--r--mongodump/options/options.go2
-rw-r--r--mongoexport/mongoexport.go7
-rw-r--r--mongoimport/common.go9
-rw-r--r--mongoimport/common_test.go32
-rw-r--r--mongoimport/csv.go19
-rw-r--r--mongoimport/csv_test.go52
-rw-r--r--mongoimport/json.go13
-rw-r--r--mongoimport/json_test.go36
-rw-r--r--mongoimport/main/mongoimport.go2
-rw-r--r--mongoimport/mongoimport.go98
-rw-r--r--mongoimport/mongoimport_test.go835
-rw-r--r--mongoimport/options/options.go13
-rw-r--r--mongoimport/tsv.go13
-rw-r--r--mongoimport/tsv_test.go20
-rw-r--r--mongorestore/main/mongorestore.go4
-rw-r--r--mongorestore/mongorestore.go11
-rw-r--r--mongorestore/options/options.go18
-rw-r--r--mongorestore/restore.go16
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/.travis.yml35
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/README.md3
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/arg.go21
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/arg_test.go53
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/assert_test.go113
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/closest.go52
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/command.go16
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/command_private.go57
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/command_test.go22
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/completion.go304
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/completion_test.go289
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/convert.go26
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/convert_test.go14
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/error.go32
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/example_test.go31
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/examples/bash-completion9
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/examples/main.go4
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/flags.go385
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/group.go8
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/group_private.go45
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/group_test.go27
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/help.go181
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/help_test.go188
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/ini.go4
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/ini_private.go201
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/ini_test.go420
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/man.go24
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/marshal_test.go2
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/option.go72
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/option_private.go81
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/optstyle_other.go6
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/optstyle_windows.go15
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/parser.go118
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/parser_private.go169
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/parser_test.go316
-rw-r--r--vendor/src/github.com/jessevdk/go-flags/short_test.go7
-rw-r--r--vendor/src/github.com/spacemonkeygo/spacelog/setup.go4
-rw-r--r--vendor/src/github.com/spacemonkeygo/spacelog/templates_others.go22
-rw-r--r--vendor/src/github.com/spacemonkeygo/spacelog/templates_windows.go20
64 files changed, 3318 insertions, 1441 deletions
diff --git a/Godeps b/Godeps
index ee9bfdcaabc..4686895b521 100644
--- a/Godeps
+++ b/Godeps
@@ -1,7 +1,7 @@
gopkg.in/mgo.v2 e2e914857713db7497cca2bd7fc0b030fc9cb22d
github.com/jacobsa/oglematchers 4fc24f97b5b74022c2a3f4ca7eed57ca29083d3e
github.com/smartystreets/goconvey 75bc4a2dad71e5c5b51c5009b33eba766ec57051
-github.com/jessevdk/go-flags 8ec9564882e7923e632f012761c81c46dcf5bec1
+github.com/jessevdk/go-flags 37e89eb6730ccfebfa05c17940954e4308719317
gopkg.in/tomb.v2 14b3d72120e8d10ea6e6b7f87f7175734b1faab8
github.com/3rf/mongo-lint 3550fdcf1f43b89aaeabaa4559eaae6dc4407e42
github.com/spacemonkeygo/openssl 500a817338511aad07db09254b7ccf6916ac6a19 github.com/gabrielrussell/openssl
diff --git a/common/log/tool_logger.go b/common/log/tool_logger.go
index 8f7577b0f34..565be55d08e 100644
--- a/common/log/tool_logger.go
+++ b/common/log/tool_logger.go
@@ -2,7 +2,6 @@ package log
import (
"fmt"
- "github.com/mongodb/mongo-tools/common/options"
"io"
"os"
"sync"
@@ -30,11 +29,21 @@ type ToolLogger struct {
verbosity int
}
-func (tl *ToolLogger) SetVerbosity(verbosity *options.Verbosity) {
- if verbosity.Quiet {
+type VerbosityLevel interface {
+ Level() int
+ IsQuiet() bool
+}
+
+func (tl *ToolLogger) SetVerbosity(level VerbosityLevel) {
+ if level == nil {
+ tl.verbosity = 0
+ return
+ }
+
+ if level.IsQuiet() {
tl.verbosity = -1
} else {
- tl.verbosity = len(verbosity.Verbose)
+ tl.verbosity = level.Level()
}
}
@@ -74,7 +83,7 @@ func (tl *ToolLogger) log(msg string) {
fmt.Fprintf(tl.writer, "%v\t%v\n", time.Now().Format(tl.format), msg)
}
-func NewToolLogger(verbosity *options.Verbosity) *ToolLogger {
+func NewToolLogger(verbosity VerbosityLevel) *ToolLogger {
tl := &ToolLogger{
mutex: &sync.Mutex{},
writer: os.Stderr, // default to stderr
@@ -111,10 +120,7 @@ var globalToolLogger *ToolLogger
func init() {
if globalToolLogger == nil {
// initialize tool logger with verbosity level = 0
- globalToolLogger = NewToolLogger(&options.Verbosity{
- []bool{}, // Verbose
- false, // Quiet
- })
+ globalToolLogger = NewToolLogger(nil)
}
}
@@ -126,7 +132,7 @@ func Log(minVerb int, msg string) {
globalToolLogger.Log(minVerb, msg)
}
-func SetVerbosity(verbosity *options.Verbosity) {
+func SetVerbosity(verbosity VerbosityLevel) {
globalToolLogger.SetVerbosity(verbosity)
}
diff --git a/common/options/options.go b/common/options/options.go
index ae06180f203..03bd0b435f7 100644
--- a/common/options/options.go
+++ b/common/options/options.go
@@ -5,7 +5,10 @@ package options
import (
"fmt"
"github.com/jessevdk/go-flags"
+ "github.com/mongodb/mongo-tools/common/log"
"os"
+ "runtime"
+ "strconv"
)
const (
@@ -33,6 +36,7 @@ type ToolOptions struct {
*Auth
*Kerberos
*Namespace
+ *HiddenOptions
//Force direct connection to the server and disable the
//drivers automatic repl set discovery logic.
@@ -42,15 +46,22 @@ type ToolOptions struct {
parser *flags.Parser
}
+type HiddenOptions struct {
+ MaxProcs int
+ BulkWriters int
+ BulkBufferSize int
+
+ // Specifies the number of threads to use in processing data read from the input source
+ NumDecodingWorkers int
+
+ // Specifies the number of threads to use in sending processed data over to the server
+ NumInsertionWorkers int
+}
+
type Namespace struct {
// Specified database and collection
DB string `short:"d" long:"db" description:"database to use"`
Collection string `short:"c" long:"collection" description:"collection to use"`
-
- // DBPath for direct-storage interface
- DBPath string `long:"dbpath"`
- DirectoryPerDB bool `long:"directoryperdb" default:"false"`
- Journal bool `long:"journal" default:"false"`
}
// Struct holding generic options
@@ -65,6 +76,14 @@ type Verbosity struct {
Quiet bool `long:"quiet" description:"Run in quiet mode, attempting to limit the amount of output"`
}
+func (v Verbosity) Level() int {
+ return len(v.Verbose)
+}
+
+func (v Verbosity) IsQuiet() bool {
+ return v.Quiet
+}
+
// Struct holding connection-related options
type Connection struct {
Host string `short:"h" long:"host" description:"Specify a resolvable hostname to which to connect"`
@@ -109,22 +128,30 @@ type EnabledOptions struct {
// Ask for a new instance of tool options
func New(appName, usageStr string, enabled EnabledOptions) *ToolOptions {
+ hiddenOpts := &HiddenOptions{
+ BulkWriters: 1,
+ BulkBufferSize: 10000,
+ }
+
opts := &ToolOptions{
AppName: appName,
VersionStr: VersionStr,
UsageStr: usageStr,
- General: &General{},
- Verbosity: &Verbosity{},
- Connection: &Connection{},
- SSL: &SSL{},
- Auth: &Auth{},
- Namespace: &Namespace{},
- Kerberos: &Kerberos{},
- parser: flags.NewNamedParser(appName, flags.None),
+ General: &General{},
+ Verbosity: &Verbosity{},
+ Connection: &Connection{},
+ SSL: &SSL{},
+ Auth: &Auth{},
+ Namespace: &Namespace{},
+ HiddenOptions: hiddenOpts,
+ Kerberos: &Kerberos{},
+ parser: flags.NewNamedParser(appName, flags.None),
}
- opts.parser.Usage = usageStr
+ opts.parser.UnknownOptionHandler = func(option string, args []string) ([]string, error) {
+ return parseHiddenOption(hiddenOpts, option, args)
+ }
if _, err := opts.parser.AddGroup("general options", "", opts.General); err != nil {
panic(fmt.Errorf("couldn't register general options: %v", err))
@@ -156,6 +183,12 @@ func New(appName, usageStr string, enabled EnabledOptions) *ToolOptions {
panic(fmt.Errorf("couldn't register namespace options"))
}
}
+
+ if opts.MaxProcs <= 0 {
+ opts.MaxProcs = runtime.NumCPU()
+ }
+ log.Logf(log.Info, "Setting num cpus to %v", opts.MaxProcs)
+ runtime.GOMAXPROCS(opts.MaxProcs)
return opts
}
@@ -211,12 +244,40 @@ func (self *ToolOptions) Parse() ([]string, error) {
return self.parser.Parse()
}
-//Validate() runs validation checks that are global to all tools.
-func (self *ToolOptions) Validate() error {
- switch {
- case self.DBPath != "" && self.Host != "":
- return fmt.Errorf("--dbpath is not allowed when --host is specified")
+func parseHiddenOption(opts *HiddenOptions, option string, args []string) ([]string, error) {
+ if option == "dbpath" || option == "directoryperdb" || option == "journal" {
+ return args, fmt.Errorf(`--dbpath and related flags are not supported in 2.8 tools.
+See http://dochub.mongodb.org/core/tools-dbpath-deprecated for more information`)
+ }
+ var err error
+ optionValue, err := getInt(args)
+ switch option {
+ case "numThreads":
+ opts.MaxProcs = optionValue
+ case "numInsertionWorkersPerCollection":
+ opts.BulkWriters = optionValue
+ case "batchSize":
+ opts.BulkBufferSize = optionValue
+ case "numDecodingWorkers":
+ opts.NumDecodingWorkers = optionValue
+ case "numInsertionWorkers":
+ opts.NumInsertionWorkers = optionValue
+ default:
+ return args, fmt.Errorf(`unknown option "%v"`, option)
+ }
+ if err != nil {
+ return args, fmt.Errorf(`error parsing value for "%v": %v`, option, err)
}
+ return args[1:], nil
+}
- return nil
+func getInt(args []string) (int, error) {
+ if len(args) == 0 {
+ return 0, fmt.Errorf("no value specified")
+ }
+ val, err := strconv.Atoi(args[0])
+ if err != nil {
+ return 0, fmt.Errorf("expected an integer value but got '%v'")
+ }
+ return val, nil
}
diff --git a/mongodump/main/mongodump.go b/mongodump/main/mongodump.go
index 89c4d2deffb..ec8f5b128ff 100644
--- a/mongodump/main/mongodump.go
+++ b/mongodump/main/mongodump.go
@@ -6,7 +6,6 @@ import (
"github.com/mongodb/mongo-tools/mongodump"
"github.com/mongodb/mongo-tools/mongodump/options"
"os"
- "runtime"
)
func main() {
@@ -38,11 +37,6 @@ func main() {
// init logger
log.SetVerbosity(opts.Verbosity)
- if outputOpts.MaxProcs > 0 {
- log.Logf(log.DebugHigh, "setting GOMAXPROCS to %v", outputOpts.MaxProcs)
- runtime.GOMAXPROCS(outputOpts.MaxProcs)
- }
-
// don't attempt to discover other members of a replica set
opts.Direct = true
diff --git a/mongodump/mongodump.go b/mongodump/mongodump.go
index 1c52b4d61bd..abba8edd280 100644
--- a/mongodump/mongodump.go
+++ b/mongodump/mongodump.go
@@ -49,9 +49,6 @@ type MongoDump struct {
// ValidateOptions checks for any incompatible sets of options
func (dump *MongoDump) ValidateOptions() error {
- if err := dump.ToolOptions.Validate(); err != nil {
- return err
- }
switch {
case dump.OutputOptions.Out == "-" && dump.ToolOptions.Namespace.Collection == "":
return fmt.Errorf("can only dump a single collection to stdout")
@@ -75,8 +72,6 @@ func (dump *MongoDump) ValidateOptions() error {
return fmt.Errorf("--db is required when --excludeCollectionsWithPrefix is specified")
case dump.OutputOptions.Repair && dump.InputOptions.Query != "":
return fmt.Errorf("cannot run a query with --repair enabled")
- case dump.OutputOptions.JobThreads < 1:
- return fmt.Errorf("number of processing threads must be >= 1")
}
return nil
}
@@ -260,7 +255,10 @@ func (dump *MongoDump) Dump() error {
func (dump *MongoDump) DumpIntents() error {
resultChan := make(chan error)
- jobs := dump.OutputOptions.JobThreads
+ jobs := dump.ToolOptions.HiddenOptions.MaxProcs
+ if jobs <= 0 {
+ jobs = 1
+ }
if jobs > 1 {
dump.manager.Finalize(intents.LongestTaskFirst)
} else {
diff --git a/mongodump/mongodump_kerberos_test.go b/mongodump/mongodump_kerberos_test.go
index e5105cf0b57..822bf680b14 100644
--- a/mongodump/mongodump_kerberos_test.go
+++ b/mongodump/mongodump_kerberos_test.go
@@ -25,9 +25,6 @@ func TestMongoDumpKerberos(t *testing.T) {
mongoDump := MongoDump{
ToolOptions: opts,
InputOptions: &options.InputOptions{},
- OutputOptions: &options.OutputOptions{
- JobThreads: 1,
- },
}
mongoDump.OutputOptions.Out = KERBEROS_DUMP_DIRECTORY
diff --git a/mongodump/mongodump_test.go b/mongodump/mongodump_test.go
index f93615accaf..dd61c7c599a 100644
--- a/mongodump/mongodump_test.go
+++ b/mongodump/mongodump_test.go
@@ -43,15 +43,14 @@ func simpleMongoDumpInstance() *MongoDump {
Port: testPort,
}
toolOptions := &commonOpts.ToolOptions{
- SSL: &ssl,
- Namespace: namespace,
- Connection: connection,
- Auth: &auth,
- Verbosity: &commonOpts.Verbosity{},
- }
- outputOptions := &options.OutputOptions{
- JobThreads: 1,
+ SSL: &ssl,
+ Namespace: namespace,
+ Connection: connection,
+ Auth: &auth,
+ HiddenOptions: &commonOpts.HiddenOptions{},
+ Verbosity: &commonOpts.Verbosity{},
}
+ outputOptions := &options.OutputOptions{}
inputOptions := &options.InputOptions{}
log.SetVerbosity(toolOptions.Verbosity)
diff --git a/mongodump/options/options.go b/mongodump/options/options.go
index 05f6fa6a0b2..b4f0d1086dd 100644
--- a/mongodump/options/options.go
+++ b/mongodump/options/options.go
@@ -19,8 +19,6 @@ type OutputOptions struct {
DumpDBUsersAndRoles bool `long:"dumpDbUsersAndRoles" description:"Dump user and role definitions for the given database"`
ExcludedCollections []string `long:"excludeCollection" description:"Collections to exclude from the dump"`
ExcludedCollectionPrefixes []string `long:"excludeCollectionsWithPrefix" description:"Exclude all collections from the dump that have the given prefix"`
- JobThreads int `long:"numParallelCollections" short:"j" description:"Number of collections to dump in parallel" default:"4"`
- MaxProcs int `long:"numCPUThreads" description:"GOMAXPROCS for testing"` // TODO: hide this option
}
func (self *OutputOptions) Name() string {
diff --git a/mongoexport/mongoexport.go b/mongoexport/mongoexport.go
index 2246b790adc..826aa3a8f63 100644
--- a/mongoexport/mongoexport.go
+++ b/mongoexport/mongoexport.go
@@ -65,13 +65,6 @@ type ExportOutput interface {
// ValidateSettings returns an error if any settings specified on the command line
// were invalid, or nil if they are valid.
func (exp *MongoExport) ValidateSettings() error {
- // TODO - on legacy mongoexport, if -d is blank, it assumes some default database.
- // Do we want to use that same behavior? It seems very odd to assume the DB
- // when only a collection is provided, but that's the behavior of the legacy tools.
- if err := exp.ToolOptions.Validate(); err != nil {
- return err
- }
-
// Namespace must have a valid database if none is specified,
// use 'test'
if exp.ToolOptions.Namespace.DB == "" {
diff --git a/mongoimport/common.go b/mongoimport/common.go
index 8279cefbc9b..6ad3ff33958 100644
--- a/mongoimport/common.go
+++ b/mongoimport/common.go
@@ -194,13 +194,16 @@ func setNestedValue(key string, value interface{}, document *bson.D) {
// channel in parallel and then sends over the processed data to the outputChan
// channel - either in sequence or concurrently (depending on the value of
// ordered) - in which the data was received
-func streamDocuments(ordered bool, inputChan chan ConvertibleDoc, outputChan chan bson.D, errChan chan error) {
+func streamDocuments(ordered bool, numDecoders int, inputChan chan ConvertibleDoc, outputChan chan bson.D, errChan chan error) {
+ if numDecoders == 0 {
+ numDecoders = 1
+ }
var importWorkers []*ImportWorker
// initialize all our concurrent processing threads
wg := &sync.WaitGroup{}
inChan := inputChan
outChan := outputChan
- for i := 0; i < numDecodingWorkers; i++ {
+ for i := 0; i < numDecoders; i++ {
if ordered {
// TODO: experiment with buffered channel size; the buffer size of
// inChan should always be the same as that of outChan
@@ -234,7 +237,7 @@ func streamDocuments(ordered bool, inputChan chan ConvertibleDoc, outputChan cha
// tokensToBSON reads in slice of records - along with ordered fields names -
// and returns a BSON document for the record.
func tokensToBSON(fields, tokens []string, numProcessed uint64) (bson.D, error) {
- log.Logf(log.DebugLow, "got line: %v", tokens)
+ log.Logf(log.DebugHigh, "got line: %v", tokens)
var parsedValue interface{}
document := bson.D{}
for index, token := range tokens {
diff --git a/mongoimport/common_test.go b/mongoimport/common_test.go
index ab262abd785..23740ffb8f9 100644
--- a/mongoimport/common_test.go
+++ b/mongoimport/common_test.go
@@ -102,51 +102,51 @@ func TestValidateHeaders(t *testing.T) {
So(err, ShouldBeNil)
Convey("if headerLine is true, the first line in the input should be used", func() {
- headers, err := validateHeaders(NewCSVInputReader(fields, fileHandle), true)
+ headers, err := validateHeaders(NewCSVInputReader(fields, fileHandle, 1), true)
So(err, ShouldBeNil)
So(len(headers), ShouldEqual, 2)
// spaces are trimed in the header
So(headers, ShouldResemble, strings.Split(strings.Replace(contents, " ", "", -1), ","))
})
Convey("if headerLine is false, the fields passed in should be used", func() {
- headers, err := validateHeaders(NewCSVInputReader(fields, fileHandle), false)
+ headers, err := validateHeaders(NewCSVInputReader(fields, fileHandle, 1), false)
So(err, ShouldBeNil)
So(len(headers), ShouldEqual, 3)
// spaces are trimed in the header
So(headers, ShouldResemble, fields)
})
Convey("if the fields contain '..', an error should be thrown", func() {
- _, err := validateHeaders(NewCSVInputReader([]string{"a..a"}, fileHandle), false)
+ _, err := validateHeaders(NewCSVInputReader([]string{"a..a"}, fileHandle, 1), false)
So(err, ShouldNotBeNil)
})
Convey("if the fields start/end in a '.', an error should be thrown", func() {
- _, err := validateHeaders(NewCSVInputReader([]string{".a"}, fileHandle), false)
+ _, err := validateHeaders(NewCSVInputReader([]string{".a"}, fileHandle, 1), false)
So(err, ShouldNotBeNil)
- _, err = validateHeaders(NewCSVInputReader([]string{"a."}, fileHandle), false)
+ _, err = validateHeaders(NewCSVInputReader([]string{"a."}, fileHandle, 1), false)
So(err, ShouldNotBeNil)
})
Convey("if the fields collide, an error should be thrown", func() {
- _, err := validateHeaders(NewCSVInputReader([]string{"a", "a.a"}, fileHandle), false)
+ _, err := validateHeaders(NewCSVInputReader([]string{"a", "a.a"}, fileHandle, 1), false)
So(err, ShouldNotBeNil)
- _, err = validateHeaders(NewCSVInputReader([]string{"a", "a.ba", "b.a"}, fileHandle), false)
+ _, err = validateHeaders(NewCSVInputReader([]string{"a", "a.ba", "b.a"}, fileHandle, 1), false)
So(err, ShouldNotBeNil)
- _, err = validateHeaders(NewCSVInputReader([]string{"a", "a.b.c"}, fileHandle), false)
+ _, err = validateHeaders(NewCSVInputReader([]string{"a", "a.b.c"}, fileHandle, 1), false)
So(err, ShouldNotBeNil)
})
Convey("if the fields don't collide, no error should be thrown", func() {
- _, err := validateHeaders(NewCSVInputReader([]string{"a", "aa"}, fileHandle), false)
+ _, err := validateHeaders(NewCSVInputReader([]string{"a", "aa"}, fileHandle, 1), false)
So(err, ShouldBeNil)
- _, err = validateHeaders(NewCSVInputReader([]string{"a", "aa", "b.a", "b.c"}, fileHandle), false)
+ _, err = validateHeaders(NewCSVInputReader([]string{"a", "aa", "b.a", "b.c"}, fileHandle, 1), false)
So(err, ShouldBeNil)
- _, err = validateHeaders(NewCSVInputReader([]string{"a", "ba", "ab", "b.a"}, fileHandle), false)
+ _, err = validateHeaders(NewCSVInputReader([]string{"a", "ba", "ab", "b.a"}, fileHandle, 1), false)
So(err, ShouldBeNil)
- _, err = validateHeaders(NewCSVInputReader([]string{"a", "ba", "ab", "b.a", "b.c.d"}, fileHandle), false)
+ _, err = validateHeaders(NewCSVInputReader([]string{"a", "ba", "ab", "b.a", "b.c.d"}, fileHandle, 1), false)
So(err, ShouldBeNil)
- _, err = validateHeaders(NewCSVInputReader([]string{"a", "ab.c"}, fileHandle), false)
+ _, err = validateHeaders(NewCSVInputReader([]string{"a", "ab.c"}, fileHandle, 1), false)
So(err, ShouldBeNil)
})
Convey("if the fields contain the same keys, an error should be thrown", func() {
- _, err := validateHeaders(NewCSVInputReader([]string{"a", "ba", "a"}, fileHandle), false)
+ _, err := validateHeaders(NewCSVInputReader([]string{"a", "ba", "a"}, fileHandle, 1), false)
So(err, ShouldNotBeNil)
})
})
@@ -509,7 +509,7 @@ func TestStreamDocuments(t *testing.T) {
inputChannel <- csvConvertibleDoc
}
close(inputChannel)
- streamDocuments(true, inputChannel, outputChannel, errorChannel)
+ streamDocuments(true, 3, inputChannel, outputChannel, errorChannel)
// ensure documents are streamed out and processed in the correct manner
for _, expectedDocument := range expectedDocuments {
So(<-outputChannel, ShouldResemble, expectedDocument)
@@ -525,7 +525,7 @@ func TestStreamDocuments(t *testing.T) {
}
inputChannel <- csvConvertibleDoc
close(inputChannel)
- go streamDocuments(true, inputChannel, outputChannel, errorChannel)
+ go streamDocuments(true, 3, inputChannel, outputChannel, errorChannel)
// ensure that an error is returned on the error channel
So(<-errorChannel, ShouldNotBeNil)
})
diff --git a/mongoimport/csv.go b/mongoimport/csv.go
index 82acb0ce207..caddac19496 100644
--- a/mongoimport/csv.go
+++ b/mongoimport/csv.go
@@ -10,16 +10,20 @@ import (
// CSVInputReader is a struct that implements the InputReader interface for a
// CSV input source
type CSVInputReader struct {
+
// Fields is a list of field names in the BSON documents to be imported
Fields []string
- // csvReader is the underlying reader used to read data in from the CSV
- // or CSV file
+
+ // csvReader is the underlying reader used to read data in from the CSV or CSV file
csvReader *csv.Reader
+
// csvRecord stores each line of input we read from the underlying reader
csvRecord []string
- // numProcessed tracks the number of CSV records processed by the underlying
- // reader
+
+ // numProcessed tracks how many the number of CSV records processed by the underlying reader
numProcessed uint64
+
+ numDecoders int
}
// CSVConvertibleDoc implements the ConvertibleDoc interface for CSV input
@@ -30,7 +34,7 @@ type CSVConvertibleDoc struct {
// NewCSVInputReader returns a CSVInputReader configured to read input from the
// given io.Reader, extracting the specified fields only.
-func NewCSVInputReader(fields []string, in io.Reader) *CSVInputReader {
+func NewCSVInputReader(fields []string, in io.Reader, numDecoders int) *CSVInputReader {
csvReader := csv.NewReader(in)
// allow variable number of fields in document
csvReader.FieldsPerRecord = -1
@@ -39,6 +43,7 @@ func NewCSVInputReader(fields []string, in io.Reader) *CSVInputReader {
Fields: fields,
csvReader: csvReader,
numProcessed: uint64(0),
+ numDecoders: numDecoders,
}
}
@@ -68,7 +73,7 @@ func (csvInputReader *CSVInputReader) ReadHeadersFromSource() ([]string, error)
// hits EOF or an error. If ordered is true, it streams the documents in which
// the documents are read
func (csvInputReader *CSVInputReader) StreamDocument(ordered bool, readChan chan bson.D, errChan chan error) {
- csvRecordChan := make(chan ConvertibleDoc, numDecodingWorkers)
+ csvRecordChan := make(chan ConvertibleDoc, csvInputReader.numDecoders)
var err error
go func() {
@@ -92,7 +97,7 @@ func (csvInputReader *CSVInputReader) StreamDocument(ordered bool, readChan chan
csvInputReader.numProcessed++
}
}()
- streamDocuments(ordered, csvRecordChan, readChan, errChan)
+ streamDocuments(ordered, csvInputReader.numDecoders, csvRecordChan, readChan, errChan)
}
// This is required to satisfy the ConvertibleDoc interface for CSV input. It
diff --git a/mongoimport/csv_test.go b/mongoimport/csv_test.go
index 03e81bbab82..9a6808cd418 100644
--- a/mongoimport/csv_test.go
+++ b/mongoimport/csv_test.go
@@ -25,7 +25,7 @@ func TestCSVStreamDocument(t *testing.T) {
Convey("badly encoded CSV should result in a parsing error", func() {
contents := `1, 2, foo"bar`
fields := []string{"a", "b", "c"}
- csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go csvInputReader.StreamDocument(true, docChan, errChan)
@@ -34,7 +34,7 @@ func TestCSVStreamDocument(t *testing.T) {
Convey("escaped quotes are parsed correctly", func() {
contents := `1, 2, "foo""bar"`
fields := []string{"a", "b", "c"}
- csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go csvInputReader.StreamDocument(true, docChan, errChan)
@@ -43,7 +43,7 @@ func TestCSVStreamDocument(t *testing.T) {
Convey("whitespace separated quoted strings are still an error", func() {
contents := `1, 2, "foo" "bar"`
fields := []string{"a", "b", "c"}
- csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go csvInputReader.StreamDocument(true, docChan, errChan)
@@ -57,7 +57,7 @@ func TestCSVStreamDocument(t *testing.T) {
bson.DocElem{"b", 2},
bson.DocElem{"c", `foo" "bar`},
}
- csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go csvInputReader.StreamDocument(true, docChan, errChan)
@@ -72,7 +72,7 @@ func TestCSVStreamDocument(t *testing.T) {
bson.DocElem{"b", 2},
bson.DocElem{"c", " 3e"},
}
- csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go csvInputReader.StreamDocument(true, docChan, errChan)
@@ -89,7 +89,7 @@ func TestCSVStreamDocument(t *testing.T) {
bson.DocElem{"c", " 3e"},
bson.DocElem{"field3", " may"},
}
- csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go csvInputReader.StreamDocument(true, docChan, errChan)
@@ -108,7 +108,7 @@ func TestCSVStreamDocument(t *testing.T) {
bson.DocElem{"c", " 3e"},
bson.DocElem{"field3", " may"},
}
- csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go csvInputReader.StreamDocument(true, docChan, errChan)
@@ -124,7 +124,7 @@ func TestCSVStreamDocument(t *testing.T) {
Convey("nested CSV fields causing header collisions should error", func() {
contents := `1, 2f , " 3e" , " may", june`
fields := []string{"a", "b.c", "field3"}
- csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go csvInputReader.StreamDocument(true, docChan, errChan)
@@ -145,7 +145,7 @@ func TestCSVStreamDocument(t *testing.T) {
bson.DocElem{"b", 5},
bson.DocElem{"c", 6},
}
- csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go csvInputReader.StreamDocument(true, docChan, errChan)
@@ -164,7 +164,7 @@ func TestCSVSetHeader(t *testing.T) {
func() {
contents := "extraHeader1, extraHeader2, extraHeader3"
fields := []string{}
- csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
So(csvInputReader.SetHeader(true), ShouldBeNil)
So(len(csvInputReader.Fields), ShouldEqual, 3)
})
@@ -172,24 +172,24 @@ func TestCSVSetHeader(t *testing.T) {
Convey("setting non-colliding nested CSV headers should not raise an error", func() {
contents := "a, b, c"
fields := []string{}
- csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
So(csvInputReader.SetHeader(true), ShouldBeNil)
So(len(csvInputReader.Fields), ShouldEqual, 3)
contents = "a.b.c, a.b.d, c"
fields = []string{}
- csvInputReader = NewCSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ csvInputReader = NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
So(csvInputReader.SetHeader(true), ShouldBeNil)
So(len(csvInputReader.Fields), ShouldEqual, 3)
contents = "a.b, ab, a.c"
fields = []string{}
- csvInputReader = NewCSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ csvInputReader = NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
So(csvInputReader.SetHeader(true), ShouldBeNil)
So(len(csvInputReader.Fields), ShouldEqual, 3)
contents = "a, ab, ac, dd"
fields = []string{}
- csvInputReader = NewCSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ csvInputReader = NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
So(csvInputReader.SetHeader(true), ShouldBeNil)
So(len(csvInputReader.Fields), ShouldEqual, 4)
})
@@ -197,17 +197,17 @@ func TestCSVSetHeader(t *testing.T) {
Convey("setting colliding nested CSV headers should raise an error", func() {
contents := "a, a.b, c"
fields := []string{}
- csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
So(csvInputReader.SetHeader(true), ShouldNotBeNil)
contents = "a.b.c, a.b.d.c, a.b.d"
fields = []string{}
- csvInputReader = NewCSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ csvInputReader = NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
So(csvInputReader.SetHeader(true), ShouldNotBeNil)
contents = "a, a, a"
fields = []string{}
- csvInputReader = NewCSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ csvInputReader = NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
So(csvInputReader.SetHeader(true), ShouldNotBeNil)
})
@@ -216,32 +216,32 @@ func TestCSVSetHeader(t *testing.T) {
contents := "c, a., b"
fields := []string{}
So(err, ShouldBeNil)
- So(NewCSVInputReader(fields, bytes.NewReader([]byte(contents))).SetHeader(true), ShouldNotBeNil)
+ So(NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1).SetHeader(true), ShouldNotBeNil)
})
Convey("setting the header that starts in a dot should error",
func() {
contents := "c, .a, b"
fields := []string{}
- So(NewCSVInputReader(fields, bytes.NewReader([]byte(contents))).SetHeader(true), ShouldNotBeNil)
+ So(NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1).SetHeader(true), ShouldNotBeNil)
})
Convey("setting the header that contains multiple consecutive dots should error",
func() {
contents := "c, a..a, b"
fields := []string{}
- So(NewCSVInputReader(fields, bytes.NewReader([]byte(contents))).SetHeader(true), ShouldNotBeNil)
+ So(NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1).SetHeader(true), ShouldNotBeNil)
contents = "c, a.a, b.b...b"
fields = []string{}
- So(NewCSVInputReader(fields, bytes.NewReader([]byte(contents))).SetHeader(true), ShouldNotBeNil)
+ So(NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1).SetHeader(true), ShouldNotBeNil)
})
Convey("setting the header using an empty file should return EOF",
func() {
contents := ""
fields := []string{}
- csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
So(csvInputReader.SetHeader(true), ShouldEqual, io.EOF)
So(len(csvInputReader.Fields), ShouldEqual, 0)
})
@@ -250,7 +250,7 @@ func TestCSVSetHeader(t *testing.T) {
func() {
contents := "extraHeader1,extraHeader2,extraHeader3"
fields := []string{"a", "b", "c"}
- csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
So(csvInputReader.SetHeader(true), ShouldBeNil)
// if SetHeader() is called with fields already passed in,
// the header should be replaced with the read header line
@@ -274,7 +274,7 @@ func TestCSVSetHeader(t *testing.T) {
}
fileHandle, err := os.Open("testdata/test.csv")
So(err, ShouldBeNil)
- csvInputReader := NewCSVInputReader(fields, fileHandle)
+ csvInputReader := NewCSVInputReader(fields, fileHandle, 1)
errChan := make(chan error)
docChan := make(chan bson.D)
@@ -291,7 +291,7 @@ func TestCSVGetHeaders(t *testing.T) {
Convey("With a CSV input reader", t, func() {
Convey("getting the header should return any already set headers", func() {
fields := []string{"extraHeader1", "extraHeader2", "extraHeader3"}
- csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte{}))
+ csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte{}), 1)
So(csvInputReader.GetHeaders(), ShouldResemble, fields)
})
})
@@ -304,7 +304,7 @@ func TestCSVReadHeadersFromSource(t *testing.T) {
expectedHeaders := []string{"1", "2", "3"}
fileHandle, err := os.Open("testdata/test.csv")
So(err, ShouldBeNil)
- csvInputReader := NewCSVInputReader([]string{}, fileHandle)
+ csvInputReader := NewCSVInputReader([]string{}, fileHandle, 1)
headers, err := csvInputReader.ReadHeadersFromSource()
So(err, ShouldBeNil)
So(headers, ShouldResemble, expectedHeaders)
diff --git a/mongoimport/json.go b/mongoimport/json.go
index 0f95ab66136..4e44409da23 100644
--- a/mongoimport/json.go
+++ b/mongoimport/json.go
@@ -35,6 +35,10 @@ type JSONInputReader struct {
// separator. It is a reader consisting of the decoder's buffer and the
// underlying reader
separatorReader io.Reader
+
+ // numDecoders is the number of concurrent goroutines to allow to be used
+ // for decoding the input stream
+ numDecoders int
}
// JSONConvertibleDoc implements the ConvertibleDoc interface for JSON input
@@ -60,12 +64,13 @@ var (
// NewJSONInputReader creates a new JSONInputReader in array mode if specified,
// configured to read data to the given io.Reader
-func NewJSONInputReader(isArray bool, in io.Reader) *JSONInputReader {
+func NewJSONInputReader(isArray bool, in io.Reader, numDecoders int) *JSONInputReader {
return &JSONInputReader{
IsArray: isArray,
Decoder: json.NewDecoder(in),
readOpeningBracket: false,
bytesFromReader: make([]byte, 1),
+ numDecoders: numDecoders,
}
}
@@ -90,7 +95,7 @@ func (jsonInputReader *JSONInputReader) ReadHeadersFromSource() ([]string, error
// hits EOF or an error. If ordered is true, it streams the documents in which
// the documents are read
func (jsonInputReader *JSONInputReader) StreamDocument(ordered bool, readChan chan bson.D, errChan chan error) {
- rawChan := make(chan ConvertibleDoc, numDecodingWorkers)
+ rawChan := make(chan ConvertibleDoc, jsonInputReader.numDecoders)
var err error
go func() {
for {
@@ -116,7 +121,7 @@ func (jsonInputReader *JSONInputReader) StreamDocument(ordered bool, readChan ch
jsonInputReader.numProcessed++
}
}()
- streamDocuments(ordered, rawChan, readChan, errChan)
+ streamDocuments(ordered, jsonInputReader.numDecoders, rawChan, readChan, errChan)
}
// This is required to satisfy the ConvertibleDoc interface for JSON input. It
@@ -126,7 +131,7 @@ func (jsonConvertibleDoc JSONConvertibleDoc) Convert() (bson.D, error) {
if err != nil {
return nil, fmt.Errorf("error unmarshaling bytes on document #%v: %v", 1, err)
}
- log.Logf(log.DebugLow, "got line: %v", document)
+ log.Logf(log.DebugHigh, "got line: %v", document)
// TODO: perhaps move this to decode.go
bsonD, err := bsonutil.GetExtendedBsonD(document)
if err != nil {
diff --git a/mongoimport/json_test.go b/mongoimport/json_test.go
index 0d0cead462e..eaf15eef8a0 100644
--- a/mongoimport/json_test.go
+++ b/mongoimport/json_test.go
@@ -17,7 +17,7 @@ func TestJSONArrayStreamDocument(t *testing.T) {
Convey("an error should be thrown if a plain JSON document is supplied",
func() {
contents := `{"a": "ae"}`
- jsonInputReader := NewJSONInputReader(true, bytes.NewReader([]byte(contents)))
+ jsonInputReader := NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go jsonInputReader.StreamDocument(true, docChan, errChan)
@@ -28,7 +28,7 @@ func TestJSONArrayStreamDocument(t *testing.T) {
"error out",
func() {
contents := `{"a":3},{"b":4}]`
- jsonInputReader := NewJSONInputReader(true, bytes.NewReader([]byte(contents)))
+ jsonInputReader := NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go jsonInputReader.StreamDocument(true, docChan, errChan)
@@ -39,7 +39,7 @@ func TestJSONArrayStreamDocument(t *testing.T) {
"error out",
func() {
contents := `[{"a": "ae"}`
- jsonInputReader := NewJSONInputReader(true, bytes.NewReader([]byte(contents)))
+ jsonInputReader := NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go jsonInputReader.StreamDocument(true, docChan, errChan)
@@ -54,7 +54,7 @@ func TestJSONArrayStreamDocument(t *testing.T) {
func() {
fileHandle, err := os.Open("testdata/test_plain.json")
So(err, ShouldBeNil)
- jsonInputReader := NewJSONInputReader(true, fileHandle)
+ jsonInputReader := NewJSONInputReader(true, fileHandle, 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go jsonInputReader.StreamDocument(true, docChan, errChan)
@@ -77,7 +77,7 @@ func TestJSONArrayStreamDocument(t *testing.T) {
}
fileHandle, err := os.Open("testdata/test_array.json")
So(err, ShouldBeNil)
- jsonInputReader := NewJSONInputReader(true, fileHandle)
+ jsonInputReader := NewJSONInputReader(true, fileHandle, 1)
errChan := make(chan error)
docChan := make(chan bson.D)
// TODO check error
@@ -101,7 +101,7 @@ func TestJSONPlainStreamDocument(t *testing.T) {
func() {
contents := `{"a": "ae"}`
expectedRead := bson.D{bson.DocElem{"a", "ae"}}
- jsonInputReader := NewJSONInputReader(false, bytes.NewReader([]byte(contents)))
+ jsonInputReader := NewJSONInputReader(false, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go jsonInputReader.StreamDocument(true, docChan, errChan)
@@ -114,7 +114,7 @@ func TestJSONPlainStreamDocument(t *testing.T) {
contents := `{"a": "ae"}{"b": "dc"}`
expectedReadOne := bson.D{bson.DocElem{"a", "ae"}}
expectedReadTwo := bson.D{bson.DocElem{"b", "dc"}}
- jsonInputReader := NewJSONInputReader(false, bytes.NewReader([]byte(contents)))
+ jsonInputReader := NewJSONInputReader(false, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go jsonInputReader.StreamDocument(true, docChan, errChan)
@@ -127,7 +127,7 @@ func TestJSONPlainStreamDocument(t *testing.T) {
func() {
contents := `{"a": "ae", "b": 2.0}`
expectedRead := bson.D{bson.DocElem{"a", "ae"}, bson.DocElem{"b", 2.0}}
- jsonInputReader := NewJSONInputReader(false, bytes.NewReader([]byte(contents)))
+ jsonInputReader := NewJSONInputReader(false, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go jsonInputReader.StreamDocument(true, docChan, errChan)
@@ -137,7 +137,7 @@ func TestJSONPlainStreamDocument(t *testing.T) {
Convey("JSON arrays should return an error", func() {
contents := `[{"a": "ae", "b": 2.0}]`
- jsonInputReader := NewJSONInputReader(false, bytes.NewReader([]byte(contents)))
+ jsonInputReader := NewJSONInputReader(false, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go jsonInputReader.StreamDocument(true, docChan, errChan)
@@ -166,7 +166,7 @@ func TestJSONPlainStreamDocument(t *testing.T) {
}
fileHandle, err := os.Open("testdata/test_plain.json")
So(err, ShouldBeNil)
- jsonInputReader := NewJSONInputReader(false, fileHandle)
+ jsonInputReader := NewJSONInputReader(false, fileHandle, 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go jsonInputReader.StreamDocument(true, docChan, errChan)
@@ -192,7 +192,7 @@ func TestReadJSONArraySeparator(t *testing.T) {
Convey("reading a JSON array separator should consume [",
func() {
contents := `[{"a": "ae"}`
- jsonImporter := NewJSONInputReader(true, bytes.NewReader([]byte(contents)))
+ jsonImporter := NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1)
So(jsonImporter.readJSONArraySeparator(), ShouldBeNil)
// at this point it should have consumed all bytes up to `{`
So(jsonImporter.readJSONArraySeparator(), ShouldNotBeNil)
@@ -201,14 +201,14 @@ func TestReadJSONArraySeparator(t *testing.T) {
"corresponding opening bracket should error out ",
func() {
contents := `]`
- jsonImporter := NewJSONInputReader(true, bytes.NewReader([]byte(contents)))
+ jsonImporter := NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1)
So(jsonImporter.readJSONArraySeparator(), ShouldNotBeNil)
})
Convey("reading an opening JSON array separator without a "+
"corresponding closing bracket should error out ",
func() {
contents := `[`
- jsonImporter := NewJSONInputReader(true, bytes.NewReader([]byte(contents)))
+ jsonImporter := NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1)
So(jsonImporter.readJSONArraySeparator(), ShouldBeNil)
So(jsonImporter.readJSONArraySeparator(), ShouldNotBeNil)
})
@@ -216,7 +216,7 @@ func TestReadJSONArraySeparator(t *testing.T) {
"closing bracket should return EOF",
func() {
contents := `[]`
- jsonImporter := NewJSONInputReader(true, bytes.NewReader([]byte(contents)))
+ jsonImporter := NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1)
So(jsonImporter.readJSONArraySeparator(), ShouldBeNil)
So(jsonImporter.readJSONArraySeparator(), ShouldEqual, io.EOF)
})
@@ -224,7 +224,7 @@ func TestReadJSONArraySeparator(t *testing.T) {
"bracket but then additional characters after that, should error",
func() {
contents := `[]a`
- jsonImporter := NewJSONInputReader(true, bytes.NewReader([]byte(contents)))
+ jsonImporter := NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1)
So(jsonImporter.readJSONArraySeparator(), ShouldBeNil)
So(jsonImporter.readJSONArraySeparator(), ShouldNotBeNil)
})
@@ -232,7 +232,7 @@ func TestReadJSONArraySeparator(t *testing.T) {
"error out",
func() {
contents := `[{"a":3}x{"b":4}]`
- jsonInputReader := NewJSONInputReader(true, bytes.NewReader([]byte(contents)))
+ jsonInputReader := NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go jsonInputReader.StreamDocument(true, docChan, errChan)
@@ -244,7 +244,7 @@ func TestReadJSONArraySeparator(t *testing.T) {
"valid objects should error out",
func() {
contents := `[{"a":3},b{"b":4}]`
- jsonInputReader := NewJSONInputReader(true, bytes.NewReader([]byte(contents)))
+ jsonInputReader := NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go jsonInputReader.StreamDocument(true, docChan, errChan)
@@ -252,7 +252,7 @@ func TestReadJSONArraySeparator(t *testing.T) {
So(jsonInputReader.readJSONArraySeparator(), ShouldNotBeNil)
contents = `[{"a":3},,{"b":4}]`
- jsonInputReader = NewJSONInputReader(true, bytes.NewReader([]byte(contents)))
+ jsonInputReader = NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1)
errChan = make(chan error)
docChan = make(chan bson.D)
go jsonInputReader.StreamDocument(true, docChan, errChan)
diff --git a/mongoimport/main/mongoimport.go b/mongoimport/main/mongoimport.go
index 6b6fb316111..7309030b1bf 100644
--- a/mongoimport/main/mongoimport.go
+++ b/mongoimport/main/mongoimport.go
@@ -2,12 +2,12 @@ package main
import (
"fmt"
- "os"
"github.com/mongodb/mongo-tools/common/db"
"github.com/mongodb/mongo-tools/common/log"
commonopts "github.com/mongodb/mongo-tools/common/options"
"github.com/mongodb/mongo-tools/mongoimport"
"github.com/mongodb/mongo-tools/mongoimport/options"
+ "os"
)
func main() {
diff --git a/mongoimport/mongoimport.go b/mongoimport/mongoimport.go
index b3fd4b4ed66..c3a854c63d4 100644
--- a/mongoimport/mongoimport.go
+++ b/mongoimport/mongoimport.go
@@ -13,7 +13,6 @@ import (
"io"
"os"
"path/filepath"
- "runtime"
"strings"
"sync"
)
@@ -31,13 +30,6 @@ const (
maxMessageSizeBytes = 2 * maxBSONSize
)
-// variables used by the input/ingestion goroutines
-var (
- numDecodingWorkers = 1 // will be set to numCPUs at runtime
- numInsertionWorkers = 1
- batchSize = 10000
-)
-
// Wrapper for MongoImport functionality
type MongoImport struct {
// generic mongo tool options
@@ -95,16 +87,6 @@ type InputReader interface {
// ValidateSettings ensures that the tool specific options supplied for
// MongoImport are valid
func (mongoImport *MongoImport) ValidateSettings(args []string) error {
- if err := mongoImport.ToolOptions.Validate(); err != nil {
- return err
- }
-
- // TODO: move to common
- // --dbpath is now deprecated for tools with version >= v2.8
- if mongoImport.ToolOptions.DBPath != "" {
- return fmt.Errorf("--dbpath is now deprecated. start a mongod instead")
- }
-
// Namespace must have a valid database if none is specified,
// use 'test'
if mongoImport.ToolOptions.Namespace.DB == "" {
@@ -151,58 +133,27 @@ func (mongoImport *MongoImport) ValidateSettings(args []string) error {
}
}
- numCPU := runtime.NumCPU()
-
- // set the number of operating system threads to use for imports
- if mongoImport.IngestOptions.NumOSThreads == nil {
- runtime.GOMAXPROCS(numCPU)
- } else {
- if *mongoImport.IngestOptions.NumOSThreads < 1 {
- return fmt.Errorf("--numOSThreads argument must be > 0")
- }
- runtime.GOMAXPROCS(*mongoImport.IngestOptions.NumOSThreads)
- }
-
// set the number of decoding workers to use for imports
- if mongoImport.IngestOptions.NumDecodingWorkers != nil {
- if *mongoImport.IngestOptions.NumDecodingWorkers < 1 {
- return fmt.Errorf("--numDecodingWorkers argument must be > 0")
- }
- numDecodingWorkers = *mongoImport.IngestOptions.NumDecodingWorkers
- } else {
- mongoImport.IngestOptions.NumDecodingWorkers = &numCPU
- numDecodingWorkers = numCPU
+ if mongoImport.ToolOptions.NumDecodingWorkers <= 0 {
+ mongoImport.ToolOptions.NumDecodingWorkers = mongoImport.ToolOptions.MaxProcs
}
+ log.Logf(log.DebugLow, "Using %v decoding workers", mongoImport.ToolOptions.NumDecodingWorkers)
// set the number of insertion workers to use for imports
- if mongoImport.IngestOptions.NumInsertionWorkers != nil {
- if *mongoImport.IngestOptions.NumInsertionWorkers < 1 {
- return fmt.Errorf("--numInsertionThreads argument must be > 0")
- }
- numInsertionWorkers = *mongoImport.IngestOptions.NumInsertionWorkers
- } else {
- mongoImport.IngestOptions.NumInsertionWorkers = &numInsertionWorkers
+ if mongoImport.ToolOptions.NumInsertionWorkers <= 0 {
+ mongoImport.ToolOptions.NumInsertionWorkers = 1
}
- // if maintain --maintainInsertionOrder is true, we can only have one
- // insertion worker
+ log.Logf(log.DebugLow, "Using %v insert workers", mongoImport.ToolOptions.NumInsertionWorkers)
+
+ // if --maintainInsertionOrder is set, we can only allow 1 insertion worker
if mongoImport.IngestOptions.MaintainInsertionOrder {
- if numInsertionWorkers > 1 {
- return fmt.Errorf("cannot specify --maintainInsertionOrder with more than 1 insertionWorker")
- }
- mongoImport.IngestOptions.NumInsertionWorkers = &numInsertionWorkers
+ mongoImport.ToolOptions.NumInsertionWorkers = 1
}
// get the number of documents per batch
- if mongoImport.IngestOptions.BatchSize != nil {
- if *mongoImport.IngestOptions.BatchSize < 1 {
- return fmt.Errorf("--batchSize argument must be > 0")
- }
- batchSize = *mongoImport.IngestOptions.BatchSize
- } else {
- // TODO: TOOLS-335 replace use of global variables - batch size,
- // numInsertionWorkers and numDecodingWorkers
- mongoImport.IngestOptions.BatchSize = &batchSize
+ if mongoImport.ToolOptions.BulkBufferSize <= 0 {
+ mongoImport.ToolOptions.BulkBufferSize = 10000
}
// ensure no more than one positional argument is supplied
@@ -350,7 +301,10 @@ func (mongoImport *MongoImport) importDocuments(inputReader InputReader) (numImp
ordered := mongoImport.IngestOptions.MaintainInsertionOrder
// set the batch size for ingestion
- readDocChanSize := batchSize * numDecodingWorkers
+ readDocChanSize := mongoImport.ToolOptions.BulkBufferSize * mongoImport.ToolOptions.NumDecodingWorkers
+ if readDocChanSize == 0 {
+ readDocChanSize = 1
+ }
// readDocChan is buffered with readDocChanSize to ensure we only block
// accepting reads if processing is slow
@@ -380,19 +334,22 @@ func (mongoImport *MongoImport) importDocuments(inputReader InputReader) (numImp
// IngestDocuments takes a slice of documents and either inserts/upserts them -
// based on whether an upsert is requested - into the given collection
func (mongoImport *MongoImport) IngestDocuments(readChan chan bson.D) (err error) {
- numDecodingWorkers := *mongoImport.IngestOptions.NumInsertionWorkers
-
// initialize the tomb where all goroutines go to die
mongoImport.tomb = &tomb.Tomb{}
+ numInsertionWorkers := mongoImport.ToolOptions.NumInsertionWorkers
+ if numInsertionWorkers <= 0 {
+ numInsertionWorkers = 1
+ }
+
// spawn all the worker goroutines, each in its own goroutine
- for i := 0; i < numDecodingWorkers; i++ {
+ for i := 0; i < numInsertionWorkers; i++ {
mongoImport.tomb.Go(func() error {
// Each ingest worker will return an error which may
// be nil or not. It will be not nil in any of this cases:
//
// 1. There is a problem connecting with the server
- // 2. There server becomes unreachable
+ // 2. The server becomes unreachable
// 3. There is an insertion/update error - e.g. duplicate key
// error - and stopOnError is set to true
return mongoImport.ingestDocs(readChan)
@@ -450,7 +407,7 @@ readLoop:
// limit so we self impose a limit by using maxMessageSizeBytes
// and send documents over the wire when we hit the batch size
// or when we're at/over the maximum message size threshold
- if len(documents) == batchSize || numMessageBytes >= maxMessageSizeBytes {
+ if len(documents) == mongoImport.ToolOptions.BulkBufferSize || numMessageBytes >= maxMessageSizeBytes {
if err = mongoImport.ingester(documents, collection); err != nil {
return err
}
@@ -528,6 +485,9 @@ func (mongoImport *MongoImport) ingester(documents []bson.Raw, collection *mgo.C
numInserted, err = mongoImport.handleUpsert(documents, collection)
return err
} else {
+ if len(documents) == 0 {
+ return
+ }
bulk := collection.Bulk()
for _, document := range documents {
bulk.Insert(document)
@@ -569,9 +529,9 @@ func (mongoImport *MongoImport) getInputReader(in io.Reader) (InputReader, error
}
}
if mongoImport.InputOptions.Type == CSV {
- return NewCSVInputReader(fields, in), nil
+ return NewCSVInputReader(fields, in, mongoImport.ToolOptions.NumDecodingWorkers), nil
} else if mongoImport.InputOptions.Type == TSV {
- return NewTSVInputReader(fields, in), nil
+ return NewTSVInputReader(fields, in, mongoImport.ToolOptions.NumDecodingWorkers), nil
}
- return NewJSONInputReader(mongoImport.InputOptions.JSONArray, in), nil
+ return NewJSONInputReader(mongoImport.InputOptions.JSONArray, in, mongoImport.ToolOptions.NumDecodingWorkers), nil
}
diff --git a/mongoimport/mongoimport_test.go b/mongoimport/mongoimport_test.go
index 3d6fcc22bf1..4c18dc1d379 100644
--- a/mongoimport/mongoimport_test.go
+++ b/mongoimport/mongoimport_test.go
@@ -15,24 +15,21 @@ import (
"testing"
)
-var (
- testDB = "db"
- testCollection = "c"
- testServer = "localhost"
- testPort = "27017"
- sessionProvider *db.SessionProvider
+const (
+ testDb = "db"
+ testCollection = "c"
)
// checkOnlyHasDocuments returns an error if the documents in the test
// collection don't exactly match those that are passed in
-func checkOnlyHasDocuments(expectedDocuments []bson.M) error {
+func checkOnlyHasDocuments(sessionProvider db.SessionProvider, expectedDocuments []bson.M) error {
session, err := sessionProvider.GetSession()
if err != nil {
return err
}
defer session.Close()
- collection := session.DB(testDB).C(testCollection)
+ collection := session.DB(testDb).C(testCollection)
dbDocuments := []bson.M{}
err = collection.Find(nil).Sort("_id").All(&dbDocuments)
if err != nil {
@@ -57,18 +54,31 @@ func getBasicToolOptions() *commonOpts.ToolOptions {
ssl := testutil.GetSSLOptions()
auth := testutil.GetAuthOptions()
namespace := &commonOpts.Namespace{
- DB: testDB,
+ DB: testDb,
Collection: testCollection,
}
connection := &commonOpts.Connection{
- Host: testServer,
- Port: testPort,
+ Host: "localhost",
+ Port: "27017",
}
return &commonOpts.ToolOptions{
- SSL: &ssl,
- Namespace: namespace,
- Connection: connection,
- Auth: &auth,
+ SSL: &ssl,
+ Namespace: namespace,
+ Connection: connection,
+ HiddenOptions: &commonOpts.HiddenOptions{},
+ Auth: &auth,
+ }
+}
+
+func NewMongoImport() *MongoImport {
+ toolOptions := getBasicToolOptions()
+ inputOptions := &options.InputOptions{}
+ ingestOptions := &options.IngestOptions{}
+ return &MongoImport{
+ ToolOptions: toolOptions,
+ InputOptions: inputOptions,
+ IngestOptions: ingestOptions,
+ SessionProvider: db.NewSessionProvider(*toolOptions),
}
}
@@ -76,198 +86,77 @@ func TestMongoImportValidateSettings(t *testing.T) {
testutil.VerifyTestType(t, testutil.UNIT_TEST_TYPE)
Convey("Given a mongoimport instance for validation, ", t, func() {
- Convey("an error should be thrown if no database is given", func() {
- namespace := &commonOpts.Namespace{}
- toolOptions := &commonOpts.ToolOptions{
- Namespace: namespace,
- }
- inputOptions := &options.InputOptions{
- Type: CSV,
- }
- ingestOptions := &options.IngestOptions{}
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- }
+ Convey("an error should be thrown if no collection is given", func() {
+ mongoImport := NewMongoImport()
+ mongoImport.ToolOptions.Namespace.DB = ""
+ mongoImport.ToolOptions.Namespace.Collection = ""
So(mongoImport.ValidateSettings([]string{}), ShouldNotBeNil)
})
Convey("an error should be thrown if an invalid type is given", func() {
- namespace := &commonOpts.Namespace{
- DB: testDB,
- Collection: testCollection,
- }
- toolOptions := &commonOpts.ToolOptions{
- Namespace: namespace,
- }
- inputOptions := &options.InputOptions{
- Type: "invalid",
- }
- ingestOptions := &options.IngestOptions{}
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- }
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.Type = "invalid"
So(mongoImport.ValidateSettings([]string{}), ShouldNotBeNil)
})
Convey("an error should be thrown if neither --headerline is supplied "+
"nor --fields/--fieldFile", func() {
- namespace := &commonOpts.Namespace{
- DB: testDB,
- Collection: testCollection,
- }
- toolOptions := &commonOpts.ToolOptions{
- Namespace: namespace,
- }
- inputOptions := &options.InputOptions{
- Type: CSV,
- }
- ingestOptions := &options.IngestOptions{}
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- }
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.Type = CSV
So(mongoImport.ValidateSettings([]string{}), ShouldNotBeNil)
})
Convey("no error should be thrown if --headerline is not supplied "+
"but --fields is supplied", func() {
- namespace := &commonOpts.Namespace{
- DB: testDB,
- Collection: testCollection,
- }
- toolOptions := &commonOpts.ToolOptions{
- Namespace: namespace,
- }
- inputOptions := &options.InputOptions{
- Fields: "a,b,c",
- Type: CSV,
- }
- ingestOptions := &options.IngestOptions{}
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- }
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.Fields = "a,b,c"
+ mongoImport.InputOptions.Type = CSV
So(mongoImport.ValidateSettings([]string{}), ShouldBeNil)
})
- Convey("no error should be thrown if no input type is supplied",
- func() {
- namespace := &commonOpts.Namespace{
- DB: testDB,
- Collection: testCollection,
- }
- toolOptions := &commonOpts.ToolOptions{
- Namespace: namespace,
- }
- inputOptions := &options.InputOptions{
- Fields: "a,b,c",
- }
- ingestOptions := &options.IngestOptions{}
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- }
- So(mongoImport.ValidateSettings([]string{}), ShouldBeNil)
- })
+ Convey("no error should be thrown if no input type is supplied", func() {
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.Fields = "a,b,c"
+ So(mongoImport.ValidateSettings([]string{}), ShouldBeNil)
+ })
Convey("no error should be thrown if --headerline is not supplied "+
"but --fieldFile is supplied", func() {
- namespace := &commonOpts.Namespace{
- DB: testDB,
- Collection: testCollection,
- }
- toolOptions := &commonOpts.ToolOptions{
- Namespace: namespace,
- }
- inputOptions := &options.InputOptions{
- FieldFile: "test.csv",
- Type: CSV,
- }
- ingestOptions := &options.IngestOptions{}
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- }
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.FieldFile = "test.csv"
+ mongoImport.InputOptions.Type = CSV
So(mongoImport.ValidateSettings([]string{}), ShouldBeNil)
})
- Convey("an error should be thrown if no collection and no file is "+
- "supplied", func() {
- namespace := &commonOpts.Namespace{
- DB: testDB,
- }
- toolOptions := &commonOpts.ToolOptions{
- Namespace: namespace,
- }
- inputOptions := &options.InputOptions{
- FieldFile: "test.csv",
- Type: CSV,
- }
- ingestOptions := &options.IngestOptions{}
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- }
+ Convey("an error should be thrown if no collection and no file is supplied", func() {
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.FieldFile = "test.csv"
+ mongoImport.InputOptions.Type = CSV
+ mongoImport.ToolOptions.Namespace.Collection = ""
So(mongoImport.ValidateSettings([]string{}), ShouldNotBeNil)
})
Convey("no error should be thrown if no collection but a file is "+
- "supplied and the file name should be used as the collection "+
- "name", func() {
- namespace := &commonOpts.Namespace{
- DB: testDB,
- }
- toolOptions := &commonOpts.ToolOptions{
- Namespace: namespace,
- }
- inputOptions := &options.InputOptions{
- File: "input",
- FieldFile: "test.csv",
- Type: CSV,
- }
- ingestOptions := &options.IngestOptions{}
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- }
+ "supplied and the file name should be used as the collection name", func() {
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.File = "input"
+ mongoImport.InputOptions.FieldFile = "test.csv"
+ mongoImport.InputOptions.Type = CSV
+ mongoImport.ToolOptions.Namespace.Collection = ""
So(mongoImport.ValidateSettings([]string{}), ShouldBeNil)
So(mongoImport.ToolOptions.Namespace.Collection, ShouldEqual,
mongoImport.InputOptions.File)
})
Convey("with no collection name and a file name the base name of the "+
- "file (without the extension) should be used as the collection "+
- "name", func() {
- namespace := &commonOpts.Namespace{
- DB: testDB,
- }
- toolOptions := &commonOpts.ToolOptions{
- Namespace: namespace,
- }
- inputOptions := &options.InputOptions{
- File: "/path/to/input/file/dot/input.txt",
- FieldFile: "test.csv",
- Type: CSV,
- }
- ingestOptions := &options.IngestOptions{}
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- }
+ "file (without the extension) should be used as the collection name", func() {
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.File = "/path/to/input/file/dot/input.txt"
+ mongoImport.InputOptions.FieldFile = "test.csv"
+ mongoImport.InputOptions.Type = CSV
+ mongoImport.ToolOptions.Namespace.Collection = ""
So(mongoImport.ValidateSettings([]string{}), ShouldBeNil)
- So(mongoImport.ToolOptions.Namespace.Collection, ShouldEqual,
- "input")
+ So(mongoImport.ToolOptions.Namespace.Collection, ShouldEqual, "input")
})
})
}
@@ -278,34 +167,25 @@ func TestGetSourceReader(t *testing.T) {
func() {
Convey("an error should be thrown if the given file referenced by "+
"the reader does not exist", func() {
- inputOptions := &options.InputOptions{
- File: "/path/to/input/file/dot/input.txt",
- }
- mongoImport := MongoImport{
- InputOptions: inputOptions,
- }
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.File = "/path/to/input/file/dot/input.txt"
+ mongoImport.InputOptions.Type = CSV
+ mongoImport.ToolOptions.Namespace.Collection = ""
_, err := mongoImport.getSourceReader()
So(err, ShouldNotBeNil)
})
Convey("no error should be thrown if the file exists", func() {
- inputOptions := &options.InputOptions{
- File: "testdata/test_array.json",
- }
- mongoImport := MongoImport{
- InputOptions: inputOptions,
- }
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.File = "testdata/test_array.json"
+ mongoImport.InputOptions.Type = JSON
_, err := mongoImport.getSourceReader()
So(err, ShouldBeNil)
})
Convey("no error should be thrown if stdin is used", func() {
- inputOptions := &options.InputOptions{
- File: "",
- }
- mongoImport := MongoImport{
- InputOptions: inputOptions,
- }
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.File = ""
_, err := mongoImport.getSourceReader()
So(err, ShouldBeNil)
})
@@ -317,75 +197,47 @@ func TestGetInputReader(t *testing.T) {
Convey("Given a io.Reader on calling getInputReader", t, func() {
Convey("no error should be thrown if neither --fields nor --fieldFile "+
"is used", func() {
- inputOptions := &options.InputOptions{
- File: "/path/to/input/file/dot/input.txt",
- }
- mongoImport := MongoImport{
- InputOptions: inputOptions,
- }
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.File = "/path/to/input/file/dot/input.txt"
_, err := mongoImport.getInputReader(&os.File{})
So(err, ShouldBeNil)
})
Convey("no error should be thrown if --fields is used", func() {
- inputOptions := &options.InputOptions{
- Fields: "a,b,c",
- File: "/path/to/input/file/dot/input.txt",
- }
- mongoImport := MongoImport{
- InputOptions: inputOptions,
- }
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.Fields = "a,b,c"
+ mongoImport.InputOptions.File = "/path/to/input/file/dot/input.txt"
_, err := mongoImport.getInputReader(&os.File{})
So(err, ShouldBeNil)
})
Convey("no error should be thrown if --fieldFile is used and it "+
"references a valid file", func() {
- inputOptions := &options.InputOptions{
- FieldFile: "testdata/test_array.json",
- }
- mongoImport := MongoImport{
- InputOptions: inputOptions,
- }
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.FieldFile = "testdata/test_array.json"
_, err := mongoImport.getInputReader(&os.File{})
So(err, ShouldBeNil)
})
Convey("an error should be thrown if --fieldFile is used and it "+
"references an invalid file", func() {
- inputOptions := &options.InputOptions{
- FieldFile: "/path/to/input/file/dot/input.txt",
- }
- mongoImport := MongoImport{
- InputOptions: inputOptions,
- }
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.FieldFile = "/path/to/input/file/dot/input.txt"
_, err := mongoImport.getInputReader(&os.File{})
So(err, ShouldNotBeNil)
})
Convey("no error should be thrown for CSV import inputs", func() {
- inputOptions := &options.InputOptions{
- Type: CSV,
- }
- mongoImport := MongoImport{
- InputOptions: inputOptions,
- }
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.Type = CSV
_, err := mongoImport.getInputReader(&os.File{})
So(err, ShouldBeNil)
})
Convey("no error should be thrown for TSV import inputs", func() {
- inputOptions := &options.InputOptions{
- Type: TSV,
- }
- mongoImport := MongoImport{
- InputOptions: inputOptions,
- }
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.Type = TSV
_, err := mongoImport.getInputReader(&os.File{})
So(err, ShouldBeNil)
})
Convey("no error should be thrown for JSON import inputs", func() {
- inputOptions := &options.InputOptions{
- Type: JSON,
- }
- mongoImport := MongoImport{
- InputOptions: inputOptions,
- }
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.Type = JSON
_, err := mongoImport.getInputReader(&os.File{})
So(err, ShouldBeNil)
})
@@ -394,151 +246,76 @@ func TestGetInputReader(t *testing.T) {
func TestImportDocuments(t *testing.T) {
testutil.VerifyTestType(t, testutil.INTEGRATION_TEST_TYPE)
- var err error
- Convey("Given a mongoimport instance with which to import documents, on "+
- "calling importDocuments", t, func() {
- batchSize := 1
- NumDecodingWorkers := 1
- NumInsertionWorkers := 1
+ Convey("With a mongoimport instance", t, func() {
+ Reset(func() {
+ sessionProvider := db.NewSessionProvider(*getBasicToolOptions())
+ session, err := sessionProvider.GetSession()
+ if err != nil {
+ t.Fatalf("error getting session: %v", err)
+ }
+ defer session.Close()
+ session.DB(testDb).C(testCollection).DropCollection()
+ fmt.Println("droping", testDb, testCollection)
+ })
Convey("no error should be thrown for CSV import on test data and all "+
"CSV data lines should be imported correctly", func() {
- toolOptions := getBasicToolOptions()
- inputOptions := &options.InputOptions{
- Type: CSV,
- File: "testdata/test.csv",
- Fields: "a,b,c",
- }
- ingestOptions := &options.IngestOptions{
- BatchSize: &batchSize,
- NumDecodingWorkers: &NumDecodingWorkers,
- NumInsertionWorkers: &NumInsertionWorkers,
- }
- sessionProvider, err = db.InitSessionProvider(*toolOptions)
- So(err, ShouldBeNil)
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- SessionProvider: sessionProvider,
- }
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.Type = CSV
+ mongoImport.InputOptions.File = "testdata/test.csv"
+ mongoImport.InputOptions.Fields = "a,b,c"
+ mongoImport.IngestOptions.WriteConcern = "majority"
numImported, err := mongoImport.ImportDocuments()
So(err, ShouldBeNil)
So(numImported, ShouldEqual, 3)
})
Convey("TOOLS-247: no error should be thrown for JSON import on test "+
"data and all documents should be imported correctly", func() {
- toolOptions := getBasicToolOptions()
- inputOptions := &options.InputOptions{
- File: "testdata/test_plain2.json",
- }
- ingestOptions := &options.IngestOptions{
- IgnoreBlanks: true,
- BatchSize: &batchSize,
- NumDecodingWorkers: &NumDecodingWorkers,
- NumInsertionWorkers: &NumInsertionWorkers,
- }
- sessionProvider, err = db.InitSessionProvider(*toolOptions)
- So(err, ShouldBeNil)
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- SessionProvider: sessionProvider,
- }
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.File = "testdata/test_plain2.json"
+ mongoImport.IngestOptions.WriteConcern = "majority"
numImported, err := mongoImport.ImportDocuments()
So(err, ShouldBeNil)
So(numImported, ShouldEqual, 10)
})
- Convey("no error should be thrown for CSV import on test data with "+
- "--ignoreBlanks only fields without blanks should be imported",
- func() {
- toolOptions := getBasicToolOptions()
- inputOptions := &options.InputOptions{
- Type: CSV,
- File: "testdata/test_blanks.csv",
- Fields: "_id,b,c",
- }
- ingestOptions := &options.IngestOptions{
- IgnoreBlanks: true,
- BatchSize: &batchSize,
- NumDecodingWorkers: &NumDecodingWorkers,
- NumInsertionWorkers: &NumInsertionWorkers,
- MaintainInsertionOrder: true,
- }
- sessionProvider, err = db.InitSessionProvider(*toolOptions)
- So(err, ShouldBeNil)
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- SessionProvider: sessionProvider,
- }
- numImported, err := mongoImport.ImportDocuments()
- So(err, ShouldBeNil)
- So(numImported, ShouldEqual, 3)
- expectedDocuments := []bson.M{
- bson.M{"_id": 1, "b": 2},
- bson.M{"_id": 5, "c": "6e"},
- bson.M{"_id": 7, "b": 8, "c": 6},
- }
- So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
- })
- Convey("no error should be thrown for CSV import on test data without "+
- "--ignoreBlanks supplied - fields with blanks should be imported",
- func() {
- toolOptions := getBasicToolOptions()
- inputOptions := &options.InputOptions{
- Type: CSV,
- File: "testdata/test_blanks.csv",
- Fields: "_id,b,c",
- }
- ingestOptions := &options.IngestOptions{
- BatchSize: &batchSize,
- NumDecodingWorkers: &NumDecodingWorkers,
- NumInsertionWorkers: &NumInsertionWorkers,
- MaintainInsertionOrder: true,
- }
- sessionProvider, err = db.InitSessionProvider(*toolOptions)
- So(err, ShouldBeNil)
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- SessionProvider: sessionProvider,
- }
- numImported, err := mongoImport.ImportDocuments()
- So(err, ShouldBeNil)
- So(numImported, ShouldEqual, 3)
- expectedDocuments := []bson.M{
- bson.M{"_id": 1, "b": 2, "c": ""},
- bson.M{"_id": 5, "b": "", "c": "6e"},
- bson.M{"_id": 7, "b": 8, "c": 6},
- }
- So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
- })
- Convey("no error should be thrown for CSV import on test data with "+
- "--upsert", func() {
- toolOptions := getBasicToolOptions()
- inputOptions := &options.InputOptions{
- Type: CSV,
- File: "testdata/test.csv",
- Fields: "_id,b,c",
- }
- ingestOptions := &options.IngestOptions{
- Upsert: true,
- BatchSize: &batchSize,
- NumDecodingWorkers: &NumDecodingWorkers,
- NumInsertionWorkers: &NumInsertionWorkers,
- MaintainInsertionOrder: true,
+ Convey("CSV import with --ignoreBlanks should import only non-blank fields", func() {
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.Type = CSV
+ mongoImport.InputOptions.File = "testdata/test_blanks.csv"
+ mongoImport.InputOptions.Fields = "_id,b,c"
+ mongoImport.IngestOptions.IgnoreBlanks = true
+
+ numImported, err := mongoImport.ImportDocuments()
+ So(err, ShouldBeNil)
+ So(numImported, ShouldEqual, 3)
+ expectedDocuments := []bson.M{
+ bson.M{"_id": 1, "b": 2},
+ bson.M{"_id": 5, "c": "6e"},
+ bson.M{"_id": 7, "b": 8, "c": 6},
}
- sessionProvider, err = db.InitSessionProvider(*toolOptions)
+ So(checkOnlyHasDocuments(*mongoImport.SessionProvider, expectedDocuments), ShouldBeNil)
+ })
+ Convey("CSV import without --ignoreBlanks should include blanks", func() {
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.Type = CSV
+ mongoImport.InputOptions.File = "testdata/test_blanks.csv"
+ mongoImport.InputOptions.Fields = "_id,b,c"
+ numImported, err := mongoImport.ImportDocuments()
So(err, ShouldBeNil)
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- SessionProvider: sessionProvider,
+ So(numImported, ShouldEqual, 3)
+ expectedDocuments := []bson.M{
+ bson.M{"_id": 1, "b": 2, "c": ""},
+ bson.M{"_id": 5, "b": "", "c": "6e"},
+ bson.M{"_id": 7, "b": 8, "c": 6},
}
+ So(checkOnlyHasDocuments(*mongoImport.SessionProvider, expectedDocuments), ShouldBeNil)
+ })
+ Convey("no error should be thrown for CSV import on test data with --upsert", func() {
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.Type = CSV
+ mongoImport.InputOptions.File = "testdata/test.csv"
+ mongoImport.InputOptions.Fields = "_id,b,c"
+ mongoImport.IngestOptions.Upsert = true
+ mongoImport.IngestOptions.MaintainInsertionOrder = true
numImported, err := mongoImport.ImportDocuments()
So(err, ShouldBeNil)
So(numImported, ShouldEqual, 3)
@@ -547,99 +324,55 @@ func TestImportDocuments(t *testing.T) {
bson.M{"_id": 3, "b": 5.4, "c": "string"},
bson.M{"_id": 5, "b": 6, "c": 6},
}
- So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
+ So(checkOnlyHasDocuments(*mongoImport.SessionProvider, expectedDocuments), ShouldBeNil)
})
Convey("no error should be thrown for CSV import on test data with "+
- "--stopOnError. Only documents before error should be imported",
- func() {
- toolOptions := getBasicToolOptions()
- inputOptions := &options.InputOptions{
- Type: CSV,
- File: "testdata/test.csv",
- Fields: "_id,b,c",
- }
- ingestOptions := &options.IngestOptions{
- StopOnError: true,
- BatchSize: &batchSize,
- NumDecodingWorkers: &NumDecodingWorkers,
- NumInsertionWorkers: &NumInsertionWorkers,
- MaintainInsertionOrder: true,
- }
- sessionProvider, err = db.InitSessionProvider(*toolOptions)
- So(err, ShouldBeNil)
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- SessionProvider: sessionProvider,
- }
- numImported, err := mongoImport.ImportDocuments()
- So(err, ShouldBeNil)
- So(numImported, ShouldEqual, 3)
- expectedDocuments := []bson.M{
- bson.M{"_id": 1, "b": 2, "c": 3},
- bson.M{"_id": 3, "b": 5.4, "c": "string"},
- bson.M{"_id": 5, "b": 6, "c": 6},
- }
- So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
- })
- Convey("no error should be thrown for CSV import on test data with "+
- "duplicate _id if --stopOnError is not set", func() {
- toolOptions := getBasicToolOptions()
- inputOptions := &options.InputOptions{
- Type: CSV,
- File: "testdata/test_duplicate.csv",
- Fields: "_id,b,c",
- }
- ingestOptions := &options.IngestOptions{
- BatchSize: &batchSize,
- NumDecodingWorkers: &NumDecodingWorkers,
- NumInsertionWorkers: &NumInsertionWorkers,
- }
- sessionProvider, err = db.InitSessionProvider(*toolOptions)
+ "--stopOnError. Only documents before error should be imported", func() {
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.Type = CSV
+ mongoImport.InputOptions.File = "testdata/test.csv"
+ mongoImport.InputOptions.Fields = "_id,b,c"
+ mongoImport.IngestOptions.StopOnError = true
+ mongoImport.IngestOptions.MaintainInsertionOrder = true
+ mongoImport.IngestOptions.WriteConcern = "majority"
+ numImported, err := mongoImport.ImportDocuments()
So(err, ShouldBeNil)
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- SessionProvider: sessionProvider,
+ So(numImported, ShouldEqual, 3)
+ expectedDocuments := []bson.M{
+ bson.M{"_id": 1, "b": 2, "c": 3},
+ bson.M{"_id": 3, "b": 5.4, "c": "string"},
+ bson.M{"_id": 5, "b": 6, "c": 6},
}
- _, err = mongoImport.ImportDocuments()
+ So(checkOnlyHasDocuments(*mongoImport.SessionProvider, expectedDocuments), ShouldBeNil)
+ })
+ Convey("CSV import with duplicate _id's should not error if --stopOnError is not set", func() {
+ mongoImport := NewMongoImport()
+
+ mongoImport.InputOptions.Type = CSV
+ mongoImport.InputOptions.File = "testdata/test_duplicate.csv"
+ mongoImport.InputOptions.Fields = "_id,b,c"
+ mongoImport.IngestOptions.StopOnError = false
+ numImported, err := mongoImport.ImportDocuments()
So(err, ShouldBeNil)
+ So(numImported, ShouldEqual, 5)
+
expectedDocuments := []bson.M{
bson.M{"_id": 1, "b": 2, "c": 3},
bson.M{"_id": 3, "b": 5.4, "c": "string"},
bson.M{"_id": 5, "b": 6, "c": 6},
bson.M{"_id": 8, "b": 6, "c": 6},
}
- // all documents - except for the one with a duplicate _id - should
- // be imported
- So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
+ // all docs except the one with duplicate _id - should be imported
+ So(checkOnlyHasDocuments(*mongoImport.SessionProvider, expectedDocuments), ShouldBeNil)
})
- Convey("no error should be thrown for CSV import on test data with "+
- "--drop", func() {
- toolOptions := getBasicToolOptions()
- inputOptions := &options.InputOptions{
- Type: CSV,
- File: "testdata/test.csv",
- Fields: "_id,b,c",
- }
- ingestOptions := &options.IngestOptions{
- Drop: true,
- BatchSize: &batchSize,
- NumDecodingWorkers: &NumDecodingWorkers,
- NumInsertionWorkers: &NumInsertionWorkers,
- MaintainInsertionOrder: true,
- WriteConcern: "majority",
- }
- sessionProvider, err = db.InitSessionProvider(*toolOptions)
- So(err, ShouldBeNil)
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- SessionProvider: sessionProvider,
- }
+ Convey("no error should be thrown for CSV import on test data with --drop", func() {
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.Type = CSV
+ mongoImport.InputOptions.File = "testdata/test.csv"
+ mongoImport.InputOptions.Fields = "_id,b,c"
+ mongoImport.IngestOptions.Drop = true
+ mongoImport.IngestOptions.MaintainInsertionOrder = true
+ mongoImport.IngestOptions.WriteConcern = "majority"
numImported, err := mongoImport.ImportDocuments()
So(err, ShouldBeNil)
So(numImported, ShouldEqual, 3)
@@ -648,85 +381,41 @@ func TestImportDocuments(t *testing.T) {
bson.M{"_id": 3, "b": 5.4, "c": "string"},
bson.M{"_id": 5, "b": 6, "c": 6},
}
- So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
+ So(checkOnlyHasDocuments(*mongoImport.SessionProvider, expectedDocuments), ShouldBeNil)
})
- Convey("no error should be thrown for CSV import on test data with "+
- "--headerLine", func() {
- toolOptions := getBasicToolOptions()
- inputOptions := &options.InputOptions{
- Type: CSV,
- File: "testdata/test.csv",
- HeaderLine: true,
- }
- ingestOptions := &options.IngestOptions{
- BatchSize: &batchSize,
- NumDecodingWorkers: &NumDecodingWorkers,
- NumInsertionWorkers: &NumInsertionWorkers,
- }
- sessionProvider, err = db.InitSessionProvider(*toolOptions)
- So(err, ShouldBeNil)
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- SessionProvider: sessionProvider,
- }
+ Convey("CSV import on test data with --headerLine should succeed", func() {
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.Type = CSV
+ mongoImport.InputOptions.File = "testdata/test.csv"
+ mongoImport.InputOptions.Fields = "_id,b,c"
+ mongoImport.InputOptions.HeaderLine = true
numImported, err := mongoImport.ImportDocuments()
So(err, ShouldBeNil)
So(numImported, ShouldEqual, 2)
})
- Convey("EOF should be thrown for CSV import on test data with "+
- "--headerLine if the input file is empty", func() {
- toolOptions := getBasicToolOptions()
+ Convey("EOF should be thrown for CSV import with --headerLine if file is empty", func() {
csvFile, err := ioutil.TempFile("", "mongoimport_")
So(err, ShouldBeNil)
csvFile.Close()
- inputOptions := &options.InputOptions{
- Type: CSV,
- File: csvFile.Name(),
- HeaderLine: true,
- }
- ingestOptions := &options.IngestOptions{
- BatchSize: &batchSize,
- NumDecodingWorkers: &NumDecodingWorkers,
- NumInsertionWorkers: &NumInsertionWorkers,
- }
- sessionProvider, err = db.InitSessionProvider(*toolOptions)
- So(err, ShouldBeNil)
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- SessionProvider: sessionProvider,
- }
+
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.Type = CSV
+ mongoImport.InputOptions.File = csvFile.Name()
+ mongoImport.InputOptions.Fields = "_id,b,c"
+ mongoImport.InputOptions.HeaderLine = true
numImported, err := mongoImport.ImportDocuments()
So(err, ShouldEqual, io.EOF)
So(numImported, ShouldEqual, 0)
})
- Convey("no error should be thrown for CSV import on test data with "+
- "--upsert and --upsertFields", func() {
- toolOptions := getBasicToolOptions()
- inputOptions := &options.InputOptions{
- Type: CSV,
- File: "testdata/test.csv",
- Fields: "_id,c,b",
- }
- ingestOptions := &options.IngestOptions{
- Upsert: true,
- UpsertFields: "_id",
- BatchSize: &batchSize,
- NumDecodingWorkers: &NumDecodingWorkers,
- NumInsertionWorkers: &NumInsertionWorkers,
- MaintainInsertionOrder: true,
- }
- sessionProvider, err = db.InitSessionProvider(*toolOptions)
- So(err, ShouldBeNil)
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- SessionProvider: sessionProvider,
- }
+ Convey("CSV import with --upsert and --upsertFields should succeed", func() {
+ mongoImport := NewMongoImport()
+
+ mongoImport.InputOptions.Type = CSV
+ mongoImport.InputOptions.File = "testdata/test.csv"
+ mongoImport.InputOptions.Fields = "_id,c,b"
+ mongoImport.IngestOptions.Upsert = true
+ mongoImport.IngestOptions.UpsertFields = "_id"
+ mongoImport.IngestOptions.MaintainInsertionOrder = true
numImported, err := mongoImport.ImportDocuments()
So(err, ShouldBeNil)
So(numImported, ShouldEqual, 3)
@@ -735,33 +424,16 @@ func TestImportDocuments(t *testing.T) {
bson.M{"_id": 3, "c": 5.4, "b": "string"},
bson.M{"_id": 5, "c": 6, "b": 6},
}
- So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
+ So(checkOnlyHasDocuments(*mongoImport.SessionProvider, expectedDocuments), ShouldBeNil)
})
- Convey("no error should be thrown for CSV import on test data with "+
- "--upsert and --upsertFields and duplicate _id if --stopOnError "+
- "is not set", func() {
- inputOptions := &options.InputOptions{
- Type: CSV,
- File: "testdata/test_duplicate.csv",
- Fields: "_id,b,c",
- }
- ingestOptions := &options.IngestOptions{
- Upsert: true,
- UpsertFields: "_id",
- BatchSize: &batchSize,
- NumDecodingWorkers: &NumDecodingWorkers,
- NumInsertionWorkers: &NumInsertionWorkers,
- MaintainInsertionOrder: true,
- }
- toolOptions := getBasicToolOptions()
- sessionProvider, err = db.InitSessionProvider(*toolOptions)
- So(err, ShouldBeNil)
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- SessionProvider: sessionProvider,
- }
+ Convey("CSV import with --upsert/--upsertFields with duplicate id should succeed"+
+ " if stopOnError is not set", func() {
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.Type = CSV
+ mongoImport.InputOptions.File = "testdata/test_duplicate.csv"
+ mongoImport.InputOptions.Fields = "_id,b,c"
+ mongoImport.IngestOptions.Upsert = true
+ mongoImport.IngestOptions.UpsertFields = "_id"
numImported, err := mongoImport.ImportDocuments()
So(err, ShouldBeNil)
So(numImported, ShouldEqual, 5)
@@ -771,78 +443,41 @@ func TestImportDocuments(t *testing.T) {
bson.M{"_id": 5, "b": 6, "c": 9},
bson.M{"_id": 8, "b": 6, "c": 6},
}
- So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
+ So(checkOnlyHasDocuments(*mongoImport.SessionProvider, expectedDocuments), ShouldBeNil)
})
Convey("an error should be thrown for CSV import on test data with "+
"duplicate _id if --stopOnError is set", func() {
- toolOptions := getBasicToolOptions()
- inputOptions := &options.InputOptions{
- Type: CSV,
- File: "testdata/test_duplicate.csv",
- Fields: "_id,b,c",
- }
- ingestOptions := &options.IngestOptions{
- StopOnError: true,
- BatchSize: &batchSize,
- NumDecodingWorkers: &NumDecodingWorkers,
- NumInsertionWorkers: &NumInsertionWorkers,
- WriteConcern: "1",
- MaintainInsertionOrder: true,
- }
- sessionProvider, err = db.InitSessionProvider(*toolOptions)
- So(err, ShouldBeNil)
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- SessionProvider: sessionProvider,
- }
- _, err = mongoImport.ImportDocuments()
+
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.Type = CSV
+ mongoImport.InputOptions.File = "testdata/test_duplicate.csv"
+ mongoImport.InputOptions.Fields = "_id,b,c"
+ mongoImport.IngestOptions.StopOnError = true
+ mongoImport.IngestOptions.WriteConcern = "1"
+ mongoImport.IngestOptions.MaintainInsertionOrder = true
+ _, err := mongoImport.ImportDocuments()
So(err, ShouldNotBeNil)
expectedDocuments := []bson.M{
bson.M{"_id": 1, "b": 2, "c": 3},
bson.M{"_id": 3, "b": 5.4, "c": "string"},
bson.M{"_id": 5, "b": 6, "c": 6},
}
- So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
- })
- Convey("an error should be thrown for invalid CSV import on test data",
- func() {
- inputOptions := &options.InputOptions{
- Type: CSV,
- File: "testdata/test_bad.csv",
- Fields: "_id,b,c",
- }
- toolOptions := getBasicToolOptions()
- ingestOptions := &options.IngestOptions{
- StopOnError: true,
- BatchSize: &batchSize,
- NumDecodingWorkers: &NumDecodingWorkers,
- NumInsertionWorkers: &NumInsertionWorkers,
- MaintainInsertionOrder: true,
- }
- sessionProvider, err = db.InitSessionProvider(*toolOptions)
- So(err, ShouldBeNil)
- mongoImport := MongoImport{
- ToolOptions: toolOptions,
- InputOptions: inputOptions,
- IngestOptions: ingestOptions,
- SessionProvider: sessionProvider,
- }
- _, err = mongoImport.ImportDocuments()
- So(err, ShouldNotBeNil)
- expectedDocuments := []bson.M{
- bson.M{"_id": 1, "b": 2, "c": 3},
- }
- So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil)
- })
- Reset(func() {
- session, err := sessionProvider.GetSession()
- if err != nil {
- t.Fatalf("error getting session: %v", err)
+ So(checkOnlyHasDocuments(*mongoImport.SessionProvider, expectedDocuments), ShouldBeNil)
+ })
+ Convey("an error should be thrown for invalid CSV import on test data", func() {
+ mongoImport := NewMongoImport()
+ mongoImport.InputOptions.Type = CSV
+ mongoImport.InputOptions.File = "testdata/test_bad.csv"
+ mongoImport.InputOptions.Fields = "_id,b,c"
+ mongoImport.IngestOptions.StopOnError = true
+ mongoImport.IngestOptions.WriteConcern = "1"
+ mongoImport.IngestOptions.MaintainInsertionOrder = true
+ _, err := mongoImport.ImportDocuments()
+ So(err, ShouldNotBeNil)
+ expectedDocuments := []bson.M{
+ bson.M{"_id": 1, "b": 2, "c": 3},
}
- defer session.Close()
- session.DB(testDB).C(testCollection).DropCollection()
+ So(checkOnlyHasDocuments(*mongoImport.SessionProvider, expectedDocuments), ShouldBeNil)
})
})
}
diff --git a/mongoimport/options/options.go b/mongoimport/options/options.go
index 0d883134e0d..f201b55aced 100644
--- a/mongoimport/options/options.go
+++ b/mongoimport/options/options.go
@@ -56,19 +56,6 @@ type IngestOptions struct {
// Specifies the number of operating system threads to use during the import process
MaintainInsertionOrder bool `long:"maintainInsertionOrder" description:"if given, documents should be inserted in the order of their appearance in the input source"`
- // Specifies the number of operating system threads to use during the import process
- // TODO: hide this option
- NumOSThreads *int `long:"numOsThreads" description:"number of operating system threads to use (defaults to the number of logical CPUs)"`
-
- // Specifies the number of threads to use in processing data read from the input source
- NumDecodingWorkers *int `long:"numDecodingWorkers" description:"number of goroutines to use for converting JSON to BSON (defaults to the number of logical CPUs)"`
-
- // Specifies the number of threads to use in sending processed data over to the server
- NumInsertionWorkers *int `long:"numInsertionWorkers" description:"number of goroutines to use in ingesting data (defaults to 1)"`
-
- // Specifies the maximum number of documents in each batch sent over to the server
- BatchSize *int `long:"batchSize" description:"number of documents to insert in a single batch"`
-
// Specifies the write concern for each write operation that mongoimport writes to the target database.
// By default, mongoimport waits for a majority of members from the replica set to respond before returning.
WriteConcern string `long:"writeConcern" default:"majority" description:"write concern options e.g. --writeConcern majority, --writeConcern '{w: 3, wtimeout: 500, fsync: true, j: true}'"`
diff --git a/mongoimport/tsv.go b/mongoimport/tsv.go
index 5309eb3913a..182ab904405 100644
--- a/mongoimport/tsv.go
+++ b/mongoimport/tsv.go
@@ -23,9 +23,11 @@ type TSVInputReader struct {
tsvReader *bufio.Reader
// tsvRecord stores each line of input we read from the underlying reader
tsvRecord string
- // numProcessed tracks the number of TSV records processed by the underlying
- // reader
+ // numProcessed tracks the number of TSV records processed by the underlying reader
numProcessed uint64
+
+ // numDecoders is the number of concurrent goroutines to use for decoding
+ numDecoders int
}
// TSVConvertibleDoc implements the ConvertibleDoc interface for TSV input
@@ -37,11 +39,12 @@ type TSVConvertibleDoc struct {
// NewTSVInputReader returns a TSVInputReader configured to read input from the
// given io.Reader, extracting the specified fields only.
-func NewTSVInputReader(fields []string, in io.Reader) *TSVInputReader {
+func NewTSVInputReader(fields []string, in io.Reader, numDecoders int) *TSVInputReader {
return &TSVInputReader{
Fields: fields,
tsvReader: bufio.NewReader(in),
numProcessed: uint64(0),
+ numDecoders: numDecoders,
}
}
@@ -80,7 +83,7 @@ func (tsvInputReader *TSVInputReader) ReadHeadersFromSource() ([]string, error)
// hits EOF or an error. If ordered is true, it streams the documents in which
// the documents are read
func (tsvInputReader *TSVInputReader) StreamDocument(ordered bool, readChan chan bson.D, errChan chan error) {
- tsvRecordChan := make(chan ConvertibleDoc, numDecodingWorkers)
+ tsvRecordChan := make(chan ConvertibleDoc, tsvInputReader.numDecoders)
var err error
go func() {
@@ -104,7 +107,7 @@ func (tsvInputReader *TSVInputReader) StreamDocument(ordered bool, readChan chan
tsvInputReader.numProcessed++
}
}()
- streamDocuments(ordered, tsvRecordChan, readChan, errChan)
+ streamDocuments(ordered, tsvInputReader.numDecoders, tsvRecordChan, readChan, errChan)
}
// This is required to satisfy the ConvertibleDoc interface for TSV input. It
diff --git a/mongoimport/tsv_test.go b/mongoimport/tsv_test.go
index 54f3efb505e..105f9c35a07 100644
--- a/mongoimport/tsv_test.go
+++ b/mongoimport/tsv_test.go
@@ -23,7 +23,7 @@ func TestTSVStreamDocument(t *testing.T) {
bson.DocElem{"b", 2},
bson.DocElem{"c", "3e"},
}
- tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go tsvInputReader.StreamDocument(true, docChan, errChan)
@@ -40,7 +40,7 @@ func TestTSVStreamDocument(t *testing.T) {
bson.DocElem{"c", "3e"},
bson.DocElem{"field3", " may"},
}
- tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go tsvInputReader.StreamDocument(true, docChan, errChan)
@@ -57,7 +57,7 @@ func TestTSVStreamDocument(t *testing.T) {
bson.DocElem{"c", "Inline"},
bson.DocElem{"d", 14},
}
- tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go tsvInputReader.StreamDocument(true, docChan, errChan)
@@ -81,7 +81,7 @@ func TestTSVStreamDocument(t *testing.T) {
bson.DocElem{"c", 6},
},
}
- tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go tsvInputReader.StreamDocument(true, docChan, errChan)
@@ -108,7 +108,7 @@ func TestTSVStreamDocument(t *testing.T) {
bson.DocElem{"b", `"`},
bson.DocElem{"c", 6},
}
- tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go tsvInputReader.StreamDocument(true, docChan, errChan)
@@ -133,7 +133,7 @@ func TestTSVStreamDocument(t *testing.T) {
}
fileHandle, err := os.Open("testdata/test.tsv")
So(err, ShouldBeNil)
- tsvInputReader := NewTSVInputReader(fields, fileHandle)
+ tsvInputReader := NewTSVInputReader(fields, fileHandle, 1)
errChan := make(chan error)
docChan := make(chan bson.D)
go tsvInputReader.StreamDocument(true, docChan, errChan)
@@ -150,7 +150,7 @@ func TestTSVSetHeader(t *testing.T) {
Convey("setting the header should read the first line of the TSV", func() {
contents := "extraHeader1\textraHeader2\textraHeader3\n"
fields := []string{}
- tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
So(tsvInputReader.SetHeader(true), ShouldBeNil)
So(len(tsvInputReader.Fields), ShouldEqual, 3)
})
@@ -158,7 +158,7 @@ func TestTSVSetHeader(t *testing.T) {
"the header line with the existing fields", func() {
contents := "extraHeader\textraHeader2\textraHeader3\n\n"
fields := []string{"a", "b", "c"}
- tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents)))
+ tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents)), 1)
// if SetHeader() is called with fields already passed in,
// the header should be replaced with the read header line
So(tsvInputReader.SetHeader(true), ShouldBeNil)
@@ -173,7 +173,7 @@ func TestTSVGetHeaders(t *testing.T) {
Convey("With a TSV input reader", t, func() {
Convey("getting the header should return any already set headers", func() {
fields := []string{"extraHeader1", "extraHeader2", "extraHeader3"}
- tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte{}))
+ tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte{}), 1)
So(tsvInputReader.GetHeaders(), ShouldResemble, fields)
})
})
@@ -186,7 +186,7 @@ func TestTSVReadHeadersFromSource(t *testing.T) {
expectedHeaders := []string{"1", "2", "3"}
fileHandle, err := os.Open("testdata/test.tsv")
So(err, ShouldBeNil)
- tsvInputReader := NewTSVInputReader([]string{}, fileHandle)
+ tsvInputReader := NewTSVInputReader([]string{}, fileHandle, 1)
headers, err := tsvInputReader.ReadHeadersFromSource()
So(err, ShouldBeNil)
So(headers, ShouldResemble, expectedHeaders)
diff --git a/mongorestore/main/mongorestore.go b/mongorestore/main/mongorestore.go
index 7516da7aa44..1011610522b 100644
--- a/mongorestore/main/mongorestore.go
+++ b/mongorestore/main/mongorestore.go
@@ -9,7 +9,6 @@ import (
"github.com/mongodb/mongo-tools/mongorestore"
"github.com/mongodb/mongo-tools/mongorestore/options"
"os"
- "runtime"
)
func main() {
@@ -37,9 +36,6 @@ func main() {
log.SetVerbosity(opts.Verbosity)
- runtime.GOMAXPROCS(runtime.NumCPU())
- log.Logf(log.Info, "running mongorestore with %v job threads", outputOpts.JobThreads)
-
targetDir, err := getTargetDirFromArgs(extraArgs, os.Args, inputOpts.Directory)
if err != nil {
fmt.Printf("error parsing command line options: %v\n", err)
diff --git a/mongorestore/mongorestore.go b/mongorestore/mongorestore.go
index 299f518e050..c868eaaad73 100644
--- a/mongorestore/mongorestore.go
+++ b/mongorestore/mongorestore.go
@@ -38,12 +38,8 @@ type MongoRestore struct {
func (restore *MongoRestore) ParseAndValidateOptions() error {
// Can't use option pkg defaults for --objcheck because it's two separate flags,
// and we need to be able to see if they're both being used. We default to
- // true here and then see if noobjcheck is enable.
+ // true here and then see if noobjcheck is enabled.
log.Log(log.DebugHigh, "checking options")
- err := restore.ToolOptions.Validate()
- if err != nil {
- return err
- }
restore.objCheck = true
if restore.InputOptions.NoObjcheck {
restore.objCheck = false
@@ -74,6 +70,7 @@ func (restore *MongoRestore) ParseAndValidateOptions() error {
if !restore.InputOptions.OplogReplay {
return fmt.Errorf("cannot use --oplogLimit without --oplogReplay enabled")
}
+ var err error
restore.oplogLimit, err = ParseTimestampFlag(restore.InputOptions.OplogLimit)
if err != nil {
return fmt.Errorf("error parsing timestamp argument to --oplogLimit: %v", err)
@@ -97,7 +94,7 @@ func (restore *MongoRestore) ParseAndValidateOptions() error {
restore.tempRolesCol = "temproles"
}
- if restore.OutputOptions.BulkWriters < 0 {
+ if restore.ToolOptions.HiddenOptions.BulkWriters < 0 {
return fmt.Errorf(
"cannot specify a negative number of insertion workers per collection")
}
@@ -147,7 +144,7 @@ func (restore *MongoRestore) Restore() error {
}
// 2. Restore them...
- if restore.OutputOptions.JobThreads > 0 {
+ if restore.OutputOptions.NumParallelCollections > 0 {
restore.manager.Finalize(intents.MultiDatabaseLTF)
} else {
// use legacy restoration order if we are single-threaded
diff --git a/mongorestore/options/options.go b/mongorestore/options/options.go
index 5746727ab60..3172db4dc5b 100644
--- a/mongorestore/options/options.go
+++ b/mongorestore/options/options.go
@@ -14,17 +14,13 @@ func (self *InputOptions) Name() string {
}
type OutputOptions struct {
- Drop bool `long:"drop" description:"Drop each collection before import"`
- WriteConcern string `long:"writeConcern" default:"majority" description:"Write concern options e.g. --writeConcern majority, --writeConcern '{w: 3, wtimeout: 500, fsync: true, j: true}'"`
- NoIndexRestore bool `long:"noIndexRestore" description:"Don't restore indexes"`
- NoOptionsRestore bool `long:"noOptionsRestore" description:"Don't restore options"`
- KeepIndexVersion bool `long:"keepIndexVersion" description:"Don't update index version"`
-
- JobThreads int `long:"numParallelCollections" short:"j" description:"Number of collections to restore in parallel" default:"4"`
- BulkWriters int `long:"numInsertionWorkersPerCollection" description:"Number of insert connections per collection" default:"1"`
- BulkBufferSize int `long:"batchSize" description:"Maximum number of documents to coalesce into a single bulk insertion" default:"10000"`
- PreserveDocOrder bool `long:"preserveOrder" description:"Preserve order of documents during restoration"`
- // TODO: add hidden option for NumOSThreads to set GOMAXPROCS on CLI
+ Drop bool `long:"drop" description:"Drop each collection before import"`
+ WriteConcern string `long:"writeConcern" default:"majority" description:"Write concern options e.g. --writeConcern majority, --writeConcern '{w: 3, wtimeout: 500, fsync: true, j: true}'"`
+ NoIndexRestore bool `long:"noIndexRestore" description:"Don't restore indexes"`
+ NoOptionsRestore bool `long:"noOptionsRestore" description:"Don't restore options"`
+ KeepIndexVersion bool `long:"keepIndexVersion" description:"Don't update index version"`
+ MaintainInsertionOrder bool `long:"maintainInsertionOrder" description:"Preserve order of documents during restoration"`
+ NumParallelCollections int `long:"numParallelCollections" short:"j" description:"Number of collections to restore in parallel" default:"4"`
}
func (self *OutputOptions) Name() string {
diff --git a/mongorestore/restore.go b/mongorestore/restore.go
index 81272992cf2..f99d0864339 100644
--- a/mongorestore/restore.go
+++ b/mongorestore/restore.go
@@ -26,11 +26,11 @@ func (restore *MongoRestore) RestoreIntents() error {
restore.progressManager.Start()
defer restore.progressManager.Stop()
- if restore.OutputOptions.JobThreads > 0 {
+ if restore.OutputOptions.NumParallelCollections > 0 {
resultChan := make(chan error)
// start a goroutine for each job thread
- for i := 0; i < restore.OutputOptions.JobThreads; i++ {
+ for i := 0; i < restore.OutputOptions.NumParallelCollections; i++ {
go func(id int) {
log.Logf(log.DebugHigh, "starting restore routine with id=%v", id)
for {
@@ -51,7 +51,7 @@ func (restore *MongoRestore) RestoreIntents() error {
}
// wait until all goroutines are done or one of them errors out
- for i := 0; i < restore.OutputOptions.JobThreads; i++ {
+ for i := 0; i < restore.OutputOptions.NumParallelCollections; i++ {
select {
case err := <-resultChan:
if err != nil {
@@ -234,18 +234,18 @@ func (restore *MongoRestore) RestoreCollectionToDB(dbName, colName string,
defer restore.progressManager.Detach(bar)
}
- MaxInsertThreads := restore.OutputOptions.BulkWriters
- if restore.OutputOptions.PreserveDocOrder {
+ MaxInsertThreads := restore.ToolOptions.BulkWriters
+ if restore.OutputOptions.MaintainInsertionOrder {
MaxInsertThreads = 1
}
- docChan := make(chan bson.Raw, restore.OutputOptions.BulkBufferSize*MaxInsertThreads)
+ docChan := make(chan bson.Raw, restore.ToolOptions.BulkBufferSize*MaxInsertThreads)
resultChan := make(chan error, MaxInsertThreads)
killChan := make(chan struct{})
// make sure goroutines clean up on error
defer close(killChan)
// start a goroutine for adding up the number of bytes read
- bytesReadChan := make(chan int64, restore.OutputOptions.BulkBufferSize*MaxInsertThreads)
+ bytesReadChan := make(chan int64, restore.ToolOptions.BulkBufferSize*MaxInsertThreads)
go func() {
for {
select {
@@ -272,7 +272,7 @@ func (restore *MongoRestore) RestoreCollectionToDB(dbName, colName string,
for i := 0; i < MaxInsertThreads; i++ {
go func() {
- bulk := db.NewBufferedBulkInserter(collection, restore.OutputOptions.BulkBufferSize, false)
+ bulk := db.NewBufferedBulkInserter(collection, restore.ToolOptions.BulkBufferSize, false)
for {
select {
case rawDoc, alive := <-docChan:
diff --git a/vendor/src/github.com/jessevdk/go-flags/.travis.yml b/vendor/src/github.com/jessevdk/go-flags/.travis.yml
new file mode 100644
index 00000000000..485d3183902
--- /dev/null
+++ b/vendor/src/github.com/jessevdk/go-flags/.travis.yml
@@ -0,0 +1,35 @@
+language: go
+
+install:
+ # go-flags
+ - go get -d -v ./...
+ - go build -v ./...
+
+ # linting
+ - go get code.google.com/p/go.tools/cmd/vet
+ - go get github.com/golang/lint
+ - go install github.com/golang/lint/golint
+
+ # code coverage
+ - go get code.google.com/p/go.tools/cmd/cover
+ - go get github.com/onsi/ginkgo/ginkgo
+ - go get github.com/modocache/gover
+ - if [ "$TRAVIS_SECURE_ENV_VARS" = "true" ]; then go get github.com/mattn/goveralls; fi
+
+script:
+ # go-flags
+ - $(exit $(gofmt -l . | wc -l))
+ - go test -v ./...
+
+ # linting
+ - go tool vet -all=true -v=true . || true
+ - $(go env GOPATH | awk 'BEGIN{FS=":"} {print $1}')/bin/golint ./...
+
+ # code coverage
+ - $(go env GOPATH | awk 'BEGIN{FS=":"} {print $1}')/bin/ginkgo -r -cover
+ - $(go env GOPATH | awk 'BEGIN{FS=":"} {print $1}')/bin/gover
+ - if [ "$TRAVIS_SECURE_ENV_VARS" = "true" ]; then $(go env GOPATH | awk 'BEGIN{FS=":"} {print $1}')/bin/goveralls -coverprofile=gover.coverprofile -service=travis-ci -repotoken $COVERALLS_TOKEN; fi
+
+env:
+ # coveralls.io
+ secure: "RCYbiB4P0RjQRIoUx/vG/AjP3mmYCbzOmr86DCww1Z88yNcy3hYr3Cq8rpPtYU5v0g7wTpu4adaKIcqRE9xknYGbqj3YWZiCoBP1/n4Z+9sHW3Dsd9D/GRGeHUus0laJUGARjWoCTvoEtOgTdGQDoX7mH+pUUY0FBltNYUdOiiU="
diff --git a/vendor/src/github.com/jessevdk/go-flags/README.md b/vendor/src/github.com/jessevdk/go-flags/README.md
index 0e31cb1789b..b6faef6d38a 100644
--- a/vendor/src/github.com/jessevdk/go-flags/README.md
+++ b/vendor/src/github.com/jessevdk/go-flags/README.md
@@ -1,6 +1,8 @@
go-flags: a go library for parsing command line arguments
=========================================================
+[![GoDoc](https://godoc.org/github.com/jessevdk/go-flags?status.png)](https://godoc.org/github.com/jessevdk/go-flags) [![Build Status](https://travis-ci.org/jessevdk/go-flags.svg?branch=master)](https://travis-ci.org/jessevdk/go-flags) [![Coverage Status](https://img.shields.io/coveralls/jessevdk/go-flags.svg)](https://coveralls.io/r/jessevdk/go-flags?branch=master)
+
This library provides similar functionality to the builtin flag library of
go, but provides much more functionality and nicer formatting. From the
documentation:
@@ -25,6 +27,7 @@ Supported features:
* Supports same option multiple times (can store in slice or last option counts)
* Supports maps
* Supports function callbacks
+* Supports namespaces for (nested) option groups
The flags package uses structs, reflection and struct field tags
to allow users to specify command line options. This results in very simple
diff --git a/vendor/src/github.com/jessevdk/go-flags/arg.go b/vendor/src/github.com/jessevdk/go-flags/arg.go
new file mode 100644
index 00000000000..fd8db9c777c
--- /dev/null
+++ b/vendor/src/github.com/jessevdk/go-flags/arg.go
@@ -0,0 +1,21 @@
+package flags
+
+import (
+ "reflect"
+)
+
+// Arg represents a positional argument on the command line.
+type Arg struct {
+ // The name of the positional argument (used in the help)
+ Name string
+
+ // A description of the positional argument (used in the help)
+ Description string
+
+ value reflect.Value
+ tag multiTag
+}
+
+func (a *Arg) isRemaining() bool {
+ return a.value.Type().Kind() == reflect.Slice
+}
diff --git a/vendor/src/github.com/jessevdk/go-flags/arg_test.go b/vendor/src/github.com/jessevdk/go-flags/arg_test.go
new file mode 100644
index 00000000000..faea28093a6
--- /dev/null
+++ b/vendor/src/github.com/jessevdk/go-flags/arg_test.go
@@ -0,0 +1,53 @@
+package flags
+
+import (
+ "testing"
+)
+
+func TestPositional(t *testing.T) {
+ var opts = struct {
+ Value bool `short:"v"`
+
+ Positional struct {
+ Command int
+ Filename string
+ Rest []string
+ } `positional-args:"yes" required:"yes"`
+ }{}
+
+ p := NewParser(&opts, Default)
+ ret, err := p.ParseArgs([]string{"10", "arg_test.go", "a", "b"})
+
+ if err != nil {
+ t.Fatalf("Unexpected error: %v", err)
+ return
+ }
+
+ if opts.Positional.Command != 10 {
+ t.Fatalf("Expected opts.Positional.Command to be 10, but got %v", opts.Positional.Command)
+ }
+
+ if opts.Positional.Filename != "arg_test.go" {
+ t.Fatalf("Expected opts.Positional.Filename to be \"arg_test.go\", but got %v", opts.Positional.Filename)
+ }
+
+ assertStringArray(t, opts.Positional.Rest, []string{"a", "b"})
+ assertStringArray(t, ret, []string{})
+}
+
+func TestPositionalRequired(t *testing.T) {
+ var opts = struct {
+ Value bool `short:"v"`
+
+ Positional struct {
+ Command int
+ Filename string
+ Rest []string
+ } `positional-args:"yes" required:"yes"`
+ }{}
+
+ p := NewParser(&opts, None)
+ _, err := p.ParseArgs([]string{"10"})
+
+ assertError(t, err, ErrRequired, "the required argument `Filename` was not provided")
+}
diff --git a/vendor/src/github.com/jessevdk/go-flags/assert_test.go b/vendor/src/github.com/jessevdk/go-flags/assert_test.go
index 14949be56c3..8e06636b66d 100644
--- a/vendor/src/github.com/jessevdk/go-flags/assert_test.go
+++ b/vendor/src/github.com/jessevdk/go-flags/assert_test.go
@@ -1,23 +1,70 @@
package flags
import (
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "path"
+ "runtime"
"testing"
)
+func assertCallerInfo() (string, int) {
+ ptr := make([]uintptr, 15)
+ n := runtime.Callers(1, ptr)
+
+ if n == 0 {
+ return "", 0
+ }
+
+ mef := runtime.FuncForPC(ptr[0])
+ mefile, meline := mef.FileLine(ptr[0])
+
+ for i := 2; i < n; i++ {
+ f := runtime.FuncForPC(ptr[i])
+ file, line := f.FileLine(ptr[i])
+
+ if file != mefile {
+ return file, line
+ }
+ }
+
+ return mefile, meline
+}
+
+func assertErrorf(t *testing.T, format string, args ...interface{}) {
+ msg := fmt.Sprintf(format, args...)
+
+ file, line := assertCallerInfo()
+
+ t.Errorf("%s:%d: %s", path.Base(file), line, msg)
+}
+
+func assertFatalf(t *testing.T, format string, args ...interface{}) {
+ msg := fmt.Sprintf(format, args...)
+
+ file, line := assertCallerInfo()
+
+ t.Fatalf("%s:%d: %s", path.Base(file), line, msg)
+}
+
func assertString(t *testing.T, a string, b string) {
if a != b {
- t.Errorf("Expected %#v, but got %#v", b, a)
+ assertErrorf(t, "Expected %#v, but got %#v", b, a)
}
}
+
func assertStringArray(t *testing.T, a []string, b []string) {
if len(a) != len(b) {
- t.Errorf("Expected %#v, but got %#v", b, a)
+ assertErrorf(t, "Expected %#v, but got %#v", b, a)
return
}
for i, v := range a {
if b[i] != v {
- t.Errorf("Expected %#v, but got %#v", b, a)
+ assertErrorf(t, "Expected %#v, but got %#v", b, a)
return
}
}
@@ -25,13 +72,13 @@ func assertStringArray(t *testing.T, a []string, b []string) {
func assertBoolArray(t *testing.T, a []bool, b []bool) {
if len(a) != len(b) {
- t.Errorf("Expected %#v, but got %#v", b, a)
+ assertErrorf(t, "Expected %#v, but got %#v", b, a)
return
}
for i, v := range a {
if b[i] != v {
- t.Errorf("Expected %#v, but got %#v", b, a)
+ assertErrorf(t, "Expected %#v, but got %#v", b, a)
return
}
}
@@ -56,19 +103,19 @@ func assertParseSuccess(t *testing.T, data interface{}, args ...string) []string
func assertError(t *testing.T, err error, typ ErrorType, msg string) {
if err == nil {
- t.Fatalf("Expected error: %s", msg)
+ assertFatalf(t, "Expected error: %s", msg)
return
}
if e, ok := err.(*Error); !ok {
- t.Fatalf("Expected Error type, but got %#v", err)
+ assertFatalf(t, "Expected Error type, but got %#v", err)
} else {
if e.Type != typ {
- t.Errorf("Expected error type {%s}, but got {%s}", typ, e.Type)
+ assertErrorf(t, "Expected error type {%s}, but got {%s}", typ, e.Type)
}
if e.Message != msg {
- t.Errorf("Expected error message %#v, but got %#v", msg, e.Message)
+ assertErrorf(t, "Expected error message %#v, but got %#v", msg, e.Message)
}
}
}
@@ -80,3 +127,51 @@ func assertParseFail(t *testing.T, typ ErrorType, msg string, data interface{},
assertError(t, err, typ, msg)
return ret
}
+
+func diff(a, b string) (string, error) {
+ atmp, err := ioutil.TempFile("", "help-diff")
+
+ if err != nil {
+ return "", err
+ }
+
+ btmp, err := ioutil.TempFile("", "help-diff")
+
+ if err != nil {
+ return "", err
+ }
+
+ if _, err := io.WriteString(atmp, a); err != nil {
+ return "", err
+ }
+
+ if _, err := io.WriteString(btmp, b); err != nil {
+ return "", err
+ }
+
+ ret, err := exec.Command("diff", "-u", "-d", "--label", "got", atmp.Name(), "--label", "expected", btmp.Name()).Output()
+
+ os.Remove(atmp.Name())
+ os.Remove(btmp.Name())
+
+ if err.Error() == "exit status 1" {
+ return string(ret), nil
+ }
+
+ return string(ret), err
+}
+
+func assertDiff(t *testing.T, actual, expected, msg string) {
+ if actual == expected {
+ return
+ }
+
+ ret, err := diff(actual, expected)
+
+ if err != nil {
+ assertErrorf(t, "Unexpected diff error: %s", err)
+ assertErrorf(t, "Unexpected %s, expected:\n\n%s\n\nbut got\n\n%s", msg, expected, actual)
+ } else {
+ assertErrorf(t, "Unexpected %s:\n\n%s", msg, ret)
+ }
+}
diff --git a/vendor/src/github.com/jessevdk/go-flags/closest.go b/vendor/src/github.com/jessevdk/go-flags/closest.go
index 14f58d55110..3b518757c43 100644
--- a/vendor/src/github.com/jessevdk/go-flags/closest.go
+++ b/vendor/src/github.com/jessevdk/go-flags/closest.go
@@ -9,35 +9,33 @@ func levenshtein(s string, t string) int {
return len(s)
}
- var l1, l2, l3 int
-
- if len(s) == 1 {
- l1 = len(t) + 1
- } else {
- l1 = levenshtein(s[1:len(s)-1], t) + 1
- }
-
- if len(t) == 1 {
- l2 = len(s) + 1
- } else {
- l2 = levenshtein(t[1:len(t)-1], s) + 1
- }
-
- l3 = levenshtein(s[1:len(s)], t[1:len(t)])
-
- if s[0] != t[0] {
- l3++
- }
-
- if l2 < l1 {
- l1 = l2
- }
-
- if l1 < l3 {
- return l1
+ dists := make([][]int, len(s)+1)
+ for i := range dists {
+ dists[i] = make([]int, len(t)+1)
+ dists[i][0] = i
+ }
+
+ for j := range t {
+ dists[0][j] = j
+ }
+
+ for i, sc := range s {
+ for j, tc := range t {
+ if sc == tc {
+ dists[i+1][j+1] = dists[i][j]
+ } else {
+ dists[i+1][j+1] = dists[i][j] + 1
+ if dists[i+1][j] < dists[i+1][j+1] {
+ dists[i+1][j+1] = dists[i+1][j] + 1
+ }
+ if dists[i][j+1] < dists[i+1][j+1] {
+ dists[i+1][j+1] = dists[i][j+1] + 1
+ }
+ }
+ }
}
- return l3
+ return dists[len(s)][len(t)]
}
func closestChoice(cmd string, choices []string) (string, int) {
diff --git a/vendor/src/github.com/jessevdk/go-flags/command.go b/vendor/src/github.com/jessevdk/go-flags/command.go
index 9bdac581b01..13332ae331a 100644
--- a/vendor/src/github.com/jessevdk/go-flags/command.go
+++ b/vendor/src/github.com/jessevdk/go-flags/command.go
@@ -20,8 +20,12 @@ type Command struct {
// Aliases for the command
Aliases []string
+ // Whether positional arguments are required
+ ArgsRequired bool
+
commands []*Command
hasBuiltinHelpGroup bool
+ args []*Arg
}
// Commander is an interface which can be implemented by any command added in
@@ -50,6 +54,8 @@ type Usage interface {
func (c *Command) AddCommand(command string, shortDescription string, longDescription string, data interface{}) (*Command, error) {
cmd := newCommand(command, shortDescription, longDescription, data)
+ cmd.parent = c
+
if err := cmd.scan(); err != nil {
return nil, err
}
@@ -64,6 +70,8 @@ func (c *Command) AddCommand(command string, shortDescription string, longDescri
func (c *Command) AddGroup(shortDescription string, longDescription string, data interface{}) (*Group, error) {
group := newGroup(shortDescription, longDescription, data)
+ group.parent = c
+
if err := group.scanType(c.scanSubcommandHandler(group)); err != nil {
return nil, err
}
@@ -88,3 +96,11 @@ func (c *Command) Find(name string) *Command {
return nil
}
+
+// Args returns a list of positional arguments associated with this command.
+func (c *Command) Args() []*Arg {
+ ret := make([]*Arg, len(c.args))
+ copy(ret, c.args)
+
+ return ret
+}
diff --git a/vendor/src/github.com/jessevdk/go-flags/command_private.go b/vendor/src/github.com/jessevdk/go-flags/command_private.go
index bd97cc70a33..1727a3066ce 100644
--- a/vendor/src/github.com/jessevdk/go-flags/command_private.go
+++ b/vendor/src/github.com/jessevdk/go-flags/command_private.go
@@ -11,7 +11,6 @@ type lookup struct {
shortNames map[string]*Option
longNames map[string]*Option
- required map[*Option]bool
commands map[string]*Command
}
@@ -30,6 +29,44 @@ func (c *Command) scanSubcommandHandler(parentg *Group) scanHandler {
return true, err
}
+ positional := mtag.Get("positional-args")
+
+ if len(positional) != 0 {
+ stype := realval.Type()
+
+ for i := 0; i < stype.NumField(); i++ {
+ field := stype.Field(i)
+
+ m := newMultiTag((string(field.Tag)))
+
+ if err := m.Parse(); err != nil {
+ return true, err
+ }
+
+ name := m.Get("name")
+
+ if len(name) == 0 {
+ name = field.Name
+ }
+
+ arg := &Arg{
+ Name: name,
+ Description: m.Get("description"),
+
+ value: realval.Field(i),
+ tag: m,
+ }
+
+ c.args = append(c.args, arg)
+
+ if len(mtag.Get("required")) != 0 {
+ c.ArgsRequired = true
+ }
+ }
+
+ return true, nil
+ }
+
subcommand := mtag.Get("command")
if len(subcommand) != 0 {
@@ -104,23 +141,17 @@ func (c *Command) makeLookup() lookup {
ret := lookup{
shortNames: make(map[string]*Option),
longNames: make(map[string]*Option),
-
- required: make(map[*Option]bool),
- commands: make(map[string]*Command),
+ commands: make(map[string]*Command),
}
c.eachGroup(func(g *Group) {
for _, option := range g.options {
- if option.Required && option.canCli() {
- ret.required[option] = true
- }
-
if option.ShortName != 0 {
ret.shortNames[string(option.ShortName)] = option
}
if len(option.LongName) > 0 {
- ret.longNames[option.LongName] = option
+ ret.longNames[option.LongNameWithNamespace()] = option
}
}
})
@@ -209,3 +240,11 @@ func (c *Command) hasCliOptions() bool {
return ret
}
+
+func (c *Command) fillParseState(s *parseState) {
+ s.positional = make([]*Arg, len(c.args))
+ copy(s.positional, c.args)
+
+ s.lookup = c.makeLookup()
+ s.command = c
+}
diff --git a/vendor/src/github.com/jessevdk/go-flags/command_test.go b/vendor/src/github.com/jessevdk/go-flags/command_test.go
index b2df7e3dc6a..a093e1588e2 100644
--- a/vendor/src/github.com/jessevdk/go-flags/command_test.go
+++ b/vendor/src/github.com/jessevdk/go-flags/command_test.go
@@ -1,6 +1,7 @@
package flags
import (
+ "fmt"
"testing"
)
@@ -114,6 +115,23 @@ func TestCommandEstimate(t *testing.T) {
assertError(t, err, ErrCommandRequired, "Please specify one command of: add or remove")
}
+func TestCommandEstimate2(t *testing.T) {
+ var opts = struct {
+ Value bool `short:"v"`
+
+ Cmd1 struct {
+ } `command:"remove"`
+
+ Cmd2 struct {
+ } `command:"add"`
+ }{}
+
+ p := NewParser(&opts, None)
+ _, err := p.ParseArgs([]string{"rmive"})
+
+ assertError(t, err, ErrUnknownCommand, "Unknown command `rmive', did you mean `remove'?")
+}
+
type testCommand struct {
G bool `short:"g"`
Executed bool
@@ -265,7 +283,7 @@ func TestRequiredOnCommand(t *testing.T) {
} `command:"cmd"`
}{}
- assertParseFail(t, ErrRequired, "the required flag `-v' was not specified", &opts, "cmd")
+ assertParseFail(t, ErrRequired, fmt.Sprintf("the required flag `%cv' was not specified", defaultShortOptDelimiter), &opts, "cmd")
}
func TestRequiredAllOnCommand(t *testing.T) {
@@ -278,7 +296,7 @@ func TestRequiredAllOnCommand(t *testing.T) {
} `command:"cmd"`
}{}
- assertParseFail(t, ErrRequired, "the required flags `-v' and `--missing' were not specified", &opts, "cmd")
+ assertParseFail(t, ErrRequired, fmt.Sprintf("the required flags `%smissing' and `%cv' were not specified", defaultLongOptDelimiter, defaultShortOptDelimiter), &opts, "cmd")
}
func TestDefaultOnCommand(t *testing.T) {
diff --git a/vendor/src/github.com/jessevdk/go-flags/completion.go b/vendor/src/github.com/jessevdk/go-flags/completion.go
new file mode 100644
index 00000000000..cb7aed6a725
--- /dev/null
+++ b/vendor/src/github.com/jessevdk/go-flags/completion.go
@@ -0,0 +1,304 @@
+package flags
+
+import (
+ "fmt"
+ "path/filepath"
+ "reflect"
+ "sort"
+ "strings"
+ "unicode/utf8"
+)
+
+// Completion is a type containing information of a completion.
+type Completion struct {
+ // The completed item
+ Item string
+
+ // A description of the completed item (optional)
+ Description string
+}
+
+type completions []Completion
+
+func (c completions) Len() int {
+ return len(c)
+}
+
+func (c completions) Less(i, j int) bool {
+ return c[i].Item < c[j].Item
+}
+
+func (c completions) Swap(i, j int) {
+ c[i], c[j] = c[j], c[i]
+}
+
+// Completer is an interface which can be implemented by types
+// to provide custom command line argument completion.
+type Completer interface {
+ // Complete receives a prefix representing a (partial) value
+ // for its type and should provide a list of possible valid
+ // completions.
+ Complete(match string) []Completion
+}
+
+type completion struct {
+ parser *Parser
+
+ ShowDescriptions bool
+}
+
+// Filename is a string alias which provides filename completion.
+type Filename string
+
+func completionsWithoutDescriptions(items []string) []Completion {
+ ret := make([]Completion, len(items))
+
+ for i, v := range items {
+ ret[i].Item = v
+ }
+
+ return ret
+}
+
+// Complete returns a list of existing files with the given
+// prefix.
+func (f *Filename) Complete(match string) []Completion {
+ ret, _ := filepath.Glob(match + "*")
+ return completionsWithoutDescriptions(ret)
+}
+
+func (c *completion) skipPositional(s *parseState, n int) {
+ if n >= len(s.positional) {
+ s.positional = nil
+ } else {
+ s.positional = s.positional[n:]
+ }
+}
+
+func (c *completion) completeOptionNames(names map[string]*Option, prefix string, match string) []Completion {
+ n := make([]Completion, 0, len(names))
+
+ for k, opt := range names {
+ if strings.HasPrefix(k, match) {
+ n = append(n, Completion{
+ Item: prefix + k,
+ Description: opt.Description,
+ })
+ }
+ }
+
+ return n
+}
+
+func (c *completion) completeLongNames(s *parseState, prefix string, match string) []Completion {
+ return c.completeOptionNames(s.lookup.longNames, prefix, match)
+}
+
+func (c *completion) completeShortNames(s *parseState, prefix string, match string) []Completion {
+ if len(match) != 0 {
+ return []Completion{
+ Completion{
+ Item: prefix + match,
+ },
+ }
+ }
+
+ return c.completeOptionNames(s.lookup.shortNames, prefix, match)
+}
+
+func (c *completion) completeCommands(s *parseState, match string) []Completion {
+ n := make([]Completion, 0, len(s.command.commands))
+
+ for _, cmd := range s.command.commands {
+ if cmd.data != c && strings.HasPrefix(cmd.Name, match) {
+ n = append(n, Completion{
+ Item: cmd.Name,
+ Description: cmd.ShortDescription,
+ })
+ }
+ }
+
+ return n
+}
+
+func (c *completion) completeValue(value reflect.Value, prefix string, match string) []Completion {
+ i := value.Interface()
+
+ var ret []Completion
+
+ if cmp, ok := i.(Completer); ok {
+ ret = cmp.Complete(match)
+ } else if value.CanAddr() {
+ if cmp, ok = value.Addr().Interface().(Completer); ok {
+ ret = cmp.Complete(match)
+ }
+ }
+
+ for i, v := range ret {
+ ret[i].Item = prefix + v.Item
+ }
+
+ return ret
+}
+
+func (c *completion) completeArg(arg *Arg, prefix string, match string) []Completion {
+ if arg.isRemaining() {
+ // For remaining positional args (that are parsed into a slice), complete
+ // based on the element type.
+ return c.completeValue(reflect.New(arg.value.Type().Elem()), prefix, match)
+ }
+
+ return c.completeValue(arg.value, prefix, match)
+}
+
+func (c *completion) complete(args []string) []Completion {
+ if len(args) == 0 {
+ args = []string{""}
+ }
+
+ s := &parseState{
+ args: args,
+ }
+
+ c.parser.fillParseState(s)
+
+ var opt *Option
+
+ for len(s.args) > 1 {
+ arg := s.pop()
+
+ if (c.parser.Options&PassDoubleDash) != None && arg == "--" {
+ opt = nil
+ c.skipPositional(s, len(s.args)-1)
+
+ break
+ }
+
+ if argumentIsOption(arg) {
+ prefix, optname, islong := stripOptionPrefix(arg)
+ optname, _, argument := splitOption(prefix, optname, islong)
+
+ if argument == nil {
+ var o *Option
+ canarg := true
+
+ if islong {
+ o = s.lookup.longNames[optname]
+ } else {
+ for i, r := range optname {
+ sname := string(r)
+ o = s.lookup.shortNames[sname]
+
+ if o == nil {
+ break
+ }
+
+ if i == 0 && o.canArgument() && len(optname) != len(sname) {
+ canarg = false
+ break
+ }
+ }
+ }
+
+ if o == nil && (c.parser.Options&PassAfterNonOption) != None {
+ opt = nil
+ c.skipPositional(s, len(s.args)-1)
+
+ break
+ } else if o != nil && o.canArgument() && !o.OptionalArgument && canarg {
+ if len(s.args) > 1 {
+ s.pop()
+ } else {
+ opt = o
+ }
+ }
+ }
+ } else {
+ if len(s.positional) > 0 {
+ if !s.positional[0].isRemaining() {
+ // Don't advance beyond a remaining positional arg (because
+ // it consumes all subsequent args).
+ s.positional = s.positional[1:]
+ }
+ } else if cmd, ok := s.lookup.commands[arg]; ok {
+ cmd.fillParseState(s)
+ }
+
+ opt = nil
+ }
+ }
+
+ lastarg := s.args[len(s.args)-1]
+ var ret []Completion
+
+ if opt != nil {
+ // Completion for the argument of 'opt'
+ ret = c.completeValue(opt.value, "", lastarg)
+ } else if argumentIsOption(lastarg) {
+ // Complete the option
+ prefix, optname, islong := stripOptionPrefix(lastarg)
+ optname, split, argument := splitOption(prefix, optname, islong)
+
+ if argument == nil && !islong {
+ rname, n := utf8.DecodeRuneInString(optname)
+ sname := string(rname)
+
+ if opt := s.lookup.shortNames[sname]; opt != nil && opt.canArgument() {
+ ret = c.completeValue(opt.value, prefix+sname, optname[n:])
+ } else {
+ ret = c.completeShortNames(s, prefix, optname)
+ }
+ } else if argument != nil {
+ if islong {
+ opt = s.lookup.longNames[optname]
+ } else {
+ opt = s.lookup.shortNames[optname]
+ }
+
+ if opt != nil {
+ ret = c.completeValue(opt.value, prefix+optname+split, *argument)
+ }
+ } else if islong {
+ ret = c.completeLongNames(s, prefix, optname)
+ } else {
+ ret = c.completeShortNames(s, prefix, optname)
+ }
+ } else if len(s.positional) > 0 {
+ // Complete for positional argument
+ ret = c.completeArg(s.positional[0], "", lastarg)
+ } else if len(s.command.commands) > 0 {
+ // Complete for command
+ ret = c.completeCommands(s, lastarg)
+ }
+
+ sort.Sort(completions(ret))
+ return ret
+}
+
+func (c *completion) execute(args []string) {
+ ret := c.complete(args)
+
+ if c.ShowDescriptions && len(ret) > 1 {
+ maxl := 0
+
+ for _, v := range ret {
+ if len(v.Item) > maxl {
+ maxl = len(v.Item)
+ }
+ }
+
+ for _, v := range ret {
+ fmt.Printf("%s", v.Item)
+
+ if len(v.Description) > 0 {
+ fmt.Printf("%s # %s", strings.Repeat(" ", maxl-len(v.Item)), v.Description)
+ }
+
+ fmt.Printf("\n")
+ }
+ } else {
+ for _, v := range ret {
+ fmt.Println(v.Item)
+ }
+ }
+}
diff --git a/vendor/src/github.com/jessevdk/go-flags/completion_test.go b/vendor/src/github.com/jessevdk/go-flags/completion_test.go
new file mode 100644
index 00000000000..2d5a97f5976
--- /dev/null
+++ b/vendor/src/github.com/jessevdk/go-flags/completion_test.go
@@ -0,0 +1,289 @@
+package flags
+
+import (
+ "bytes"
+ "io"
+ "os"
+ "path"
+ "path/filepath"
+ "reflect"
+ "runtime"
+ "strings"
+ "testing"
+)
+
+type TestComplete struct {
+}
+
+func (t *TestComplete) Complete(match string) []Completion {
+ options := []string{
+ "hello world",
+ "hello universe",
+ "hello multiverse",
+ }
+
+ ret := make([]Completion, 0, len(options))
+
+ for _, o := range options {
+ if strings.HasPrefix(o, match) {
+ ret = append(ret, Completion{
+ Item: o,
+ })
+ }
+ }
+
+ return ret
+}
+
+var completionTestOptions struct {
+ Verbose bool `short:"v" long:"verbose" description:"Verbose messages"`
+ Debug bool `short:"d" long:"debug" description:"Enable debug"`
+ Version bool `long:"version" description:"Show version"`
+ Required bool `long:"required" required:"true" description:"This is required"`
+
+ AddCommand struct {
+ Positional struct {
+ Filename Filename
+ } `positional-args:"yes"`
+ } `command:"add" description:"add an item"`
+
+ AddMultiCommand struct {
+ Positional struct {
+ Filename []Filename
+ } `positional-args:"yes"`
+ } `command:"add-multi" description:"add multiple items"`
+
+ RemoveCommand struct {
+ Other bool `short:"o"`
+ File Filename `short:"f" long:"filename"`
+ } `command:"rm" description:"remove an item"`
+
+ RenameCommand struct {
+ Completed TestComplete `short:"c" long:"completed"`
+ } `command:"rename" description:"rename an item"`
+}
+
+type completionTest struct {
+ Args []string
+ Completed []string
+ ShowDescriptions bool
+}
+
+var completionTests []completionTest
+
+func init() {
+ _, sourcefile, _, _ := runtime.Caller(0)
+ completionTestSourcedir := filepath.Join(filepath.SplitList(path.Dir(sourcefile))...)
+
+ completionTestFilename := []string{filepath.Join(completionTestSourcedir, "completion.go"), filepath.Join(completionTestSourcedir, "completion_test.go")}
+
+ completionTests = []completionTest{
+ {
+ // Short names
+ []string{"-"},
+ []string{"-d", "-v"},
+ false,
+ },
+
+ {
+ // Short names concatenated
+ []string{"-dv"},
+ []string{"-dv"},
+ false,
+ },
+
+ {
+ // Long names
+ []string{"--"},
+ []string{"--debug", "--required", "--verbose", "--version"},
+ false,
+ },
+
+ {
+ // Long names with descriptions
+ []string{"--"},
+ []string{
+ "--debug # Enable debug",
+ "--required # This is required",
+ "--verbose # Verbose messages",
+ "--version # Show version",
+ },
+ true,
+ },
+
+ {
+ // Long names partial
+ []string{"--ver"},
+ []string{"--verbose", "--version"},
+ false,
+ },
+
+ {
+ // Commands
+ []string{""},
+ []string{"add", "add-multi", "rename", "rm"},
+ false,
+ },
+
+ {
+ // Commands with descriptions
+ []string{""},
+ []string{
+ "add # add an item",
+ "add-multi # add multiple items",
+ "rename # rename an item",
+ "rm # remove an item",
+ },
+ true,
+ },
+
+ {
+ // Commands partial
+ []string{"r"},
+ []string{"rename", "rm"},
+ false,
+ },
+
+ {
+ // Positional filename
+ []string{"add", filepath.Join(completionTestSourcedir, "completion")},
+ completionTestFilename,
+ false,
+ },
+
+ {
+ // Multiple positional filename (1 arg)
+ []string{"add-multi", filepath.Join(completionTestSourcedir, "completion")},
+ completionTestFilename,
+ false,
+ },
+ {
+ // Multiple positional filename (2 args)
+ []string{"add-multi", filepath.Join(completionTestSourcedir, "completion.go"), filepath.Join(completionTestSourcedir, "completion")},
+ completionTestFilename,
+ false,
+ },
+ {
+ // Multiple positional filename (3 args)
+ []string{"add-multi", filepath.Join(completionTestSourcedir, "completion.go"), filepath.Join(completionTestSourcedir, "completion.go"), filepath.Join(completionTestSourcedir, "completion")},
+ completionTestFilename,
+ false,
+ },
+
+ {
+ // Flag filename
+ []string{"rm", "-f", path.Join(completionTestSourcedir, "completion")},
+ completionTestFilename,
+ false,
+ },
+
+ {
+ // Flag short concat last filename
+ []string{"rm", "-of", path.Join(completionTestSourcedir, "completion")},
+ completionTestFilename,
+ false,
+ },
+
+ {
+ // Flag concat filename
+ []string{"rm", "-f" + path.Join(completionTestSourcedir, "completion")},
+ []string{"-f" + completionTestFilename[0], "-f" + completionTestFilename[1]},
+ false,
+ },
+
+ {
+ // Flag equal concat filename
+ []string{"rm", "-f=" + path.Join(completionTestSourcedir, "completion")},
+ []string{"-f=" + completionTestFilename[0], "-f=" + completionTestFilename[1]},
+ false,
+ },
+
+ {
+ // Flag concat long filename
+ []string{"rm", "--filename=" + path.Join(completionTestSourcedir, "completion")},
+ []string{"--filename=" + completionTestFilename[0], "--filename=" + completionTestFilename[1]},
+ false,
+ },
+
+ {
+ // Flag long filename
+ []string{"rm", "--filename", path.Join(completionTestSourcedir, "completion")},
+ completionTestFilename,
+ false,
+ },
+
+ {
+ // Custom completed
+ []string{"rename", "-c", "hello un"},
+ []string{"hello universe"},
+ false,
+ },
+ }
+}
+
+func TestCompletion(t *testing.T) {
+ p := NewParser(&completionTestOptions, Default)
+ c := &completion{parser: p}
+
+ for _, test := range completionTests {
+ if test.ShowDescriptions {
+ continue
+ }
+
+ ret := c.complete(test.Args)
+ items := make([]string, len(ret))
+
+ for i, v := range ret {
+ items[i] = v.Item
+ }
+
+ if !reflect.DeepEqual(items, test.Completed) {
+ t.Errorf("Args: %#v, %#v\n Expected: %#v\n Got: %#v", test.Args, test.ShowDescriptions, test.Completed, items)
+ }
+ }
+}
+
+func TestParserCompletion(t *testing.T) {
+ for _, test := range completionTests {
+ if test.ShowDescriptions {
+ os.Setenv("GO_FLAGS_COMPLETION", "verbose")
+ } else {
+ os.Setenv("GO_FLAGS_COMPLETION", "1")
+ }
+
+ tmp := os.Stdout
+
+ r, w, _ := os.Pipe()
+ os.Stdout = w
+
+ out := make(chan string)
+
+ go func() {
+ var buf bytes.Buffer
+
+ io.Copy(&buf, r)
+
+ out <- buf.String()
+ }()
+
+ p := NewParser(&completionTestOptions, None)
+
+ _, err := p.ParseArgs(test.Args)
+
+ w.Close()
+
+ os.Stdout = tmp
+
+ if err != nil {
+ t.Fatalf("Unexpected error: %s", err)
+ }
+
+ got := strings.Split(strings.Trim(<-out, "\n"), "\n")
+
+ if !reflect.DeepEqual(got, test.Completed) {
+ t.Errorf("Expected: %#v\nGot: %#v", test.Completed, got)
+ }
+ }
+
+ os.Setenv("GO_FLAGS_COMPLETION", "")
+}
diff --git a/vendor/src/github.com/jessevdk/go-flags/convert.go b/vendor/src/github.com/jessevdk/go-flags/convert.go
index be8de393d9c..191b5f4cd1a 100644
--- a/vendor/src/github.com/jessevdk/go-flags/convert.go
+++ b/vendor/src/github.com/jessevdk/go-flags/convert.go
@@ -294,6 +294,32 @@ func convert(val string, retval reflect.Value, options multiTag) error {
return nil
}
+func isPrint(s string) bool {
+ for _, c := range s {
+ if !strconv.IsPrint(c) {
+ return false
+ }
+ }
+
+ return true
+}
+
+func quoteIfNeeded(s string) string {
+ if !isPrint(s) {
+ return strconv.Quote(s)
+ }
+
+ return s
+}
+
+func unquoteIfPossible(s string) (string, error) {
+ if len(s) == 0 || s[0] != '"' {
+ return s, nil
+ }
+
+ return strconv.Unquote(s)
+}
+
func wrapText(s string, l int, prefix string) string {
// Basic text wrapping of s at spaces to fit in l
var ret string
diff --git a/vendor/src/github.com/jessevdk/go-flags/convert_test.go b/vendor/src/github.com/jessevdk/go-flags/convert_test.go
index 4a409111ace..0de0eea7a01 100644
--- a/vendor/src/github.com/jessevdk/go-flags/convert_test.go
+++ b/vendor/src/github.com/jessevdk/go-flags/convert_test.go
@@ -71,7 +71,7 @@ func TestConvertToString(t *testing.T) {
true,
[]int{-3, 4, -2},
- map[int]float64{-2: 4.5, -3: 0.1},
+ map[int]float64{-2: 4.5},
new(bool),
float32(5.2),
@@ -104,7 +104,7 @@ func TestConvertToString(t *testing.T) {
"true",
"[-3, 4, -2]",
- "{-2:4.5, -3:0.1}",
+ "{-2:4.5}",
"false",
"5.2",
@@ -171,13 +171,5 @@ func TestWrapText(t *testing.T) {
occaecat cupidatat non proident, sunt in culpa qui officia
deserunt mollit anim id est laborum.`
- if got != expected {
- ret, err := helpDiff(got, expected)
-
- if err != nil {
- t.Errorf("Unexpected wrapped text, expected:\n\n%s\n\nbut got\n\n%s", expected, got)
- } else {
- t.Errorf("Unexpected wrapped text:\n\n%s", ret)
- }
- }
+ assertDiff(t, got, expected, "wrapped text")
}
diff --git a/vendor/src/github.com/jessevdk/go-flags/error.go b/vendor/src/github.com/jessevdk/go-flags/error.go
index 3a676932f62..fce9d312128 100644
--- a/vendor/src/github.com/jessevdk/go-flags/error.go
+++ b/vendor/src/github.com/jessevdk/go-flags/error.go
@@ -2,7 +2,6 @@ package flags
import (
"fmt"
- "reflect"
)
// ErrorType represents the type of error.
@@ -55,7 +54,36 @@ const (
)
func (e ErrorType) String() string {
- return reflect.TypeOf(e).Name()
+ switch e {
+ case ErrUnknown:
+ return "unknown"
+ case ErrExpectedArgument:
+ return "expected argument"
+ case ErrUnknownFlag:
+ return "unknown flag"
+ case ErrUnknownGroup:
+ return "unknown group"
+ case ErrMarshal:
+ return "marshal"
+ case ErrHelp:
+ return "help"
+ case ErrNoArgumentForBool:
+ return "no argument for bool"
+ case ErrRequired:
+ return "required"
+ case ErrShortNameTooLong:
+ return "short name too long"
+ case ErrDuplicatedFlag:
+ return "duplicated flag"
+ case ErrTag:
+ return "tag"
+ case ErrCommandRequired:
+ return "command required"
+ case ErrUnknownCommand:
+ return "unknown command"
+ }
+
+ return "unrecognized error type"
}
// Error represents a parser error. The error returned from Parse is of this
diff --git a/vendor/src/github.com/jessevdk/go-flags/example_test.go b/vendor/src/github.com/jessevdk/go-flags/example_test.go
index b06dd27ac70..f7be2bb14f2 100644
--- a/vendor/src/github.com/jessevdk/go-flags/example_test.go
+++ b/vendor/src/github.com/jessevdk/go-flags/example_test.go
@@ -4,7 +4,6 @@ package flags
import (
"fmt"
"os/exec"
- "strings"
)
func Example() {
@@ -36,6 +35,16 @@ func Example() {
// Example of a map
IntMap map[string]int `long:"intmap" description:"A map from string to int"`
+
+ // Example of a filename (useful for completion)
+ Filename Filename `long:"filename" description:"A filename"`
+
+ // Example of positional arguments
+ Args struct {
+ Id string
+ Num int
+ Rest []string
+ } `positional-args:"yes" required:"yes"`
}
// Callback which will invoke callto:<argument> to call a number.
@@ -59,15 +68,17 @@ func Example() {
"--ptrslice", "world",
"--intmap", "a:1",
"--intmap", "b:5",
- "arg1",
- "arg2",
- "arg3",
+ "--filename", "hello.go",
+ "id",
+ "10",
+ "remaining1",
+ "remaining2",
}
// Parse flags from `args'. Note that here we use flags.ParseArgs for
// the sake of making a working example. Normally, you would simply use
// flags.Parse(&opts) which uses os.Args
- args, err := ParseArgs(&opts, args)
+ _, err := ParseArgs(&opts, args)
if err != nil {
panic(err)
@@ -80,7 +91,10 @@ func Example() {
fmt.Printf("StringSlice: %v\n", opts.StringSlice)
fmt.Printf("PtrSlice: [%v %v]\n", *opts.PtrSlice[0], *opts.PtrSlice[1])
fmt.Printf("IntMap: [a:%v b:%v]\n", opts.IntMap["a"], opts.IntMap["b"])
- fmt.Printf("Remaining args: %s\n", strings.Join(args, " "))
+ fmt.Printf("Filename: %v\n", opts.Filename)
+ fmt.Printf("Args.Id: %s\n", opts.Args.Id)
+ fmt.Printf("Args.Num: %d\n", opts.Args.Num)
+ fmt.Printf("Args.Rest: %v\n", opts.Args.Rest)
// Output: Verbosity: [true true]
// Offset: 5
@@ -89,5 +103,8 @@ func Example() {
// StringSlice: [hello world]
// PtrSlice: [hello world]
// IntMap: [a:1 b:5]
- // Remaining args: arg1 arg2 arg3
+ // Filename: hello.go
+ // Args.Id: id
+ // Args.Num: 10
+ // Args.Rest: [remaining1 remaining2]
}
diff --git a/vendor/src/github.com/jessevdk/go-flags/examples/bash-completion b/vendor/src/github.com/jessevdk/go-flags/examples/bash-completion
new file mode 100644
index 00000000000..974f52ad43f
--- /dev/null
+++ b/vendor/src/github.com/jessevdk/go-flags/examples/bash-completion
@@ -0,0 +1,9 @@
+_examples() {
+ args=("${COMP_WORDS[@]:1:$COMP_CWORD}")
+
+ local IFS=$'\n'
+ COMPREPLY=($(GO_FLAGS_COMPLETION=1 ${COMP_WORDS[0]} "${args[@]}"))
+ return 1
+}
+
+complete -F _examples examples
diff --git a/vendor/src/github.com/jessevdk/go-flags/examples/main.go b/vendor/src/github.com/jessevdk/go-flags/examples/main.go
index 53369b0f604..4a22be6e86d 100644
--- a/vendor/src/github.com/jessevdk/go-flags/examples/main.go
+++ b/vendor/src/github.com/jessevdk/go-flags/examples/main.go
@@ -10,8 +10,8 @@ import (
)
type EditorOptions struct {
- Input string `short:"i" long:"input" description:"Input file" default:"-"`
- Output string `short:"o" long:"output" description:"Output file" default:"-"`
+ Input flags.Filename `short:"i" long:"input" description:"Input file" default:"-"`
+ Output flags.Filename `short:"o" long:"output" description:"Output file" default:"-"`
}
type Point struct {
diff --git a/vendor/src/github.com/jessevdk/go-flags/flags.go b/vendor/src/github.com/jessevdk/go-flags/flags.go
index 34adf1aee66..e3e72a32680 100644
--- a/vendor/src/github.com/jessevdk/go-flags/flags.go
+++ b/vendor/src/github.com/jessevdk/go-flags/flags.go
@@ -2,156 +2,237 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-// Package flags provides an extensive command line option parser.
-// The flags package is similar in functionality to the go built-in flag package
-// but provides more options and uses reflection to provide a convenient and
-// succinct way of specifying command line options.
-//
-// Supported features:
-// Options with short names (-v)
-// Options with long names (--verbose)
-// Options with and without arguments (bool v.s. other type)
-// Options with optional arguments and default values
-// Multiple option groups each containing a set of options
-// Generate and print well-formatted help message
-// Passing remaining command line arguments after -- (optional)
-// Ignoring unknown command line options (optional)
-// Supports -I/usr/include -I=/usr/include -I /usr/include option argument specification
-// Supports multiple short options -aux
-// Supports all primitive go types (string, int{8..64}, uint{8..64}, float)
-// Supports same option multiple times (can store in slice or last option counts)
-// Supports maps
-// Supports function callbacks
-//
-// Additional features specific to Windows:
-// Options with short names (/v)
-// Options with long names (/verbose)
-// Windows-style options with arguments use a colon as the delimiter
-// Modify generated help message with Windows-style / options
-//
-// The flags package uses structs, reflection and struct field tags
-// to allow users to specify command line options. This results in very simple
-// and concise specification of your application options. For example:
-//
-// type Options struct {
-// Verbose []bool `short:"v" long:"verbose" description:"Show verbose debug information"`
-// }
-//
-// This specifies one option with a short name -v and a long name --verbose.
-// When either -v or --verbose is found on the command line, a 'true' value
-// will be appended to the Verbose field. e.g. when specifying -vvv, the
-// resulting value of Verbose will be {[true, true, true]}.
-//
-// Slice options work exactly the same as primitive type options, except that
-// whenever the option is encountered, a value is appended to the slice.
-//
-// Map options from string to primitive type are also supported. On the command
-// line, you specify the value for such an option as key:value. For example
-//
-// type Options struct {
-// AuthorInfo string[string] `short:"a"`
-// }
-//
-// Then, the AuthorInfo map can be filled with something like
-// -a name:Jesse -a "surname:van den Kieboom".
-//
-// Finally, for full control over the conversion between command line argument
-// values and options, user defined types can choose to implement the Marshaler
-// and Unmarshaler interfaces.
-//
-// Available field tags:
-// short: the short name of the option (single character)
-// long: the long name of the option
-// required: whether an option is required to appear on the command
-// line. If a required option is not present, the parser will
-// return ErrRequired (optional)
-// description: the description of the option (optional)
-// long-description: the long description of the option. Currently only
-// displayed in generated man pages (optional)
-// no-flag: if non-empty this field is ignored as an option (optional)
-//
-// optional: whether an argument of the option is optional (optional)
-// optional-value: the value of an optional option when the option occurs
-// without an argument. This tag can be specified multiple
-// times in the case of maps or slices (optional)
-// default: the default value of an option. This tag can be specified
-// multiple times in the case of slices or maps (optional)
-// default-mask: when specified, this value will be displayed in the help
-// instead of the actual default value. This is useful
-// mostly for hiding otherwise sensitive information from
-// showing up in the help. If default-mask takes the special
-// value "-", then no default value will be shown at all
-// (optional)
-// value-name: the name of the argument value (to be shown in the help,
-// (optional)
-//
-// base: a base (radix) used to convert strings to integer values, the
-// default base is 10 (i.e. decimal) (optional)
-//
-// ini-name: the explicit ini option name (optional)
-// no-ini: if non-empty this field is ignored as an ini option
-// (optional)
-//
-// group: when specified on a struct field, makes the struct
-// field a separate group with the given name (optional)
-// command: when specified on a struct field, makes the struct
-// field a (sub)command with the given name (optional)
-// subcommands-optional: when specified on a command struct field, makes
-// any subcommands of that command optional (optional)
-// alias: when specified on a command struct field, adds the
-// specified name as an alias for the command. Can be
-// be specified multiple times to add more than one
-// alias (optional)
-//
-// Either short: or long: must be specified to make the field eligible as an
-// option.
-//
-//
-// Option groups:
-//
-// Option groups are a simple way to semantically separate your options. The
-// only real difference is in how your options will appear in the built-in
-// generated help. All options in a particular group are shown together in the
-// help under the name of the group.
-//
-// There are currently three ways to specify option groups.
-//
-// 1. Use NewNamedParser specifying the various option groups.
-// 2. Use AddGroup to add a group to an existing parser.
-// 3. Add a struct field to the top-level options annotated with the
-// group:"group-name" tag.
-//
-//
-//
-// Commands:
-//
-// The flags package also has basic support for commands. Commands are often
-// used in monolithic applications that support various commands or actions.
-// Take git for example, all of the add, commit, checkout, etc. are called
-// commands. Using commands you can easily separate multiple functions of your
-// application.
-//
-// There are currently two ways to specify a command.
-//
-// 1. Use AddCommand on an existing parser.
-// 2. Add a struct field to your options struct annotated with the
-// command:"command-name" tag.
-//
-// The most common, idiomatic way to implement commands is to define a global
-// parser instance and implement each command in a separate file. These
-// command files should define a go init function which calls AddCommand on
-// the global parser.
-//
-// When parsing ends and there is an active command and that command implements
-// the Commander interface, then its Execute method will be run with the
-// remaining command line arguments.
-//
-// Command structs can have options which become valid to parse after the
-// command has been specified on the command line. It is currently not valid
-// to specify options from the parent level of the command after the command
-// name has occurred. Thus, given a top-level option "-v" and a command "add":
-//
-// Valid: ./app -v add
-// Invalid: ./app add -v
-//
+/*
+Package flags provides an extensive command line option parser.
+The flags package is similar in functionality to the go built-in flag package
+but provides more options and uses reflection to provide a convenient and
+succinct way of specifying command line options.
+
+
+Supported features
+
+The following features are supported in go-flags:
+
+ Options with short names (-v)
+ Options with long names (--verbose)
+ Options with and without arguments (bool v.s. other type)
+ Options with optional arguments and default values
+ Option default values from ENVIRONMENT_VARIABLES, including slice and map values
+ Multiple option groups each containing a set of options
+ Generate and print well-formatted help message
+ Passing remaining command line arguments after -- (optional)
+ Ignoring unknown command line options (optional)
+ Supports -I/usr/include -I=/usr/include -I /usr/include option argument specification
+ Supports multiple short options -aux
+ Supports all primitive go types (string, int{8..64}, uint{8..64}, float)
+ Supports same option multiple times (can store in slice or last option counts)
+ Supports maps
+ Supports function callbacks
+ Supports namespaces for (nested) option groups
+
+Additional features specific to Windows:
+ Options with short names (/v)
+ Options with long names (/verbose)
+ Windows-style options with arguments use a colon as the delimiter
+ Modify generated help message with Windows-style / options
+
+
+Basic usage
+
+The flags package uses structs, reflection and struct field tags
+to allow users to specify command line options. This results in very simple
+and concise specification of your application options. For example:
+
+ type Options struct {
+ Verbose []bool `short:"v" long:"verbose" description:"Show verbose debug information"`
+ }
+
+This specifies one option with a short name -v and a long name --verbose.
+When either -v or --verbose is found on the command line, a 'true' value
+will be appended to the Verbose field. e.g. when specifying -vvv, the
+resulting value of Verbose will be {[true, true, true]}.
+
+Slice options work exactly the same as primitive type options, except that
+whenever the option is encountered, a value is appended to the slice.
+
+Map options from string to primitive type are also supported. On the command
+line, you specify the value for such an option as key:value. For example
+
+ type Options struct {
+ AuthorInfo string[string] `short:"a"`
+ }
+
+Then, the AuthorInfo map can be filled with something like
+-a name:Jesse -a "surname:van den Kieboom".
+
+Finally, for full control over the conversion between command line argument
+values and options, user defined types can choose to implement the Marshaler
+and Unmarshaler interfaces.
+
+
+Available field tags
+
+The following is a list of tags for struct fields supported by go-flags:
+
+ short: the short name of the option (single character)
+ long: the long name of the option
+ required: whether an option is required to appear on the command
+ line. If a required option is not present, the parser will
+ return ErrRequired (optional)
+ description: the description of the option (optional)
+ long-description: the long description of the option. Currently only
+ displayed in generated man pages (optional)
+ no-flag: if non-empty this field is ignored as an option (optional)
+
+ optional: whether an argument of the option is optional (optional)
+ optional-value: the value of an optional option when the option occurs
+ without an argument. This tag can be specified multiple
+ times in the case of maps or slices (optional)
+ default: the default value of an option. This tag can be specified
+ multiple times in the case of slices or maps (optional)
+ default-mask: when specified, this value will be displayed in the help
+ instead of the actual default value. This is useful
+ mostly for hiding otherwise sensitive information from
+ showing up in the help. If default-mask takes the special
+ value "-", then no default value will be shown at all
+ (optional)
+ env: the default value of the option is overridden from the
+ specified environment variable, if one has been defined.
+ (optional)
+ env-delim: the 'env' default value from environment is split into
+ multiple values with the given delimiter string, use with
+ slices and maps (optional)
+ value-name: the name of the argument value (to be shown in the help,
+ (optional)
+
+ base: a base (radix) used to convert strings to integer values, the
+ default base is 10 (i.e. decimal) (optional)
+
+ ini-name: the explicit ini option name (optional)
+ no-ini: if non-empty this field is ignored as an ini option
+ (optional)
+
+ group: when specified on a struct field, makes the struct
+ field a separate group with the given name (optional)
+ namespace: when specified on a group struct field, the namespace
+ gets prepended to every option's long name and
+ subgroup's namespace of this group, separated by
+ the parser's namespace delimiter (optional)
+ command: when specified on a struct field, makes the struct
+ field a (sub)command with the given name (optional)
+ subcommands-optional: when specified on a command struct field, makes
+ any subcommands of that command optional (optional)
+ alias: when specified on a command struct field, adds the
+ specified name as an alias for the command. Can be
+ be specified multiple times to add more than one
+ alias (optional)
+ positional-args: when specified on a field with a struct type,
+ uses the fields of that struct to parse remaining
+ positional command line arguments into (in order
+ of the fields). If a field has a slice type,
+ then all remaining arguments will be added to it.
+ Positional arguments are optional by default,
+ unless the "required" tag is specified together
+ with the "positional-args" tag (optional)
+
+Either the `short:` tag or the `long:` must be specified to make the field eligible as an
+option.
+
+
+Option groups
+
+Option groups are a simple way to semantically separate your options. All
+options in a particular group are shown together in the help under the name
+of the group. Namespaces can be used to specify option long names more
+precisely and emphasize the options affiliation to their group.
+
+There are currently three ways to specify option groups.
+
+ 1. Use NewNamedParser specifying the various option groups.
+ 2. Use AddGroup to add a group to an existing parser.
+ 3. Add a struct field to the top-level options annotated with the
+ group:"group-name" tag.
+
+
+
+Commands
+
+The flags package also has basic support for commands. Commands are often
+used in monolithic applications that support various commands or actions.
+Take git for example, all of the add, commit, checkout, etc. are called
+commands. Using commands you can easily separate multiple functions of your
+application.
+
+There are currently two ways to specify a command.
+
+ 1. Use AddCommand on an existing parser.
+ 2. Add a struct field to your options struct annotated with the
+ command:"command-name" tag.
+
+The most common, idiomatic way to implement commands is to define a global
+parser instance and implement each command in a separate file. These
+command files should define a go init function which calls AddCommand on
+the global parser.
+
+When parsing ends and there is an active command and that command implements
+the Commander interface, then its Execute method will be run with the
+remaining command line arguments.
+
+Command structs can have options which become valid to parse after the
+command has been specified on the command line. It is currently not valid
+to specify options from the parent level of the command after the command
+name has occurred. Thus, given a top-level option "-v" and a command "add":
+
+ Valid: ./app -v add
+ Invalid: ./app add -v
+
+
+Completion
+
+go-flags has builtin support to provide bash completion of flags, commands
+and argument values. To use completion, the binary which uses go-flags
+can be invoked in a special environment to list completion of the current
+command line argument. It should be noted that this `executes` your application,
+and it is up to the user to make sure there are no negative side effects (for
+example from init functions).
+
+Setting the environment variable `GO_FLAGS_COMPLETION=1` enables completion
+by replacing the argument parsing routine with the completion routine which
+outputs completions for the passed arguments. The basic invocation to
+complete a set of arguments is therefore:
+
+ GO_FLAGS_COMPLETION=1 ./completion-example arg1 arg2 arg3
+
+where `completion-example` is the binary, `arg1` and `arg2` are
+the current arguments, and `arg3` (the last argument) is the argument
+to be completed. If the GO_FLAGS_COMPLETION is set to "verbose", then
+descriptions of possible completion items will also be shown, if there
+are more than 1 completion items.
+
+To use this with bash completion, a simple file can be written which
+calls the binary which supports go-flags completion:
+
+ _completion_example() {
+ # All arguments except the first one
+ args=("${COMP_WORDS[@]:1:$COMP_CWORD}")
+
+ # Only split on newlines
+ local IFS=$'\n'
+
+ # Call completion (note that the first element of COMP_WORDS is
+ # the executable itself)
+ COMPREPLY=($(GO_FLAGS_COMPLETION=1 ${COMP_WORDS[0]} "${args[@]}"))
+ return 0
+ }
+
+ complete -F _completion_example completion-example
+
+Completion requires the parser option PassDoubleDash and is therefore enforced if the environment variable GO_FLAGS_COMPLETION is set.
+
+Customized completion for argument values is supported by implementing
+the flags.Completer interface for the argument value type. An example
+of a type which does so is the flags.Filename type, an alias of string
+allowing simple filename completion. A slice or array argument value
+whose element type implements flags.Completer will also be completed.
+*/
package flags
diff --git a/vendor/src/github.com/jessevdk/go-flags/group.go b/vendor/src/github.com/jessevdk/go-flags/group.go
index a2b368d1937..8b609a3af7b 100644
--- a/vendor/src/github.com/jessevdk/go-flags/group.go
+++ b/vendor/src/github.com/jessevdk/go-flags/group.go
@@ -29,6 +29,12 @@ type Group struct {
// (Command embeds Group) in the built-in generated help and man pages.
LongDescription string
+ // The namespace of the group
+ Namespace string
+
+ // The parent of the group or nil if it has no parent
+ parent interface{}
+
// All the options in the group
options []*Option
@@ -47,6 +53,8 @@ type Group struct {
func (g *Group) AddGroup(shortDescription string, longDescription string, data interface{}) (*Group, error) {
group := newGroup(shortDescription, longDescription, data)
+ group.parent = g
+
if err := group.scan(); err != nil {
return nil, err
}
diff --git a/vendor/src/github.com/jessevdk/go-flags/group_private.go b/vendor/src/github.com/jessevdk/go-flags/group_private.go
index 5242f5dae8b..15251ce39c3 100644
--- a/vendor/src/github.com/jessevdk/go-flags/group_private.go
+++ b/vendor/src/github.com/jessevdk/go-flags/group_private.go
@@ -32,7 +32,7 @@ func (g *Group) optionByName(name string, namematch func(*Option, string) bool)
prio = 3
}
- if name == opt.LongName && prio < 2 {
+ if name == opt.LongNameWithNamespace() && prio < 2 {
retopt = opt
prio = 2
}
@@ -46,33 +46,6 @@ func (g *Group) optionByName(name string, namematch func(*Option, string) bool)
return retopt
}
-func (g *Group) storeDefaults() {
- for _, option := range g.options {
- // First. empty out the value
- if len(option.Default) > 0 {
- option.clear()
- }
-
- for _, d := range option.Default {
- option.set(&d)
- }
-
- if !option.value.CanSet() {
- continue
- }
-
- if option.value.Kind() == reflect.Map {
- option.defaultValue = reflect.MakeMap(option.value.Type())
-
- for _, k := range option.value.MapKeys() {
- option.defaultValue.SetMapIndex(k, option.value.MapIndex(k))
- }
- } else {
- option.defaultValue = reflect.ValueOf(option.value.Interface())
- }
- }
-}
-
func (g *Group) eachGroup(f func(*Group)) {
f(g)
@@ -151,6 +124,7 @@ func (g *Group) scanStruct(realval reflect.Value, sfield *reflect.StructField, h
description := mtag.Get("description")
def := mtag.GetMany("default")
+
optionalValue := mtag.GetMany("optional-value")
valueName := mtag.Get("value-name")
defaultMask := mtag.Get("default-mask")
@@ -163,12 +137,16 @@ func (g *Group) scanStruct(realval reflect.Value, sfield *reflect.StructField, h
ShortName: short,
LongName: longname,
Default: def,
+ EnvDefaultKey: mtag.Get("env"),
+ EnvDefaultDelim: mtag.Get("env-delim"),
OptionalArgument: optional,
OptionalValue: optionalValue,
Required: required,
ValueName: valueName,
DefaultMask: defaultMask,
+ group: g,
+
field: field,
value: realval.Field(i),
tag: mtag,
@@ -189,11 +167,13 @@ func (g *Group) checkForDuplicateFlags() *Error {
g.eachGroup(func(g *Group) {
for _, option := range g.options {
if option.LongName != "" {
- if otherOption, ok := longNames[option.LongName]; ok {
+ longName := option.LongNameWithNamespace()
+
+ if otherOption, ok := longNames[longName]; ok {
duplicateError = newErrorf(ErrDuplicatedFlag, "option `%s' uses the same long name as option `%s'", option, otherOption)
return
}
- longNames[option.LongName] = option
+ longNames[longName] = option
}
if option.ShortName != 0 {
if otherOption, ok := shortNames[option.ShortName]; ok {
@@ -221,10 +201,13 @@ func (g *Group) scanSubGroupHandler(realval reflect.Value, sfield *reflect.Struc
ptrval := reflect.NewAt(realval.Type(), unsafe.Pointer(realval.UnsafeAddr()))
description := mtag.Get("description")
- if _, err := g.AddGroup(subgroup, description, ptrval.Interface()); err != nil {
+ group, err := g.AddGroup(subgroup, description, ptrval.Interface())
+ if err != nil {
return true, err
}
+ group.Namespace = mtag.Get("namespace")
+
return true, nil
}
diff --git a/vendor/src/github.com/jessevdk/go-flags/group_test.go b/vendor/src/github.com/jessevdk/go-flags/group_test.go
index 35d0767aca6..b5ed9d492d5 100644
--- a/vendor/src/github.com/jessevdk/go-flags/group_test.go
+++ b/vendor/src/github.com/jessevdk/go-flags/group_test.go
@@ -113,6 +113,33 @@ func TestGroupNestedInline(t *testing.T) {
}
}
+func TestGroupNestedInlineNamespace(t *testing.T) {
+ var opts = struct {
+ Opt string `long:"opt"`
+
+ Group struct {
+ Opt string `long:"opt"`
+ Group struct {
+ Opt string `long:"opt"`
+ } `group:"Subsubgroup" namespace:"sap"`
+ } `group:"Subgroup" namespace:"sip"`
+ }{}
+
+ p, ret := assertParserSuccess(t, &opts, "--opt", "a", "--sip.opt", "b", "--sip.sap.opt", "c", "rest")
+
+ assertStringArray(t, ret, []string{"rest"})
+
+ assertString(t, opts.Opt, "a")
+ assertString(t, opts.Group.Opt, "b")
+ assertString(t, opts.Group.Group.Opt, "c")
+
+ for _, name := range []string{"Subgroup", "Subsubgroup"} {
+ if p.Command.Group.Find(name) == nil {
+ t.Errorf("Expected to find group '%s'", name)
+ }
+ }
+}
+
func TestDuplicateShortFlags(t *testing.T) {
var opts struct {
Verbose []bool `short:"v" long:"verbose" description:"Show verbose debug information"`
diff --git a/vendor/src/github.com/jessevdk/go-flags/help.go b/vendor/src/github.com/jessevdk/go-flags/help.go
index 4ded286d03b..e26fcd01cd4 100644
--- a/vendor/src/github.com/jessevdk/go-flags/help.go
+++ b/vendor/src/github.com/jessevdk/go-flags/help.go
@@ -10,6 +10,7 @@ import (
"fmt"
"io"
"reflect"
+ "runtime"
"strings"
"unicode/utf8"
)
@@ -22,6 +23,41 @@ type alignmentInfo struct {
indent bool
}
+const (
+ paddingBeforeOption = 2
+ distanceBetweenOptionAndDescription = 2
+)
+
+func (a *alignmentInfo) descriptionStart() int {
+ ret := a.maxLongLen + distanceBetweenOptionAndDescription
+
+ if a.hasShort {
+ ret += 2
+ }
+
+ if a.maxLongLen > 0 {
+ ret += 4
+ }
+
+ if a.hasValueName {
+ ret += 3
+ }
+
+ return ret
+}
+
+func (a *alignmentInfo) updateLen(name string, indent bool) {
+ l := utf8.RuneCountInString(name)
+
+ if indent {
+ l = l + 4
+ }
+
+ if l > a.maxLongLen {
+ a.maxLongLen = l
+ }
+}
+
func (p *Parser) getAlignmentInfo() alignmentInfo {
ret := alignmentInfo{
maxLongLen: 0,
@@ -34,7 +70,15 @@ func (p *Parser) getAlignmentInfo() alignmentInfo {
ret.terminalColumns = 80
}
+ var prevcmd *Command
+
p.eachActiveGroup(func(c *Command, grp *Group) {
+ if c != prevcmd {
+ for _, arg := range c.args {
+ ret.updateLen(arg.Name, c != p.Command)
+ }
+ }
+
for _, info := range grp.options {
if !info.canCli() {
continue
@@ -44,22 +88,11 @@ func (p *Parser) getAlignmentInfo() alignmentInfo {
ret.hasShort = true
}
- lv := utf8.RuneCountInString(info.ValueName)
-
- if lv != 0 {
+ if len(info.ValueName) > 0 {
ret.hasValueName = true
}
- l := utf8.RuneCountInString(info.LongName) + lv
-
- if c != p.Command {
- // for indenting
- l = l + 4
- }
-
- if l > ret.maxLongLen {
- ret.maxLongLen = l
- }
+ ret.updateLen(info.LongNameWithNamespace()+info.ValueName, c != p.Command)
}
})
@@ -69,9 +102,6 @@ func (p *Parser) getAlignmentInfo() alignmentInfo {
func (p *Parser) writeHelpOption(writer *bufio.Writer, option *Option, info alignmentInfo) {
line := &bytes.Buffer{}
- distanceBetweenOptionAndDescription := 2
- paddingBeforeOption := 2
-
prefix := paddingBeforeOption
if info.indent {
@@ -87,19 +117,7 @@ func (p *Parser) writeHelpOption(writer *bufio.Writer, option *Option, info alig
line.WriteString(" ")
}
- descstart := info.maxLongLen + paddingBeforeOption + distanceBetweenOptionAndDescription
-
- if info.hasShort {
- descstart += 2
- }
-
- if info.maxLongLen > 0 {
- descstart += 4
- }
-
- if info.hasValueName {
- descstart += 3
- }
+ descstart := info.descriptionStart() + paddingBeforeOption
if len(option.LongName) > 0 {
if option.ShortName != 0 {
@@ -109,7 +127,7 @@ func (p *Parser) writeHelpOption(writer *bufio.Writer, option *Option, info alig
}
line.WriteString(defaultLongOptDelimiter)
- line.WriteString(option.LongName)
+ line.WriteString(option.LongNameWithNamespace())
}
if option.canArgument() {
@@ -153,15 +171,32 @@ func (p *Parser) writeHelpOption(writer *bufio.Writer, option *Option, info alig
def, _ = convertToString(option.value, option.tag)
}
} else if len(defs) != 0 {
- def = strings.Join(defs, ", ")
+ l := len(defs) - 1
+
+ for i := 0; i < l; i++ {
+ def += quoteIfNeeded(defs[i]) + ", "
+ }
+
+ def += quoteIfNeeded(defs[l])
+ }
+
+ var envDef string
+ if option.EnvDefaultKey != "" {
+ var envPrintable string
+ if runtime.GOOS == "windows" {
+ envPrintable = "%" + option.EnvDefaultKey + "%"
+ } else {
+ envPrintable = "$" + option.EnvDefaultKey
+ }
+ envDef = fmt.Sprintf(" [%s]", envPrintable)
}
var desc string
if def != "" {
- desc = fmt.Sprintf("%s (%v)", option.Description, def)
+ desc = fmt.Sprintf("%s (%v)%s", option.Description, def, envDef)
} else {
- desc = option.Description
+ desc = option.Description + envDef
}
writer.WriteString(wrapText(desc,
@@ -236,6 +271,28 @@ func (p *Parser) WriteHelp(writer io.Writer) {
fmt.Fprintf(wr, " %s", allcmd.Name)
}
+ if len(allcmd.args) > 0 {
+ fmt.Fprintf(wr, " ")
+ }
+
+ for i, arg := range allcmd.args {
+ if i != 0 {
+ fmt.Fprintf(wr, " ")
+ }
+
+ name := arg.Name
+
+ if arg.isRemaining() {
+ name = name + "..."
+ }
+
+ if !allcmd.ArgsRequired {
+ fmt.Fprintf(wr, "[%s]", name)
+ } else {
+ fmt.Fprintf(wr, "%s", name)
+ }
+ }
+
if allcmd.Active == nil && len(allcmd.commands) > 0 {
var co, cc string
@@ -275,26 +332,32 @@ func (p *Parser) WriteHelp(writer io.Writer) {
}
}
- prevcmd := p.Command
+ c := p.Command
- p.eachActiveGroup(func(c *Command, grp *Group) {
- first := true
+ for c != nil {
+ printcmd := c != p.Command
- // Skip built-in help group for all commands except the top-level
- // parser
- if grp.isBuiltinHelp && c != p.Command {
- return
- }
+ c.eachGroup(func(grp *Group) {
+ first := true
- for _, info := range grp.options {
- if info.canCli() {
- if prevcmd != c {
+ // Skip built-in help group for all commands except the top-level
+ // parser
+ if grp.isBuiltinHelp && c != p.Command {
+ return
+ }
+
+ for _, info := range grp.options {
+ if !info.canCli() {
+ continue
+ }
+
+ if printcmd {
fmt.Fprintf(wr, "\n[%s command options]\n", c.Name)
- prevcmd = c
aligninfo.indent = true
+ printcmd = false
}
- if first && prevcmd.Group != grp {
+ if first && cmd.Group != grp {
fmt.Fprintln(wr)
if aligninfo.indent {
@@ -307,8 +370,32 @@ func (p *Parser) WriteHelp(writer io.Writer) {
p.writeHelpOption(wr, info, aligninfo)
}
+ })
+
+ if len(c.args) > 0 {
+ if c == p.Command {
+ fmt.Fprintf(wr, "\nArguments:\n")
+ } else {
+ fmt.Fprintf(wr, "\n[%s command arguments]\n", c.Name)
+ }
+
+ maxlen := aligninfo.descriptionStart()
+
+ for _, arg := range c.args {
+ prefix := strings.Repeat(" ", paddingBeforeOption)
+ fmt.Fprintf(wr, "%s%s", prefix, arg.Name)
+
+ if len(arg.Description) > 0 {
+ align := strings.Repeat(" ", maxlen-len(arg.Name)-1)
+ fmt.Fprintf(wr, ":%s%s", align, arg.Description)
+ }
+
+ fmt.Fprintln(wr)
+ }
}
- })
+
+ c = c.Active
+ }
scommands := cmd.sortedCommands()
diff --git a/vendor/src/github.com/jessevdk/go-flags/help_test.go b/vendor/src/github.com/jessevdk/go-flags/help_test.go
index e3e84d8dc26..32220fbe48e 100644
--- a/vendor/src/github.com/jessevdk/go-flags/help_test.go
+++ b/vendor/src/github.com/jessevdk/go-flags/help_test.go
@@ -3,52 +3,23 @@ package flags
import (
"bytes"
"fmt"
- "io"
- "io/ioutil"
"os"
- "os/exec"
+ "runtime"
"testing"
"time"
)
-func helpDiff(a, b string) (string, error) {
- atmp, err := ioutil.TempFile("", "help-diff")
-
- if err != nil {
- return "", err
- }
-
- btmp, err := ioutil.TempFile("", "help-diff")
-
- if err != nil {
- return "", err
- }
-
- if _, err := io.WriteString(atmp, a); err != nil {
- return "", err
- }
-
- if _, err := io.WriteString(btmp, b); err != nil {
- return "", err
- }
-
- ret, err := exec.Command("diff", "-u", "-d", "--label", "got", atmp.Name(), "--label", "expected", btmp.Name()).Output()
-
- os.Remove(atmp.Name())
- os.Remove(btmp.Name())
-
- return string(ret), nil
-}
-
type helpOptions struct {
Verbose []bool `short:"v" long:"verbose" description:"Show verbose debug information" ini-name:"verbose"`
Call func(string) `short:"c" description:"Call phone number" ini-name:"call"`
PtrSlice []*string `long:"ptrslice" description:"A slice of pointers to string"`
EmptyDescription bool `long:"empty-description"`
- Default string `long:"default" default:"Some value" description:"Test default value"`
- DefaultArray []string `long:"default-array" default:"Some value" default:"Another value" description:"Test default array value"`
+ Default string `long:"default" default:"Some\nvalue" description:"Test default value"`
+ DefaultArray []string `long:"default-array" default:"Some value" default:"Other\tvalue" description:"Test default array value"`
DefaultMap map[string]string `long:"default-map" default:"some:value" default:"another:value" description:"Testdefault map value"`
+ EnvDefault1 string `long:"env-default1" default:"Some value" env:"ENV_DEFAULT" description:"Test env-default1 value"`
+ EnvDefault2 string `long:"env-default2" env:"ENV_DEFAULT" description:"Test env-default2 value"`
OnlyIni string `ini-name:"only-ini" description:"Option only available in ini"`
@@ -57,14 +28,30 @@ type helpOptions struct {
IntMap map[string]int `long:"intmap" default:"a:1" description:"A map from string to int" ini-name:"int-map"`
} `group:"Other Options"`
+ Group struct {
+ Opt string `long:"opt" description:"This is a subgroup option"`
+
+ Group struct {
+ Opt string `long:"opt" description:"This is a subsubgroup option"`
+ } `group:"Subsubgroup" namespace:"sap"`
+ } `group:"Subgroup" namespace:"sip"`
+
Command struct {
ExtraVerbose []bool `long:"extra-verbose" description:"Use for extra verbosity"`
} `command:"command" alias:"cm" alias:"cmd" description:"A command"`
+
+ Args struct {
+ Filename string `name:"filename" description:"A filename"`
+ Num int `name:"num" description:"A number"`
+ } `positional-args:"yes"`
}
func TestHelp(t *testing.T) {
- var opts helpOptions
+ oldEnv := EnvSnapshot()
+ defer oldEnv.Restore()
+ os.Setenv("ENV_DEFAULT", "env-def")
+ var opts helpOptions
p := NewNamedParser("TestHelp", HelpFlag)
p.AddGroup("Application Options", "The application options", &opts)
@@ -81,45 +68,91 @@ func TestHelp(t *testing.T) {
t.Errorf("Expected flags.ErrHelp type, but got %s", e.Type)
}
- expected := `Usage:
- TestHelp [OPTIONS] <command>
+ var expected string
+
+ if runtime.GOOS == "windows" {
+ expected = `Usage:
+ TestHelp [OPTIONS] [filename] [num] <command>
+
+Application Options:
+ /v, /verbose Show verbose debug information
+ /c: Call phone number
+ /ptrslice: A slice of pointers to string
+ /empty-description
+ /default: Test default value ("Some\nvalue")
+ /default-array: Test default array value (Some value, "Other\tvalue")
+ /default-map: Testdefault map value (some:value, another:value)
+ /env-default1: Test env-default1 value (Some value) [%ENV_DEFAULT%]
+ /env-default2: Test env-default2 value [%ENV_DEFAULT%]
+
+Other Options:
+ /s: A slice of strings (some, value)
+ /intmap: A map from string to int (a:1)
+
+Subgroup:
+ /sip.opt: This is a subgroup option
+
+Subsubgroup:
+ /sip.sap.opt: This is a subsubgroup option
+
+Help Options:
+ /? Show this help message
+ /h, /help Show this help message
+
+Arguments:
+ filename: A filename
+ num: A number
+
+Available commands:
+ command A command (aliases: cm, cmd)
+`
+ } else {
+ expected = `Usage:
+ TestHelp [OPTIONS] [filename] [num] <command>
Application Options:
-v, --verbose Show verbose debug information
-c= Call phone number
--ptrslice= A slice of pointers to string
--empty-description
- --default= Test default value (Some value)
- --default-array= Test default array value (Some value, Another value)
+ --default= Test default value ("Some\nvalue")
+ --default-array= Test default array value (Some value, "Other\tvalue")
--default-map= Testdefault map value (some:value, another:value)
+ --env-default1= Test env-default1 value (Some value) [$ENV_DEFAULT]
+ --env-default2= Test env-default2 value [$ENV_DEFAULT]
Other Options:
-s= A slice of strings (some, value)
--intmap= A map from string to int (a:1)
+Subgroup:
+ --sip.opt= This is a subgroup option
+
+Subsubgroup:
+ --sip.sap.opt= This is a subsubgroup option
+
Help Options:
-h, --help Show this help message
+Arguments:
+ filename: A filename
+ num: A number
+
Available commands:
command A command (aliases: cm, cmd)
`
-
- if e.Message != expected {
- ret, err := helpDiff(e.Message, expected)
-
- if err != nil {
- t.Errorf("Unexpected diff error: %s", err)
- t.Errorf("Unexpected help message, expected:\n\n%s\n\nbut got\n\n%s", expected, e.Message)
- } else {
- t.Errorf("Unexpected help message:\n\n%s", ret)
- }
}
+
+ assertDiff(t, e.Message, expected, "help message")
}
}
func TestMan(t *testing.T) {
- var opts helpOptions
+ oldEnv := EnvSnapshot()
+ defer oldEnv.Restore()
+ os.Setenv("ENV_DEFAULT", "env-def")
+ var opts helpOptions
p := NewNamedParser("TestMan", HelpFlag)
p.ShortDescription = "Test manpage generation"
p.LongDescription = "This is a somewhat `longer' description of what this does"
@@ -163,17 +196,32 @@ Test default array value
\fB--default-map\fP
Testdefault map value
.TP
+\fB--env-default1\fP
+Test env-default1 value
+.TP
+\fB--env-default2\fP
+Test env-default2 value
+.TP
\fB-s\fP
A slice of strings
.TP
\fB--intmap\fP
A map from string to int
+.TP
+\fB--sip.opt\fP
+This is a subgroup option
+.TP
+\fB--sip.sap.opt\fP
+This is a subsubgroup option
.SH COMMANDS
.SS command
A command
Longer \fBcommand\fP description
+\fBUsage\fP: TestMan [OPTIONS] command [command-OPTIONS]
+
+
\fBAliases\fP: cm, cmd
.TP
@@ -181,15 +229,7 @@ Longer \fBcommand\fP description
Use for extra verbosity
`, tt.Format("2 January 2006"))
- if got != expected {
- ret, err := helpDiff(got, expected)
-
- if err != nil {
- t.Errorf("Unexpected man page, expected:\n\n%s\n\nbut got\n\n%s", expected, got)
- } else {
- t.Errorf("Unexpected man page:\n\n%s", ret)
- }
- }
+ assertDiff(t, got, expected, "man page")
}
type helpCommandNoOptions struct {
@@ -198,8 +238,11 @@ type helpCommandNoOptions struct {
}
func TestHelpCommand(t *testing.T) {
- var opts helpCommandNoOptions
+ oldEnv := EnvSnapshot()
+ defer oldEnv.Restore()
+ os.Setenv("ENV_DEFAULT", "env-def")
+ var opts helpCommandNoOptions
p := NewNamedParser("TestHelpCommand", HelpFlag)
p.AddGroup("Application Options", "The application options", &opts)
@@ -216,22 +259,25 @@ func TestHelpCommand(t *testing.T) {
t.Errorf("Expected flags.ErrHelp type, but got %s", e.Type)
}
- expected := `Usage:
+ var expected string
+
+ if runtime.GOOS == "windows" {
+ expected = `Usage:
TestHelpCommand [OPTIONS] command
Help Options:
- -h, --help Show this help message
+ /? Show this help message
+ /h, /help Show this help message
`
+ } else {
+ expected = `Usage:
+ TestHelpCommand [OPTIONS] command
- if e.Message != expected {
- ret, err := helpDiff(e.Message, expected)
-
- if err != nil {
- t.Errorf("Unexpected diff error: %s", err)
- t.Errorf("Unexpected help message, expected:\n\n%s\n\nbut got\n\n%s", expected, e.Message)
- } else {
- t.Errorf("Unexpected help message:\n\n%s", ret)
- }
+Help Options:
+ -h, --help Show this help message
+`
}
+
+ assertDiff(t, e.Message, expected, "help message")
}
}
diff --git a/vendor/src/github.com/jessevdk/go-flags/ini.go b/vendor/src/github.com/jessevdk/go-flags/ini.go
index f92faff482e..72250522559 100644
--- a/vendor/src/github.com/jessevdk/go-flags/ini.go
+++ b/vendor/src/github.com/jessevdk/go-flags/ini.go
@@ -76,7 +76,7 @@ func IniParse(filename string, data interface{}) error {
// information on the ini file format. The returned errors can be of the type
// flags.Error or flags.IniError.
func (i *IniParser) ParseFile(filename string) error {
- i.parser.storeDefaults()
+ i.parser.clearIsSet()
ini, err := readIniFromFile(filename)
@@ -112,7 +112,7 @@ func (i *IniParser) ParseFile(filename string) error {
//
// The returned errors can be of the type flags.Error or flags.IniError.
func (i *IniParser) Parse(reader io.Reader) error {
- i.parser.storeDefaults()
+ i.parser.clearIsSet()
ini, err := readIni(reader, "")
diff --git a/vendor/src/github.com/jessevdk/go-flags/ini_private.go b/vendor/src/github.com/jessevdk/go-flags/ini_private.go
index 64f8fb8d950..887aa767984 100644
--- a/vendor/src/github.com/jessevdk/go-flags/ini_private.go
+++ b/vendor/src/github.com/jessevdk/go-flags/ini_private.go
@@ -6,16 +6,23 @@ import (
"io"
"os"
"reflect"
+ "sort"
+ "strconv"
"strings"
)
type iniValue struct {
- Name string
- Value string
+ Name string
+ Value string
+ Quoted bool
+ LineNumber uint
}
type iniSection []iniValue
-type ini map[string]iniSection
+type ini struct {
+ File string
+ Sections map[string]iniSection
+}
func readFullLine(reader *bufio.Reader) (string, error) {
var line []byte
@@ -57,13 +64,19 @@ func optionIniName(option *Option) string {
return option.field.Name
}
-func writeGroupIni(group *Group, namespace string, writer io.Writer, options IniOptions) {
+func writeGroupIni(cmd *Command, group *Group, namespace string, writer io.Writer, options IniOptions) {
var sname string
if len(namespace) != 0 {
- sname = namespace + "." + group.ShortDescription
- } else {
- sname = group.ShortDescription
+ sname = namespace
+ }
+
+ if cmd.Group != group && len(group.ShortDescription) != 0 {
+ if len(sname) != 0 {
+ sname += "."
+ }
+
+ sname += group.ShortDescription
}
sectionwritten := false
@@ -80,7 +93,7 @@ func writeGroupIni(group *Group, namespace string, writer io.Writer, options Ini
val := option.value
- if (options&IniIncludeDefaults) == IniNone && reflect.DeepEqual(val.Interface(), option.defaultValue.Interface()) {
+ if (options&IniIncludeDefaults) == IniNone && option.valueIsDefault() {
continue
}
@@ -95,40 +108,49 @@ func writeGroupIni(group *Group, namespace string, writer io.Writer, options Ini
oname := optionIniName(option)
- commentOption := ""
- if (options&(IniIncludeDefaults|IniCommentDefaults)) == IniIncludeDefaults|IniCommentDefaults && reflect.DeepEqual(val.Interface(), option.defaultValue.Interface()) {
- commentOption = "; "
- }
+ commentOption := (options&(IniIncludeDefaults|IniCommentDefaults)) == IniIncludeDefaults|IniCommentDefaults && option.valueIsDefault()
- switch val.Type().Kind() {
+ kind := val.Type().Kind()
+ switch kind {
case reflect.Slice:
- for idx := 0; idx < val.Len(); idx++ {
- v, _ := convertToString(val.Index(idx), option.tag)
- fmt.Fprintf(writer, "%s%s = %s\n", commentOption, oname, v)
- }
+ kind = val.Type().Elem().Kind()
if val.Len() == 0 {
- fmt.Fprintf(writer, "; %s =\n", oname)
- }
- case reflect.Map:
- for _, key := range val.MapKeys() {
- k, _ := convertToString(key, option.tag)
- v, _ := convertToString(val.MapIndex(key), option.tag)
+ writeOption(writer, oname, kind, "", "", true, option.iniQuote)
+ } else {
+ for idx := 0; idx < val.Len(); idx++ {
+ v, _ := convertToString(val.Index(idx), option.tag)
- fmt.Fprintf(writer, "%s%s = %s:%s\n", commentOption, oname, k, v)
+ writeOption(writer, oname, kind, "", v, commentOption, option.iniQuote)
+ }
}
+ case reflect.Map:
+ kind = val.Type().Elem().Kind()
if val.Len() == 0 {
- fmt.Fprintf(writer, "; %s =\n", oname)
+ writeOption(writer, oname, kind, "", "", true, option.iniQuote)
+ } else {
+ mkeys := val.MapKeys()
+ keys := make([]string, len(val.MapKeys()))
+ kkmap := make(map[string]reflect.Value)
+
+ for i, k := range mkeys {
+ keys[i], _ = convertToString(k, option.tag)
+ kkmap[keys[i]] = k
+ }
+
+ sort.Strings(keys)
+
+ for _, k := range keys {
+ v, _ := convertToString(val.MapIndex(kkmap[k]), option.tag)
+
+ writeOption(writer, oname, kind, k, v, commentOption, option.iniQuote)
+ }
}
default:
v, _ := convertToString(val, option.tag)
- if len(v) != 0 {
- fmt.Fprintf(writer, "%s%s = %s\n", commentOption, oname, v)
- } else {
- fmt.Fprintf(writer, "%s%s =\n", commentOption, oname)
- }
+ writeOption(writer, oname, kind, "", v, commentOption, option.iniQuote)
}
if comments {
@@ -141,9 +163,30 @@ func writeGroupIni(group *Group, namespace string, writer io.Writer, options Ini
}
}
+func writeOption(writer io.Writer, optionName string, optionType reflect.Kind, optionKey string, optionValue string, commentOption bool, forceQuote bool) {
+ if forceQuote || (optionType == reflect.String && !isPrint(optionValue)) {
+ optionValue = strconv.Quote(optionValue)
+ }
+
+ comment := ""
+ if commentOption {
+ comment = "; "
+ }
+
+ fmt.Fprintf(writer, "%s%s =", comment, optionName)
+
+ if optionKey != "" {
+ fmt.Fprintf(writer, " %s:%s", optionKey, optionValue)
+ } else if optionValue != "" {
+ fmt.Fprintf(writer, " %s", optionValue)
+ }
+
+ fmt.Fprintln(writer)
+}
+
func writeCommandIni(command *Command, namespace string, writer io.Writer, options IniOptions) {
command.eachGroup(func(group *Group) {
- writeGroupIni(group, namespace, writer, options)
+ writeGroupIni(command, group, namespace, writer, options)
})
for _, c := range command.commands {
@@ -177,7 +220,7 @@ func writeIniToFile(parser *IniParser, filename string, options IniOptions) erro
return nil
}
-func readIniFromFile(filename string) (ini, error) {
+func readIniFromFile(filename string) (*ini, error) {
file, err := os.Open(filename)
if err != nil {
@@ -189,8 +232,11 @@ func readIniFromFile(filename string) (ini, error) {
return readIni(file, filename)
}
-func readIni(contents io.Reader, filename string) (ini, error) {
- ret := make(ini)
+func readIni(contents io.Reader, filename string) (*ini, error) {
+ ret := &ini{
+ File: filename,
+ Sections: make(map[string]iniSection),
+ }
reader := bufio.NewReader(contents)
@@ -198,7 +244,7 @@ func readIni(contents io.Reader, filename string) (ini, error) {
section := make(iniSection, 0, 10)
sectionname := ""
- ret[sectionname] = section
+ ret.Sections[sectionname] = section
var lineno uint
@@ -239,11 +285,11 @@ func readIni(contents io.Reader, filename string) (ini, error) {
}
sectionname = name
- section = ret[name]
+ section = ret.Sections[name]
if section == nil {
section = make(iniSection, 0, 10)
- ret[name] = section
+ ret.Sections[name] = section
}
continue
@@ -262,13 +308,30 @@ func readIni(contents io.Reader, filename string) (ini, error) {
name := strings.TrimSpace(keyval[0])
value := strings.TrimSpace(keyval[1])
+ quoted := false
+
+ if len(value) != 0 && value[0] == '"' {
+ if v, err := strconv.Unquote(value); err == nil {
+ value = v
+
+ quoted = true
+ } else {
+ return nil, &IniError{
+ Message: err.Error(),
+ File: filename,
+ LineNumber: lineno,
+ }
+ }
+ }
section = append(section, iniValue{
- Name: name,
- Value: value,
+ Name: name,
+ Value: value,
+ Quoted: quoted,
+ LineNumber: lineno,
})
- ret[sectionname] = section
+ ret.Sections[sectionname] = section
}
return ret, nil
@@ -294,17 +357,16 @@ func (i *IniParser) matchingGroups(name string) []*Group {
return nil
}
-func (i *IniParser) parse(ini ini) error {
+func (i *IniParser) parse(ini *ini) error {
p := i.parser
- for name, section := range ini {
+ var quotesLookup = make(map[*Option]bool)
+
+ for name, section := range ini.Sections {
groups := i.matchingGroups(name)
if len(groups) == 0 {
- return newError(
- ErrUnknownGroup,
- fmt.Sprintf("could not find option group `%s'", name),
- )
+ return newErrorf(ErrUnknownGroup, "could not find option group `%s'", name)
}
for _, inival := range section {
@@ -326,10 +388,11 @@ func (i *IniParser) parse(ini ini) error {
if opt == nil {
if (p.Options & IgnoreUnknown) == None {
- return newError(
- ErrUnknownFlag,
- fmt.Sprintf("unknown option: %s", inival.Name),
- )
+ return &IniError{
+ Message: fmt.Sprintf("unknown option: %s", inival.Name),
+ File: ini.File,
+ LineNumber: inival.LineNumber,
+ }
}
continue
@@ -339,15 +402,51 @@ func (i *IniParser) parse(ini ini) error {
if !opt.canArgument() && len(inival.Value) == 0 {
pval = nil
+ } else {
+ if opt.value.Type().Kind() == reflect.Map {
+ parts := strings.SplitN(inival.Value, ":", 2)
+
+ // only handle unquoting
+ if len(parts) == 2 && parts[1][0] == '"' {
+ if v, err := strconv.Unquote(parts[1]); err == nil {
+ parts[1] = v
+
+ inival.Quoted = true
+ } else {
+ return &IniError{
+ Message: err.Error(),
+ File: ini.File,
+ LineNumber: inival.LineNumber,
+ }
+ }
+
+ s := parts[0] + ":" + parts[1]
+
+ pval = &s
+ }
+ }
}
if err := opt.set(pval); err != nil {
- return wrapError(err)
+ return &IniError{
+ Message: err.Error(),
+ File: ini.File,
+ LineNumber: inival.LineNumber,
+ }
+ }
+
+ // either all INI values are quoted or only values who need quoting
+ if _, ok := quotesLookup[opt]; !inival.Quoted || !ok {
+ quotesLookup[opt] = inival.Quoted
}
opt.tag.Set("_read-ini-name", inival.Name)
}
}
+ for opt, quoted := range quotesLookup {
+ opt.iniQuote = quoted
+ }
+
return nil
}
diff --git a/vendor/src/github.com/jessevdk/go-flags/ini_test.go b/vendor/src/github.com/jessevdk/go-flags/ini_test.go
index b8fcd267e5a..8c3176c9489 100644
--- a/vendor/src/github.com/jessevdk/go-flags/ini_test.go
+++ b/vendor/src/github.com/jessevdk/go-flags/ini_test.go
@@ -2,26 +2,38 @@ package flags
import (
"bytes"
+ "fmt"
"io/ioutil"
"os"
+ "reflect"
"strings"
"testing"
)
func TestWriteIni(t *testing.T) {
+ oldEnv := EnvSnapshot()
+ defer oldEnv.Restore()
+ os.Setenv("ENV_DEFAULT", "env-def")
+
var tests = []struct {
args []string
options IniOptions
expected string
}{
{
- []string{"-vv", "--intmap=a:2", "--intmap", "b:3", "command"},
+ []string{"-vv", "--intmap=a:2", "--intmap", "b:3", "filename", "0", "command"},
IniDefault,
`[Application Options]
; Show verbose debug information
verbose = true
verbose = true
+; Test env-default1 value
+EnvDefault1 = env-def
+
+; Test env-default2 value
+EnvDefault2 = env-def
+
[Other Options]
; A map from string to int
int-map = a:2
@@ -30,7 +42,7 @@ int-map = b:3
`,
},
{
- []string{"-vv", "--intmap=a:2", "--intmap", "b:3", "command"},
+ []string{"-vv", "--intmap=a:2", "--intmap", "b:3", "filename", "0", "command"},
IniDefault | IniIncludeDefaults,
`[Application Options]
; Show verbose debug information
@@ -43,15 +55,21 @@ verbose = true
EmptyDescription = false
; Test default value
-Default = Some value
+Default = "Some\nvalue"
; Test default array value
DefaultArray = Some value
-DefaultArray = Another value
+DefaultArray = "Other\tvalue"
; Testdefault map value
-DefaultMap = some:value
DefaultMap = another:value
+DefaultMap = some:value
+
+; Test env-default1 value
+EnvDefault1 = env-def
+
+; Test env-default2 value
+EnvDefault2 = env-def
; Option only available in ini
only-ini =
@@ -65,14 +83,22 @@ StringSlice = value
int-map = a:2
int-map = b:3
-[command.A command]
+[Subgroup]
+; This is a subgroup option
+Opt =
+
+[Subsubgroup]
+; This is a subsubgroup option
+Opt =
+
+[command]
; Use for extra verbosity
; ExtraVerbose =
`,
},
{
- []string{"command"},
+ []string{"filename", "0", "command"},
IniDefault | IniIncludeDefaults | IniCommentDefaults,
`[Application Options]
; Show verbose debug information
@@ -84,15 +110,21 @@ int-map = b:3
; EmptyDescription = false
; Test default value
-; Default = Some value
+; Default = "Some\nvalue"
; Test default array value
; DefaultArray = Some value
-; DefaultArray = Another value
+; DefaultArray = "Other\tvalue"
; Testdefault map value
-; DefaultMap = some:value
; DefaultMap = another:value
+; DefaultMap = some:value
+
+; Test env-default1 value
+EnvDefault1 = env-def
+
+; Test env-default2 value
+EnvDefault2 = env-def
; Option only available in ini
; only-ini =
@@ -105,14 +137,22 @@ int-map = b:3
; A map from string to int
; int-map = a:1
-[command.A command]
+[Subgroup]
+; This is a subgroup option
+; Opt =
+
+[Subsubgroup]
+; This is a subsubgroup option
+; Opt =
+
+[command]
; Use for extra verbosity
; ExtraVerbose =
`,
},
{
- []string{"--default=New value", "--default-array=New value", "--default-map=new:value", "command"},
+ []string{"--default=New value", "--default-array=New value", "--default-map=new:value", "filename", "0", "command"},
IniDefault | IniIncludeDefaults | IniCommentDefaults,
`[Application Options]
; Show verbose debug information
@@ -132,6 +172,12 @@ DefaultArray = New value
; Testdefault map value
DefaultMap = new:value
+; Test env-default1 value
+EnvDefault1 = env-def
+
+; Test env-default2 value
+EnvDefault2 = env-def
+
; Option only available in ini
; only-ini =
@@ -143,7 +189,15 @@ DefaultMap = new:value
; A map from string to int
; int-map = a:1
-[command.A command]
+[Subgroup]
+; This is a subgroup option
+; Opt =
+
+[Subsubgroup]
+; This is a subsubgroup option
+; Opt =
+
+[command]
; Use for extra verbosity
; ExtraVerbose =
@@ -171,15 +225,8 @@ DefaultMap = new:value
got := b.String()
expected := test.expected
- if got != expected {
- ret, err := helpDiff(got, expected)
-
- if err != nil {
- t.Errorf("Unexpected ini with arguments %+v and ini options %b, expected:\n\n%s\n\nbut got\n\n%s", test.args, test.options, expected, got)
- } else {
- t.Errorf("Unexpected ini with arguments %+v and ini options %b:\n\n%s", test.args, test.options, ret)
- }
- }
+ msg := fmt.Sprintf("with arguments %+v and ini options %b", test.args, test.options)
+ assertDiff(t, got, expected, msg)
}
}
@@ -196,16 +243,23 @@ func TestReadIni(t *testing.T) {
verbose = true
verbose = true
+DefaultMap = another:"value\n1"
+DefaultMap = some:value 2
+
[Application Options]
; A slice of pointers to string
; PtrSlice =
; Test default value
-Default = Some value
+Default = "New\nvalue"
+
+; Test env-default1 value
+EnvDefault1 = New value
[Other Options]
# A slice of strings
-# StringSlice =
+StringSlice = "some\nvalue"
+StringSlice = another value
; A map from string to int
int-map = a:2
@@ -222,6 +276,16 @@ int-map = b:3
assertBoolArray(t, opts.Verbose, []bool{true, true})
+ if v := map[string]string{"another": "value\n1", "some": "value 2"}; !reflect.DeepEqual(opts.DefaultMap, v) {
+ t.Fatalf("Expected %#v for DefaultMap but got %#v", v, opts.DefaultMap)
+ }
+
+ assertString(t, opts.Default, "New\nvalue")
+
+ assertString(t, opts.EnvDefault1, "New value")
+
+ assertStringArray(t, opts.Other.StringSlice, []string{"some\nvalue", "another value"})
+
if v, ok := opts.Other.IntMap["a"]; !ok {
t.Errorf("Expected \"a\" in Other.IntMap")
} else if v != 2 {
@@ -235,6 +299,222 @@ int-map = b:3
}
}
+func TestReadAndWriteIni(t *testing.T) {
+ var tests = []struct {
+ options IniOptions
+ read string
+ write string
+ }{
+ {
+ IniIncludeComments,
+ `[Application Options]
+; Show verbose debug information
+verbose = true
+verbose = true
+
+; Test default value
+Default = "quote me"
+
+; Test default array value
+DefaultArray = 1
+DefaultArray = "2"
+DefaultArray = 3
+
+; Testdefault map value
+; DefaultMap =
+
+; Test env-default1 value
+EnvDefault1 = env-def
+
+; Test env-default2 value
+EnvDefault2 = env-def
+
+[Other Options]
+; A slice of strings
+; StringSlice =
+
+; A map from string to int
+int-map = a:2
+int-map = b:"3"
+
+`,
+ `[Application Options]
+; Show verbose debug information
+verbose = true
+verbose = true
+
+; Test default value
+Default = "quote me"
+
+; Test default array value
+DefaultArray = 1
+DefaultArray = 2
+DefaultArray = 3
+
+; Testdefault map value
+; DefaultMap =
+
+; Test env-default1 value
+EnvDefault1 = env-def
+
+; Test env-default2 value
+EnvDefault2 = env-def
+
+[Other Options]
+; A slice of strings
+; StringSlice =
+
+; A map from string to int
+int-map = a:2
+int-map = b:3
+
+`,
+ },
+ {
+ IniIncludeComments,
+ `[Application Options]
+; Show verbose debug information
+verbose = true
+verbose = true
+
+; Test default value
+Default = "quote me"
+
+; Test default array value
+DefaultArray = "1"
+DefaultArray = "2"
+DefaultArray = "3"
+
+; Testdefault map value
+; DefaultMap =
+
+; Test env-default1 value
+EnvDefault1 = env-def
+
+; Test env-default2 value
+EnvDefault2 = env-def
+
+[Other Options]
+; A slice of strings
+; StringSlice =
+
+; A map from string to int
+int-map = a:"2"
+int-map = b:"3"
+
+`,
+ `[Application Options]
+; Show verbose debug information
+verbose = true
+verbose = true
+
+; Test default value
+Default = "quote me"
+
+; Test default array value
+DefaultArray = "1"
+DefaultArray = "2"
+DefaultArray = "3"
+
+; Testdefault map value
+; DefaultMap =
+
+; Test env-default1 value
+EnvDefault1 = env-def
+
+; Test env-default2 value
+EnvDefault2 = env-def
+
+[Other Options]
+; A slice of strings
+; StringSlice =
+
+; A map from string to int
+int-map = a:"2"
+int-map = b:"3"
+
+`,
+ },
+ }
+
+ for _, test := range tests {
+ var opts helpOptions
+
+ p := NewNamedParser("TestIni", Default)
+ p.AddGroup("Application Options", "The application options", &opts)
+
+ inip := NewIniParser(p)
+
+ read := strings.NewReader(test.read)
+ err := inip.Parse(read)
+ if err != nil {
+ t.Fatalf("Unexpected error: %s", err)
+ }
+
+ var write bytes.Buffer
+ inip.Write(&write, test.options)
+
+ got := write.String()
+
+ msg := fmt.Sprintf("with ini options %b", test.options)
+ assertDiff(t, got, test.write, msg)
+ }
+}
+
+func TestReadIniWrongQuoting(t *testing.T) {
+ var tests = []struct {
+ iniFile string
+ lineNumber uint
+ }{
+ {
+ iniFile: `Default = "New\nvalue`,
+ lineNumber: 1,
+ },
+ {
+ iniFile: `StringSlice = "New\nvalue`,
+ lineNumber: 1,
+ },
+ {
+ iniFile: `StringSlice = "New\nvalue"
+ StringSlice = "Second\nvalue`,
+ lineNumber: 2,
+ },
+ {
+ iniFile: `DefaultMap = some:"value`,
+ lineNumber: 1,
+ },
+ {
+ iniFile: `DefaultMap = some:value
+ DefaultMap = another:"value`,
+ lineNumber: 2,
+ },
+ }
+
+ for _, test := range tests {
+ var opts helpOptions
+
+ p := NewNamedParser("TestIni", Default)
+ p.AddGroup("Application Options", "The application options", &opts)
+
+ inip := NewIniParser(p)
+
+ inic := test.iniFile
+
+ b := strings.NewReader(inic)
+ err := inip.Parse(b)
+
+ if err == nil {
+ t.Fatalf("Expect error")
+ }
+
+ iniError := err.(*IniError)
+
+ if iniError.LineNumber != test.lineNumber {
+ t.Fatalf("Expect error on line %d", test.lineNumber)
+ }
+ }
+}
+
func TestIniCommands(t *testing.T) {
var opts struct {
Value string `short:"v" long:"value"`
@@ -261,6 +541,7 @@ AliasName = 5
[add.Other Options]
other = subgroup
+
`
b := strings.NewReader(inic)
@@ -277,6 +558,13 @@ other = subgroup
}
assertString(t, opts.Add.Other.O, "subgroup")
+
+ // Test writing it back
+ buf := &bytes.Buffer{}
+
+ inip.Write(buf, IniDefault)
+
+ assertDiff(t, buf.String(), inic, "ini contents")
}
func TestIniNoIni(t *testing.T) {
@@ -300,7 +588,15 @@ value = some value
t.Fatalf("Expected error")
}
- assertError(t, err, ErrUnknownFlag, "unknown option: value")
+ iniError := err.(*IniError)
+
+ if v := uint(2); iniError.LineNumber != v {
+ t.Errorf("Expected opts.Add.Name to be %d, but got %d", v, iniError.LineNumber)
+ }
+
+ if v := "unknown option: value"; iniError.Message != v {
+ t.Errorf("Expected opts.Add.Name to be %s, but got %s", v, iniError.Message)
+ }
}
func TestIniParse(t *testing.T) {
@@ -359,7 +655,77 @@ func TestWriteFile(t *testing.T) {
expected := "[Application Options]\nValue = 123\n\n"
- if string(found) != expected {
- t.Fatalf("Expected file content to be \"%s\" but was \"%s\"", expected, found)
+ assertDiff(t, string(found), expected, "ini content")
+}
+
+func TestOverwriteRequiredOptions(t *testing.T) {
+ var tests = []struct {
+ args []string
+ expected []string
+ }{
+ {
+ args: []string{"--value", "from CLI"},
+ expected: []string{
+ "from CLI",
+ "from default",
+ },
+ },
+ {
+ args: []string{"--value", "from CLI", "--default", "from CLI"},
+ expected: []string{
+ "from CLI",
+ "from CLI",
+ },
+ },
+ {
+ args: []string{"--config", "no file name"},
+ expected: []string{
+ "from INI",
+ "from INI",
+ },
+ },
+ {
+ args: []string{"--value", "from CLI before", "--default", "from CLI before", "--config", "no file name"},
+ expected: []string{
+ "from INI",
+ "from INI",
+ },
+ },
+ {
+ args: []string{"--value", "from CLI before", "--default", "from CLI before", "--config", "no file name", "--value", "from CLI after", "--default", "from CLI after"},
+ expected: []string{
+ "from CLI after",
+ "from CLI after",
+ },
+ },
+ }
+
+ for _, test := range tests {
+ var opts struct {
+ Config func(s string) error `long:"config" no-ini:"true"`
+ Value string `long:"value" required:"true"`
+ Default string `long:"default" required:"true" default:"from default"`
+ }
+
+ p := NewParser(&opts, Default)
+
+ opts.Config = func(s string) error {
+ ini := NewIniParser(p)
+
+ return ini.Parse(bytes.NewBufferString("value = from INI\ndefault = from INI"))
+ }
+
+ _, err := p.ParseArgs(test.args)
+ if err != nil {
+ t.Fatalf("Unexpected error %s with args %+v", err, test.args)
+ }
+
+ if opts.Value != test.expected[0] {
+ t.Fatalf("Expected Value to be \"%s\" but was \"%s\" with args %+v", test.expected[0], opts.Value, test.args)
+ }
+
+ if opts.Default != test.expected[1] {
+ t.Fatalf("Expected Default to be \"%s\" but was \"%s\" with args %+v", test.expected[1], opts.Default, test.args)
+ }
}
}
diff --git a/vendor/src/github.com/jessevdk/go-flags/man.go b/vendor/src/github.com/jessevdk/go-flags/man.go
index 9e1f0369f6c..e8e5916c061 100644
--- a/vendor/src/github.com/jessevdk/go-flags/man.go
+++ b/vendor/src/github.com/jessevdk/go-flags/man.go
@@ -50,7 +50,7 @@ func writeManPageOptions(wr io.Writer, grp *Group) {
fmt.Fprintf(wr, ", ")
}
- fmt.Fprintf(wr, "--%s", opt.LongName)
+ fmt.Fprintf(wr, "--%s", opt.LongNameWithNamespace())
}
fmt.Fprintln(wr, "\\fP")
@@ -74,11 +74,11 @@ func writeManPageSubcommands(wr io.Writer, name string, root *Command) {
nn = c.Name
}
- writeManPageCommand(wr, nn, c)
+ writeManPageCommand(wr, nn, root, c)
}
}
-func writeManPageCommand(wr io.Writer, name string, command *Command) {
+func writeManPageCommand(wr io.Writer, name string, root *Command, command *Command) {
fmt.Fprintf(wr, ".SS %s\n", name)
fmt.Fprintln(wr, command.ShortDescription)
@@ -98,6 +98,24 @@ func writeManPageCommand(wr io.Writer, name string, command *Command) {
}
}
+ var usage string
+ if us, ok := command.data.(Usage); ok {
+ usage = us.Usage()
+ } else if command.hasCliOptions() {
+ usage = fmt.Sprintf("[%s-OPTIONS]", command.Name)
+ }
+
+ var pre string
+ if root.hasCliOptions() {
+ pre = fmt.Sprintf("%s [OPTIONS] %s", root.Name, command.Name)
+ } else {
+ pre = fmt.Sprintf("%s %s", root.Name, command.Name)
+ }
+
+ if len(usage) > 0 {
+ fmt.Fprintf(wr, "\n\\fBUsage\\fP: %s %s\n\n", pre, usage)
+ }
+
if len(command.Aliases) > 0 {
fmt.Fprintf(wr, "\n\\fBAliases\\fP: %s\n\n", strings.Join(command.Aliases, ", "))
}
diff --git a/vendor/src/github.com/jessevdk/go-flags/marshal_test.go b/vendor/src/github.com/jessevdk/go-flags/marshal_test.go
index 362d0f285b3..59c9ccefb96 100644
--- a/vendor/src/github.com/jessevdk/go-flags/marshal_test.go
+++ b/vendor/src/github.com/jessevdk/go-flags/marshal_test.go
@@ -80,7 +80,7 @@ func TestUnmarshalError(t *testing.T) {
Value marshalled `short:"v"`
}{}
- assertParseFail(t, ErrMarshal, "invalid argument for flag `-v' (expected flags.marshalled): `invalid' is not a valid value, please specify `yes' or `no'", &opts, "-vinvalid")
+ assertParseFail(t, ErrMarshal, fmt.Sprintf("invalid argument for flag `%cv' (expected flags.marshalled): `invalid' is not a valid value, please specify `yes' or `no'", defaultShortOptDelimiter), &opts, "-vinvalid")
}
func TestMarshalError(t *testing.T) {
diff --git a/vendor/src/github.com/jessevdk/go-flags/option.go b/vendor/src/github.com/jessevdk/go-flags/option.go
index f4041d694e1..29e702c1960 100644
--- a/vendor/src/github.com/jessevdk/go-flags/option.go
+++ b/vendor/src/github.com/jessevdk/go-flags/option.go
@@ -27,6 +27,12 @@ type Option struct {
// The default value of the option.
Default []string
+ // The optional environment default value key name.
+ EnvDefaultKey string
+
+ // The optional delimiter string for EnvDefaultKey values.
+ EnvDefaultDelim string
+
// If true, specifies that the argument to an option flag is optional.
// When no argument to the flag is specified on the command line, the
// value of Default will be set in the field this option represents.
@@ -53,15 +59,71 @@ type Option struct {
// passwords.
DefaultMask string
+ // The group which the option belongs to
+ group *Group
+
// The struct field which the option represents.
field reflect.StructField
// The struct field value which the option represents.
value reflect.Value
- defaultValue reflect.Value
- iniUsedName string
- tag multiTag
+ // Determines if the option will be always quoted in the INI output
+ iniQuote bool
+
+ tag multiTag
+ isSet bool
+}
+
+// LongNameWithNamespace returns the option's long name with the group namespaces
+// prepended by walking up the option's group tree. Namespaces and the long name
+// itself are separated by the parser's namespace delimiter. If the long name is
+// empty an empty string is returned.
+func (option *Option) LongNameWithNamespace() string {
+ if len(option.LongName) == 0 {
+ return ""
+ }
+
+ // fetch the namespace delimiter from the parser which is always at the
+ // end of the group hierarchy
+ namespaceDelimiter := ""
+ g := option.group
+
+ for {
+ if p, ok := g.parent.(*Parser); ok {
+ namespaceDelimiter = p.NamespaceDelimiter
+
+ break
+ }
+
+ switch i := g.parent.(type) {
+ case *Command:
+ g = i.Group
+ case *Group:
+ g = i
+ }
+ }
+
+ // concatenate long name with namespace
+ longName := option.LongName
+ g = option.group
+
+ for g != nil {
+ if g.Namespace != "" {
+ longName = g.Namespace + namespaceDelimiter + longName
+ }
+
+ switch i := g.parent.(type) {
+ case *Command:
+ g = i.Group
+ case *Group:
+ g = i
+ case *Parser:
+ g = nil
+ }
+ }
+
+ return longName
}
// String converts an option to a human friendly readable string describing the
@@ -78,12 +140,12 @@ func (option *Option) String() string {
if len(option.LongName) != 0 {
s = fmt.Sprintf("%s%s, %s%s",
string(defaultShortOptDelimiter), short,
- defaultLongOptDelimiter, option.LongName)
+ defaultLongOptDelimiter, option.LongNameWithNamespace())
} else {
s = fmt.Sprintf("%s%s", string(defaultShortOptDelimiter), short)
}
} else if len(option.LongName) != 0 {
- s = fmt.Sprintf("%s%s", defaultLongOptDelimiter, option.LongName)
+ s = fmt.Sprintf("%s%s", defaultLongOptDelimiter, option.LongNameWithNamespace())
}
return s
diff --git a/vendor/src/github.com/jessevdk/go-flags/option_private.go b/vendor/src/github.com/jessevdk/go-flags/option_private.go
index ea836bd083d..d36c8411786 100644
--- a/vendor/src/github.com/jessevdk/go-flags/option_private.go
+++ b/vendor/src/github.com/jessevdk/go-flags/option_private.go
@@ -2,12 +2,16 @@ package flags
import (
"reflect"
+ "strings"
+ "syscall"
)
// Set the value of an option to the specified value. An error will be returned
// if the specified value could not be converted to the corresponding option
// value type.
func (option *Option) set(value *string) error {
+ option.isSet = true
+
if option.isFunc() {
return option.call(value)
} else if value != nil {
@@ -29,21 +33,78 @@ func (option *Option) canArgument() bool {
return !option.isBool()
}
-func (option *Option) clear() {
+func (option *Option) emptyValue() reflect.Value {
tp := option.value.Type()
- switch tp.Kind() {
- case reflect.Func:
- // Skip
- case reflect.Map:
- // Empty the map
- option.value.Set(reflect.MakeMap(tp))
- default:
- zeroval := reflect.Zero(tp)
- option.value.Set(zeroval)
+ if tp.Kind() == reflect.Map {
+ return reflect.MakeMap(tp)
+ }
+
+ return reflect.Zero(tp)
+}
+
+func (option *Option) empty() {
+ if !option.isFunc() {
+ option.value.Set(option.emptyValue())
+ }
+}
+
+func (option *Option) clearDefault() {
+ usedDefault := option.Default
+ if envKey := option.EnvDefaultKey; envKey != "" {
+ // os.Getenv() makes no distinction between undefined and
+ // empty values, so we use syscall.Getenv()
+ if value, ok := syscall.Getenv(envKey); ok {
+ if option.EnvDefaultDelim != "" {
+ usedDefault = strings.Split(value,
+ option.EnvDefaultDelim)
+ } else {
+ usedDefault = []string{value}
+ }
+ }
+ }
+
+ if len(usedDefault) > 0 {
+ option.empty()
+
+ for _, d := range usedDefault {
+ option.set(&d)
+ }
+ } else {
+ tp := option.value.Type()
+
+ switch tp.Kind() {
+ case reflect.Map:
+ if option.value.IsNil() {
+ option.empty()
+ }
+ case reflect.Slice:
+ if option.value.IsNil() {
+ option.empty()
+ }
+ }
}
}
+func (option *Option) valueIsDefault() bool {
+ // Check if the value of the option corresponds to its
+ // default value
+ emptyval := option.emptyValue()
+
+ checkvalptr := reflect.New(emptyval.Type())
+ checkval := reflect.Indirect(checkvalptr)
+
+ checkval.Set(emptyval)
+
+ if len(option.Default) != 0 {
+ for _, v := range option.Default {
+ convert(v, checkval, option.tag)
+ }
+ }
+
+ return reflect.DeepEqual(option.value.Interface(), checkval.Interface())
+}
+
func (option *Option) isUnmarshaler() Unmarshaler {
v := option.value
diff --git a/vendor/src/github.com/jessevdk/go-flags/optstyle_other.go b/vendor/src/github.com/jessevdk/go-flags/optstyle_other.go
index 8ee886eef68..794de23e09a 100644
--- a/vendor/src/github.com/jessevdk/go-flags/optstyle_other.go
+++ b/vendor/src/github.com/jessevdk/go-flags/optstyle_other.go
@@ -30,15 +30,15 @@ func stripOptionPrefix(optname string) (prefix string, name string, islong bool)
// splitOption attempts to split the passed option into a name and an argument.
// When there is no argument specified, nil will be returned for it.
-func splitOption(prefix string, option string, islong bool) (string, *string) {
+func splitOption(prefix string, option string, islong bool) (string, string, *string) {
pos := strings.Index(option, "=")
if (islong && pos >= 0) || (!islong && pos == 1) {
rest := option[pos+1:]
- return option[:pos], &rest
+ return option[:pos], "=", &rest
}
- return option, nil
+ return option, "", nil
}
// addHelpGroup adds a new group that contains default help parameters.
diff --git a/vendor/src/github.com/jessevdk/go-flags/optstyle_windows.go b/vendor/src/github.com/jessevdk/go-flags/optstyle_windows.go
index edd398a5936..096bbffe683 100644
--- a/vendor/src/github.com/jessevdk/go-flags/optstyle_windows.go
+++ b/vendor/src/github.com/jessevdk/go-flags/optstyle_windows.go
@@ -43,9 +43,9 @@ func stripOptionPrefix(optname string) (prefix string, name string, islong bool)
// splitOption attempts to split the passed option into a name and an argument.
// When there is no argument specified, nil will be returned for it.
-func splitOption(prefix string, option string, islong bool) (string, *string) {
+func splitOption(prefix string, option string, islong bool) (string, string, *string) {
if len(option) == 0 {
- return option, nil
+ return option, "", nil
}
// Windows typically uses a colon for the option name and argument
@@ -53,19 +53,22 @@ func splitOption(prefix string, option string, islong bool) (string, *string) {
// but don't allow the two to be mixed. That is to say /foo:bar and
// --foo=bar are acceptable, but /foo=bar and --foo:bar are not.
var pos int
+ var sp string
if prefix == "/" {
- pos = strings.Index(option, ":")
+ sp = ":"
+ pos = strings.Index(option, sp)
} else if len(prefix) > 0 {
- pos = strings.Index(option, "=")
+ sp = "="
+ pos = strings.Index(option, sp)
}
if (islong && pos >= 0) || (!islong && pos == 1) {
rest := option[pos+1:]
- return option[:pos], &rest
+ return option[:pos], sp, &rest
}
- return option, nil
+ return option, "", nil
}
// addHelpGroup adds a new group that contains default help parameters.
diff --git a/vendor/src/github.com/jessevdk/go-flags/parser.go b/vendor/src/github.com/jessevdk/go-flags/parser.go
index 674d0e4337f..a417ba9b238 100644
--- a/vendor/src/github.com/jessevdk/go-flags/parser.go
+++ b/vendor/src/github.com/jessevdk/go-flags/parser.go
@@ -7,7 +7,6 @@ package flags
import (
"os"
"path"
- "reflect"
)
// A Parser provides command line option parsing. It can contain several
@@ -22,6 +21,16 @@ type Parser struct {
// Option flags changing the behavior of the parser.
Options Options
+ // NamespaceDelimiter separates group namespaces and option long names
+ NamespaceDelimiter string
+
+ // UnknownOptionsHandler is a function which gets called when the parser
+ // encounters an unknown option. The function receives the unknown option
+ // name and the remaining command line arguments.
+ // It should return a new list of remaining arguments to continue parsing,
+ // or an error to indicate a parse failure.
+ UnknownOptionHandler func(option string, args []string) ([]string, error)
+
internalError error
}
@@ -88,23 +97,34 @@ func ParseArgs(data interface{}, args []string) ([]string, error) {
// group should not be added. The options parameter specifies a set of options
// for the parser.
func NewParser(data interface{}, options Options) *Parser {
- ret := NewNamedParser(path.Base(os.Args[0]), options)
+ p := NewNamedParser(path.Base(os.Args[0]), options)
if data != nil {
- _, ret.internalError = ret.AddGroup("Application Options", "", data)
+ g, err := p.AddGroup("Application Options", "", data)
+
+ if err == nil {
+ g.parent = p
+ }
+
+ p.internalError = err
}
- return ret
+ return p
}
// NewNamedParser creates a new parser. The appname is used to display the
// executable name in the built-in help message. Option groups and commands can
// be added to this parser by using AddGroup and AddCommand.
func NewNamedParser(appname string, options Options) *Parser {
- return &Parser{
- Command: newCommand(appname, "", "", nil),
- Options: options,
+ p := &Parser{
+ Command: newCommand(appname, "", "", nil),
+ Options: options,
+ NamespaceDelimiter: ".",
}
+
+ p.Command.parent = p
+
+ return p
}
// Parse parses the command line arguments from os.Args using Parser.ParseArgs.
@@ -128,38 +148,41 @@ func (p *Parser) ParseArgs(args []string) ([]string, error) {
return nil, p.internalError
}
- p.eachCommand(func(c *Command) {
- c.eachGroup(func(g *Group) {
- g.storeDefaults()
-
- for _, option := range g.options {
- switch option.value.Type().Kind() {
- case reflect.Func, reflect.Map, reflect.Slice:
- option.clear()
- }
- }
- })
- }, true)
+ p.clearIsSet()
// Add built-in help group to all commands if necessary
if (p.Options & HelpFlag) != None {
p.addHelpGroups(p.showBuiltinHelp)
}
+ compval := os.Getenv("GO_FLAGS_COMPLETION")
+
+ if len(compval) != 0 {
+ comp := &completion{parser: p}
+
+ if compval == "verbose" {
+ comp.ShowDescriptions = true
+ }
+
+ comp.execute(args)
+
+ return nil, nil
+ }
+
s := &parseState{
args: args,
retargs: make([]string, 0, len(args)),
- command: p.Command,
- lookup: p.makeLookup(),
}
+ p.fillParseState(s)
+
for !s.eof() {
arg := s.pop()
// When PassDoubleDash is set and we encounter a --, then
// simply append all the rest as arguments and break out
if (p.Options&PassDoubleDash) != None && arg == "--" {
- s.retargs = append(s.retargs, s.args...)
+ s.addArgs(s.args...)
break
}
@@ -174,32 +197,36 @@ func (p *Parser) ParseArgs(args []string) ([]string, error) {
}
var err error
- var options []*Option
prefix, optname, islong := stripOptionPrefix(arg)
- optname, argument := splitOption(prefix, optname, islong)
+ optname, _, argument := splitOption(prefix, optname, islong)
if islong {
- options, err = p.parseLong(s, optname, argument)
+ err = p.parseLong(s, optname, argument)
} else {
- options, err = p.parseShort(s, optname, argument)
+ err = p.parseShort(s, optname, argument)
}
if err != nil {
ignoreUnknown := (p.Options & IgnoreUnknown) != None
parseErr := wrapError(err)
- if !(parseErr.Type == ErrUnknownFlag && ignoreUnknown) {
+ if parseErr.Type != ErrUnknownFlag || (!ignoreUnknown && p.UnknownOptionHandler == nil) {
s.err = parseErr
break
}
if ignoreUnknown {
- s.retargs = append(s.retargs, arg)
- }
- } else {
- for _, option := range options {
- delete(s.lookup.required, option)
+ s.addArgs(arg)
+ } else if p.UnknownOptionHandler != nil {
+ modifiedArgs, err := p.UnknownOptionHandler(optname, s.args)
+
+ if err != nil {
+ s.err = err
+ break
+ }
+
+ s.args = modifiedArgs
}
}
}
@@ -208,39 +235,30 @@ func (p *Parser) ParseArgs(args []string) ([]string, error) {
p.eachCommand(func(c *Command) {
c.eachGroup(func(g *Group) {
for _, option := range g.options {
- tp := option.value.Type()
-
- switch tp.Kind() {
- case reflect.Map:
- if option.value.Len() == 0 {
- for _, k := range option.defaultValue.MapKeys() {
- option.value.SetMapIndex(k, option.defaultValue.MapIndex(k))
- }
- }
- case reflect.Slice:
- if reflect.DeepEqual(option.value.Interface(), reflect.Zero(tp).Interface()) {
- option.value.Set(option.defaultValue)
- }
+ if option.isSet {
+ continue
}
+
+ option.clearDefault()
}
})
}, true)
- s.checkRequired()
+ s.checkRequired(p)
}
var reterr error
if s.err != nil {
- reterr = p.printError(s.err)
+ reterr = s.err
} else if len(s.command.commands) != 0 && !s.command.SubcommandsOptional {
- reterr = p.printError(s.estimateCommand())
+ reterr = s.estimateCommand()
} else if cmd, ok := s.command.data.(Commander); ok {
- reterr = p.printError(cmd.Execute(s.retargs))
+ reterr = cmd.Execute(s.retargs)
}
if reterr != nil {
- return append([]string{s.arg}, s.args...), reterr
+ return append([]string{s.arg}, s.args...), p.printError(reterr)
}
return s.retargs, nil
diff --git a/vendor/src/github.com/jessevdk/go-flags/parser_private.go b/vendor/src/github.com/jessevdk/go-flags/parser_private.go
index 8d29b6b6bf0..6ee506fd660 100644
--- a/vendor/src/github.com/jessevdk/go-flags/parser_private.go
+++ b/vendor/src/github.com/jessevdk/go-flags/parser_private.go
@@ -4,15 +4,17 @@ import (
"bytes"
"fmt"
"os"
+ "sort"
"strings"
"unicode/utf8"
)
type parseState struct {
- arg string
- args []string
- retargs []string
- err error
+ arg string
+ args []string
+ retargs []string
+ positional []*Arg
+ err error
command *Command
lookup lookup
@@ -41,19 +43,63 @@ func (p *parseState) peek() string {
return p.args[0]
}
-func (p *parseState) checkRequired() error {
- required := p.lookup.required
+func (p *parseState) checkRequired(parser *Parser) error {
+ c := parser.Command
+
+ var required []*Option
+
+ for c != nil {
+ c.eachGroup(func(g *Group) {
+ for _, option := range g.options {
+ if !option.isSet && option.Required {
+ required = append(required, option)
+ }
+ }
+ })
+
+ c = c.Active
+ }
if len(required) == 0 {
+ if len(p.positional) > 0 && p.command.ArgsRequired {
+ var reqnames []string
+
+ for _, arg := range p.positional {
+ if arg.isRemaining() {
+ break
+ }
+
+ reqnames = append(reqnames, "`"+arg.Name+"`")
+ }
+
+ if len(reqnames) == 0 {
+ return nil
+ }
+
+ var msg string
+
+ if len(reqnames) == 1 {
+ msg = fmt.Sprintf("the required argument %s was not provided", reqnames[0])
+ } else {
+ msg = fmt.Sprintf("the required arguments %s and %s were not provided",
+ strings.Join(reqnames[:len(reqnames)-1], ", "), reqnames[len(reqnames)-1])
+ }
+
+ p.err = newError(ErrRequired, msg)
+ return p.err
+ }
+
return nil
}
names := make([]string, 0, len(required))
- for k := range required {
+ for _, k := range required {
names = append(names, "`"+k.String()+"'")
}
+ sort.Strings(names)
+
var msg string
if len(names) == 1 {
@@ -113,18 +159,34 @@ func (p *parseState) estimateCommand() error {
func (p *Parser) parseOption(s *parseState, name string, option *Option, canarg bool, argument *string) (err error) {
if !option.canArgument() {
if argument != nil {
- msg := fmt.Sprintf("bool flag `%s' cannot have an argument", option)
- return newError(ErrNoArgumentForBool, msg)
+ return newErrorf(ErrNoArgumentForBool, "bool flag `%s' cannot have an argument", option)
}
err = option.set(nil)
- } else if argument != nil {
- err = option.set(argument)
- } else if canarg && !s.eof() {
- arg := s.pop()
- err = option.set(&arg)
+ } else if argument != nil || (canarg && !s.eof()) {
+ var arg string
+
+ if argument != nil {
+ arg = *argument
+ } else {
+ arg = s.pop()
+
+ // Accept any single character arguments including '-'.
+ // '-' is the special file name for the standard input or the standard output in many cases.
+ if len(arg) > 1 && argumentIsOption(arg) {
+ return newErrorf(ErrExpectedArgument, "expected argument for flag `%s', but got option `%s'", option, arg)
+ }
+ }
+
+ if option.tag.Get("unquote") != "false" {
+ arg, err = unquoteIfPossible(arg)
+ }
+
+ if err == nil {
+ err = option.set(&arg)
+ }
} else if option.OptionalArgument {
- option.clear()
+ option.empty()
for _, v := range option.OptionalValue {
err = option.set(&v)
@@ -134,38 +196,31 @@ func (p *Parser) parseOption(s *parseState, name string, option *Option, canarg
}
}
} else {
- msg := fmt.Sprintf("expected argument for flag `%s'", option)
- err = newError(ErrExpectedArgument, msg)
+ err = newErrorf(ErrExpectedArgument, "expected argument for flag `%s'", option)
}
if err != nil {
if _, ok := err.(*Error); !ok {
- msg := fmt.Sprintf("invalid argument for flag `%s' (expected %s): %s",
+ err = newErrorf(ErrMarshal, "invalid argument for flag `%s' (expected %s): %s",
option,
option.value.Type(),
err.Error())
-
- err = newError(ErrMarshal, msg)
}
}
return err
}
-func (p *Parser) parseLong(s *parseState, name string, argument *string) (options []*Option, err error) {
+func (p *Parser) parseLong(s *parseState, name string, argument *string) error {
if option := s.lookup.longNames[name]; option != nil {
- options = append(options, option)
-
// Only long options that are required can consume an argument
// from the argument list
canarg := !option.OptionalArgument
- err := p.parseOption(s, name, option, canarg, argument)
-
- return options, err
+ return p.parseOption(s, name, option, canarg, argument)
}
- return nil, newError(ErrUnknownFlag, fmt.Sprintf("unknown flag `%s'", name))
+ return newErrorf(ErrUnknownFlag, "unknown flag `%s'", name)
}
func (p *Parser) splitShortConcatArg(s *parseState, optname string) (string, *string) {
@@ -185,7 +240,7 @@ func (p *Parser) splitShortConcatArg(s *parseState, optname string) (string, *st
return optname, nil
}
-func (p *Parser) parseShort(s *parseState, optname string, argument *string) (options []*Option, err error) {
+func (p *Parser) parseShort(s *parseState, optname string, argument *string) error {
if argument == nil {
optname, argument = p.splitShortConcatArg(s, optname)
}
@@ -194,17 +249,15 @@ func (p *Parser) parseShort(s *parseState, optname string, argument *string) (op
shortname := string(c)
if option := s.lookup.shortNames[shortname]; option != nil {
- options = append(options, option)
-
// Only the last short argument can consume an argument from
// the arguments list, and only if it's non optional
canarg := (i+utf8.RuneLen(c) == len(optname)) && !option.OptionalArgument
if err := p.parseOption(s, shortname, option, canarg, argument); err != nil {
- return options, err
+ return err
}
} else {
- return nil, newError(ErrUnknownFlag, fmt.Sprintf("unknown flag `%s'", shortname))
+ return newErrorf(ErrUnknownFlag, "unknown flag `%s'", shortname)
}
// Only the first option can have a concatted argument, so just
@@ -212,26 +265,50 @@ func (p *Parser) parseShort(s *parseState, optname string, argument *string) (op
argument = nil
}
- return options, nil
+ return nil
}
-func (p *Parser) parseNonOption(s *parseState) error {
- if cmd := s.lookup.commands[s.arg]; cmd != nil {
- if err := s.checkRequired(); err != nil {
+func (p *parseState) addArgs(args ...string) error {
+ for len(p.positional) > 0 && len(args) > 0 {
+ arg := p.positional[0]
+
+ if err := convert(args[0], arg.value, arg.tag); err != nil {
return err
}
- s.command.Active = cmd
+ if !arg.isRemaining() {
+ p.positional = p.positional[1:]
+ }
+
+ args = args[1:]
+ }
+
+ p.retargs = append(p.retargs, args...)
+ return nil
+}
- s.command = cmd
- s.lookup = cmd.makeLookup()
+func (p *Parser) parseNonOption(s *parseState) error {
+ if len(s.positional) > 0 {
+ return s.addArgs(s.arg)
+ }
+
+ if cmd := s.lookup.commands[s.arg]; cmd != nil {
+ s.command.Active = cmd
+ cmd.fillParseState(s)
} else if (p.Options & PassAfterNonOption) != None {
// If PassAfterNonOption is set then all remaining arguments
// are considered positional
- s.retargs = append(append(s.retargs, s.arg), s.args...)
+ if err := s.addArgs(s.arg); err != nil {
+ return err
+ }
+
+ if err := s.addArgs(s.args...); err != nil {
+ return err
+ }
+
s.args = []string{}
} else {
- s.retargs = append(s.retargs, s.arg)
+ return s.addArgs(s.arg)
}
return nil
@@ -251,3 +328,13 @@ func (p *Parser) printError(err error) error {
return err
}
+
+func (p *Parser) clearIsSet() {
+ p.eachCommand(func(c *Command) {
+ c.eachGroup(func(g *Group) {
+ for _, option := range g.options {
+ option.isSet = false
+ }
+ })
+ }, true)
+}
diff --git a/vendor/src/github.com/jessevdk/go-flags/parser_test.go b/vendor/src/github.com/jessevdk/go-flags/parser_test.go
index b227244d78c..bba7576e717 100644
--- a/vendor/src/github.com/jessevdk/go-flags/parser_test.go
+++ b/vendor/src/github.com/jessevdk/go-flags/parser_test.go
@@ -1,7 +1,11 @@
package flags
import (
+ "fmt"
+ "os"
"reflect"
+ "strconv"
+ "strings"
"testing"
"time"
)
@@ -10,6 +14,10 @@ type defaultOptions struct {
Int int `long:"i"`
IntDefault int `long:"id" default:"1"`
+ String string `long:"str"`
+ StringDefault string `long:"strd" default:"abc"`
+ StringNotUnquoted string `long:"strnot" unquote:"false"`
+
Time time.Duration `long:"t"`
TimeDefault time.Duration `long:"td" default:"1m"`
@@ -33,6 +41,9 @@ func TestDefaults(t *testing.T) {
Int: 0,
IntDefault: 1,
+ String: "",
+ StringDefault: "abc",
+
Time: 0,
TimeDefault: time.Minute,
@@ -45,11 +56,14 @@ func TestDefaults(t *testing.T) {
},
{
msg: "non-zero value arguments, expecting overwritten arguments",
- args: []string{"--i=3", "--id=3", "--t=3ms", "--td=3ms", "--m=c:3", "--md=c:3", "--s=3", "--sd=3"},
+ args: []string{"--i=3", "--id=3", "--str=def", "--strd=def", "--t=3ms", "--td=3ms", "--m=c:3", "--md=c:3", "--s=3", "--sd=3"},
expected: defaultOptions{
Int: 3,
IntDefault: 3,
+ String: "def",
+ StringDefault: "def",
+
Time: 3 * time.Millisecond,
TimeDefault: 3 * time.Millisecond,
@@ -62,11 +76,14 @@ func TestDefaults(t *testing.T) {
},
{
msg: "zero value arguments, expecting overwritten arguments",
- args: []string{"--i=0", "--id=0", "--t=0ms", "--td=0s", "--m=:0", "--md=:0", "--s=0", "--sd=0"},
+ args: []string{"--i=0", "--id=0", "--str", "", "--strd=\"\"", "--t=0ms", "--td=0s", "--m=:0", "--md=:0", "--s=0", "--sd=0"},
expected: defaultOptions{
Int: 0,
IntDefault: 0,
+ String: "",
+ StringDefault: "",
+
Time: 0,
TimeDefault: 0,
@@ -96,3 +113,298 @@ func TestDefaults(t *testing.T) {
}
}
}
+
+func TestUnquoting(t *testing.T) {
+ var tests = []struct {
+ arg string
+ err error
+ value string
+ }{
+ {
+ arg: "\"abc",
+ err: strconv.ErrSyntax,
+ value: "",
+ },
+ {
+ arg: "\"\"abc\"",
+ err: strconv.ErrSyntax,
+ value: "",
+ },
+ {
+ arg: "\"abc\"",
+ err: nil,
+ value: "abc",
+ },
+ {
+ arg: "\"\\\"abc\\\"\"",
+ err: nil,
+ value: "\"abc\"",
+ },
+ {
+ arg: "\"\\\"abc\"",
+ err: nil,
+ value: "\"abc",
+ },
+ }
+
+ for _, test := range tests {
+ var opts defaultOptions
+
+ for _, delimiter := range []bool{false, true} {
+ p := NewParser(&opts, None)
+
+ var err error
+ if delimiter {
+ _, err = p.ParseArgs([]string{"--str=" + test.arg, "--strnot=" + test.arg})
+ } else {
+ _, err = p.ParseArgs([]string{"--str", test.arg, "--strnot", test.arg})
+ }
+
+ if test.err == nil {
+ if err != nil {
+ t.Fatalf("Expected no error but got: %v", err)
+ }
+
+ if test.value != opts.String {
+ t.Fatalf("Expected String to be %q but got %q", test.value, opts.String)
+ }
+ if q := strconv.Quote(test.value); q != opts.StringNotUnquoted {
+ t.Fatalf("Expected StringDefault to be %q but got %q", q, opts.StringNotUnquoted)
+ }
+ } else {
+ if err == nil {
+ t.Fatalf("Expected error")
+ } else if e, ok := err.(*Error); ok {
+ if strings.HasPrefix(e.Message, test.err.Error()) {
+ t.Fatalf("Expected error message to end with %q but got %v", test.err.Error(), e.Message)
+ }
+ }
+ }
+ }
+ }
+}
+
+// envRestorer keeps a copy of a set of env variables and can restore the env from them
+type envRestorer struct {
+ env map[string]string
+}
+
+func (r *envRestorer) Restore() {
+ os.Clearenv()
+ for k, v := range r.env {
+ os.Setenv(k, v)
+ }
+}
+
+// EnvSnapshot returns a snapshot of the currently set env variables
+func EnvSnapshot() *envRestorer {
+ r := envRestorer{make(map[string]string)}
+ for _, kv := range os.Environ() {
+ parts := strings.SplitN(kv, "=", 2)
+ if len(parts) != 2 {
+ panic("got a weird env variable: " + kv)
+ }
+ r.env[parts[0]] = parts[1]
+ }
+ return &r
+}
+
+type envDefaultOptions struct {
+ Int int `long:"i" default:"1" env:"TEST_I"`
+ Time time.Duration `long:"t" default:"1m" env:"TEST_T"`
+ Map map[string]int `long:"m" default:"a:1" env:"TEST_M" env-delim:";"`
+ Slice []int `long:"s" default:"1" default:"2" env:"TEST_S" env-delim:","`
+}
+
+func TestEnvDefaults(t *testing.T) {
+ var tests = []struct {
+ msg string
+ args []string
+ expected envDefaultOptions
+ env map[string]string
+ }{
+ {
+ msg: "no arguments, no env, expecting default values",
+ args: []string{},
+ expected: envDefaultOptions{
+ Int: 1,
+ Time: time.Minute,
+ Map: map[string]int{"a": 1},
+ Slice: []int{1, 2},
+ },
+ },
+ {
+ msg: "no arguments, env defaults, expecting env default values",
+ args: []string{},
+ expected: envDefaultOptions{
+ Int: 2,
+ Time: 2 * time.Minute,
+ Map: map[string]int{"a": 2, "b": 3},
+ Slice: []int{4, 5, 6},
+ },
+ env: map[string]string{
+ "TEST_I": "2",
+ "TEST_T": "2m",
+ "TEST_M": "a:2;b:3",
+ "TEST_S": "4,5,6",
+ },
+ },
+ {
+ msg: "non-zero value arguments, expecting overwritten arguments",
+ args: []string{"--i=3", "--t=3ms", "--m=c:3", "--s=3"},
+ expected: envDefaultOptions{
+ Int: 3,
+ Time: 3 * time.Millisecond,
+ Map: map[string]int{"c": 3},
+ Slice: []int{3},
+ },
+ env: map[string]string{
+ "TEST_I": "2",
+ "TEST_T": "2m",
+ "TEST_M": "a:2;b:3",
+ "TEST_S": "4,5,6",
+ },
+ },
+ {
+ msg: "zero value arguments, expecting overwritten arguments",
+ args: []string{"--i=0", "--t=0ms", "--m=:0", "--s=0"},
+ expected: envDefaultOptions{
+ Int: 0,
+ Time: 0,
+ Map: map[string]int{"": 0},
+ Slice: []int{0},
+ },
+ env: map[string]string{
+ "TEST_I": "2",
+ "TEST_T": "2m",
+ "TEST_M": "a:2;b:3",
+ "TEST_S": "4,5,6",
+ },
+ },
+ }
+
+ oldEnv := EnvSnapshot()
+ defer oldEnv.Restore()
+
+ for _, test := range tests {
+ var opts envDefaultOptions
+ oldEnv.Restore()
+ for envKey, envValue := range test.env {
+ os.Setenv(envKey, envValue)
+ }
+ _, err := ParseArgs(&opts, test.args)
+ if err != nil {
+ t.Fatalf("%s:\nUnexpected error: %v", test.msg, err)
+ }
+
+ if opts.Slice == nil {
+ opts.Slice = []int{}
+ }
+
+ if !reflect.DeepEqual(opts, test.expected) {
+ t.Errorf("%s:\nUnexpected options with arguments %+v\nexpected\n%+v\nbut got\n%+v\n", test.msg, test.args, test.expected, opts)
+ }
+ }
+}
+
+func TestOptionAsArgument(t *testing.T) {
+ var tests = []struct {
+ args []string
+ expectError bool
+ errType ErrorType
+ errMsg string
+ }{
+ {
+ // short option must not be accepted as argument
+ args: []string{"--string-slice", "foobar", "--string-slice", "-o"},
+ expectError: true,
+ errType: ErrExpectedArgument,
+ errMsg: "expected argument for flag `--string-slice', but got option `-o'",
+ },
+ {
+ // long option must not be accepted as argument
+ args: []string{"--string-slice", "foobar", "--string-slice", "--other-option"},
+ expectError: true,
+ errType: ErrExpectedArgument,
+ errMsg: "expected argument for flag `--string-slice', but got option `--other-option'",
+ },
+ {
+ // long option must not be accepted as argument
+ args: []string{"--string-slice", "--"},
+ expectError: true,
+ errType: ErrExpectedArgument,
+ errMsg: "expected argument for flag `--string-slice', but got option `--'",
+ },
+ {
+ // quoted and appended option should be accepted as argument (even if it looks like an option)
+ args: []string{"--string-slice", "foobar", "--string-slice=\"--other-option\""},
+ },
+ {
+ // Accept any single character arguments including '-'
+ args: []string{"--string-slice", "-"},
+ },
+ }
+ var opts struct {
+ StringSlice []string `long:"string-slice"`
+ OtherOption bool `long:"other-option" short:"o"`
+ }
+
+ for _, test := range tests {
+ if test.expectError {
+ assertParseFail(t, test.errType, test.errMsg, &opts, test.args...)
+ } else {
+ assertParseSuccess(t, &opts, test.args...)
+ }
+ }
+}
+
+func TestUnknownFlagHandler(t *testing.T) {
+
+ var opts struct {
+ Flag1 string `long:"flag1"`
+ Flag2 string `long:"flag2"`
+ }
+
+ p := NewParser(&opts, None)
+
+ var unknownFlag1 string
+ var unknownFlag2 bool
+
+ // Set up a callback to intercept unknown options during parsing
+ p.UnknownOptionHandler = func(option string, args []string) ([]string, error) {
+ if option == "unknownFlag1" {
+ // consume a value from remaining args list
+ unknownFlag1 = args[0]
+ return args[1:], nil
+ } else if option == "unknownFlag2" {
+ // treat this one as a bool switch, don't consume any args
+ unknownFlag2 = true
+ return args, nil
+ }
+
+ return args, fmt.Errorf("Unknown flag: %v", option)
+ }
+
+ // Parse args containing some unknown flags, verify that
+ // our callback can handle all of them
+ _, err := p.ParseArgs([]string{"--flag1=stuff", "--unknownFlag1", "blah", "--unknownFlag2", "--flag2=foo"})
+
+ if err != nil {
+ assertErrorf(t, "Parser returned unexpected error %v", err)
+ }
+
+ assertString(t, opts.Flag1, "stuff")
+ assertString(t, opts.Flag2, "foo")
+ assertString(t, unknownFlag1, "blah")
+
+ if !unknownFlag2 {
+ assertErrorf(t, "Flag should have been set by unknown handler, but had value: %v", unknownFlag2)
+ }
+
+ // Parse args with unknown flags that callback doesn't handle, verify it returns error
+ _, err = p.ParseArgs([]string{"--flag1=stuff", "--unknownFlagX", "blah", "--flag2=foo"})
+
+ if err == nil {
+ assertErrorf(t, "Parser should have returned error, but returned nil")
+ }
+}
diff --git a/vendor/src/github.com/jessevdk/go-flags/short_test.go b/vendor/src/github.com/jessevdk/go-flags/short_test.go
index d08702b9ae2..95712c16238 100644
--- a/vendor/src/github.com/jessevdk/go-flags/short_test.go
+++ b/vendor/src/github.com/jessevdk/go-flags/short_test.go
@@ -1,6 +1,7 @@
package flags
import (
+ "fmt"
"testing"
)
@@ -31,7 +32,7 @@ func TestShortRequired(t *testing.T) {
Value bool `short:"v" required:"true"`
}{}
- assertParseFail(t, ErrRequired, "the required flag `-v' was not specified", &opts)
+ assertParseFail(t, ErrRequired, fmt.Sprintf("the required flag `%cv' was not specified", defaultShortOptDelimiter), &opts)
}
func TestShortMultiConcat(t *testing.T) {
@@ -143,7 +144,7 @@ func TestShortMultiWithEqualArg(t *testing.T) {
Value string `short:"v"`
}{}
- assertParseFail(t, ErrExpectedArgument, "expected argument for flag `-v'", &opts, "-ffv=value")
+ assertParseFail(t, ErrExpectedArgument, fmt.Sprintf("expected argument for flag `%cv'", defaultShortOptDelimiter), &opts, "-ffv=value")
}
func TestShortMultiArg(t *testing.T) {
@@ -165,7 +166,7 @@ func TestShortMultiArgConcatFail(t *testing.T) {
Value string `short:"v"`
}{}
- assertParseFail(t, ErrExpectedArgument, "expected argument for flag `-v'", &opts, "-ffvvalue")
+ assertParseFail(t, ErrExpectedArgument, fmt.Sprintf("expected argument for flag `%cv'", defaultShortOptDelimiter), &opts, "-ffvvalue")
}
func TestShortMultiArgConcat(t *testing.T) {
diff --git a/vendor/src/github.com/spacemonkeygo/spacelog/setup.go b/vendor/src/github.com/spacemonkeygo/spacelog/setup.go
index bd46b0ab27e..26ad00572c9 100644
--- a/vendor/src/github.com/spacemonkeygo/spacelog/setup.go
+++ b/vendor/src/github.com/spacemonkeygo/spacelog/setup.go
@@ -137,12 +137,12 @@ func Setup(procname string, config SetupConfig) error {
textout = w
case "stdout":
if t == nil {
- t = ColorTemplate
+ t = DefaultTemplate
}
textout = NewWriterOutput(os.Stdout)
case "stderr", "":
if t == nil {
- t = ColorTemplate
+ t = DefaultTemplate
}
textout = NewWriterOutput(os.Stderr)
default:
diff --git a/vendor/src/github.com/spacemonkeygo/spacelog/templates_others.go b/vendor/src/github.com/spacemonkeygo/spacelog/templates_others.go
new file mode 100644
index 00000000000..114e2e14312
--- /dev/null
+++ b/vendor/src/github.com/spacemonkeygo/spacelog/templates_others.go
@@ -0,0 +1,22 @@
+// Copyright (C) 2014 Space Monkey, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !windows
+
+package spacelog
+
+var (
+ // DefaultTemplate is default template for stdout/stderr for the platform
+ DefaultTemplate = ColorTemplate
+)
diff --git a/vendor/src/github.com/spacemonkeygo/spacelog/templates_windows.go b/vendor/src/github.com/spacemonkeygo/spacelog/templates_windows.go
new file mode 100644
index 00000000000..512b600481e
--- /dev/null
+++ b/vendor/src/github.com/spacemonkeygo/spacelog/templates_windows.go
@@ -0,0 +1,20 @@
+// Copyright (C) 2014 Space Monkey, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package spacelog
+
+var (
+ // DefaultTemplate is default template for stdout/stderr for the platform
+ DefaultTemplate = StandardTemplate
+)