diff options
author | mike o'brien <mpobrien005@gmail.com> | 2014-11-18 19:09:14 -0500 |
---|---|---|
committer | mike o'brien <mpobrien005@gmail.com> | 2014-11-20 14:51:25 -0500 |
commit | e4fa10d01cf26f6808c5b52fff351a16b66b67bd (patch) | |
tree | 15fd28ebfc6159cc0bdffeac2b2bf0e1a821f4b6 | |
parent | 47c44d0a4ca508729510b57a8557b9ed75f55cc3 (diff) | |
download | mongo-e4fa10d01cf26f6808c5b52fff351a16b66b67bd.tar.gz |
TOOLS-295 make certain flags hidden, make --dbpath trigger error message
Former-commit-id: 17ddf0fa38c75b4886969f8de0fcdc8a12745874
64 files changed, 3318 insertions, 1441 deletions
@@ -1,7 +1,7 @@ gopkg.in/mgo.v2 e2e914857713db7497cca2bd7fc0b030fc9cb22d github.com/jacobsa/oglematchers 4fc24f97b5b74022c2a3f4ca7eed57ca29083d3e github.com/smartystreets/goconvey 75bc4a2dad71e5c5b51c5009b33eba766ec57051 -github.com/jessevdk/go-flags 8ec9564882e7923e632f012761c81c46dcf5bec1 +github.com/jessevdk/go-flags 37e89eb6730ccfebfa05c17940954e4308719317 gopkg.in/tomb.v2 14b3d72120e8d10ea6e6b7f87f7175734b1faab8 github.com/3rf/mongo-lint 3550fdcf1f43b89aaeabaa4559eaae6dc4407e42 github.com/spacemonkeygo/openssl 500a817338511aad07db09254b7ccf6916ac6a19 github.com/gabrielrussell/openssl diff --git a/common/log/tool_logger.go b/common/log/tool_logger.go index 8f7577b0f34..565be55d08e 100644 --- a/common/log/tool_logger.go +++ b/common/log/tool_logger.go @@ -2,7 +2,6 @@ package log import ( "fmt" - "github.com/mongodb/mongo-tools/common/options" "io" "os" "sync" @@ -30,11 +29,21 @@ type ToolLogger struct { verbosity int } -func (tl *ToolLogger) SetVerbosity(verbosity *options.Verbosity) { - if verbosity.Quiet { +type VerbosityLevel interface { + Level() int + IsQuiet() bool +} + +func (tl *ToolLogger) SetVerbosity(level VerbosityLevel) { + if level == nil { + tl.verbosity = 0 + return + } + + if level.IsQuiet() { tl.verbosity = -1 } else { - tl.verbosity = len(verbosity.Verbose) + tl.verbosity = level.Level() } } @@ -74,7 +83,7 @@ func (tl *ToolLogger) log(msg string) { fmt.Fprintf(tl.writer, "%v\t%v\n", time.Now().Format(tl.format), msg) } -func NewToolLogger(verbosity *options.Verbosity) *ToolLogger { +func NewToolLogger(verbosity VerbosityLevel) *ToolLogger { tl := &ToolLogger{ mutex: &sync.Mutex{}, writer: os.Stderr, // default to stderr @@ -111,10 +120,7 @@ var globalToolLogger *ToolLogger func init() { if globalToolLogger == nil { // initialize tool logger with verbosity level = 0 - globalToolLogger = NewToolLogger(&options.Verbosity{ - []bool{}, // Verbose - false, // Quiet - }) + globalToolLogger = NewToolLogger(nil) } } @@ -126,7 +132,7 @@ func Log(minVerb int, msg string) { globalToolLogger.Log(minVerb, msg) } -func SetVerbosity(verbosity *options.Verbosity) { +func SetVerbosity(verbosity VerbosityLevel) { globalToolLogger.SetVerbosity(verbosity) } diff --git a/common/options/options.go b/common/options/options.go index ae06180f203..03bd0b435f7 100644 --- a/common/options/options.go +++ b/common/options/options.go @@ -5,7 +5,10 @@ package options import ( "fmt" "github.com/jessevdk/go-flags" + "github.com/mongodb/mongo-tools/common/log" "os" + "runtime" + "strconv" ) const ( @@ -33,6 +36,7 @@ type ToolOptions struct { *Auth *Kerberos *Namespace + *HiddenOptions //Force direct connection to the server and disable the //drivers automatic repl set discovery logic. @@ -42,15 +46,22 @@ type ToolOptions struct { parser *flags.Parser } +type HiddenOptions struct { + MaxProcs int + BulkWriters int + BulkBufferSize int + + // Specifies the number of threads to use in processing data read from the input source + NumDecodingWorkers int + + // Specifies the number of threads to use in sending processed data over to the server + NumInsertionWorkers int +} + type Namespace struct { // Specified database and collection DB string `short:"d" long:"db" description:"database to use"` Collection string `short:"c" long:"collection" description:"collection to use"` - - // DBPath for direct-storage interface - DBPath string `long:"dbpath"` - DirectoryPerDB bool `long:"directoryperdb" default:"false"` - Journal bool `long:"journal" default:"false"` } // Struct holding generic options @@ -65,6 +76,14 @@ type Verbosity struct { Quiet bool `long:"quiet" description:"Run in quiet mode, attempting to limit the amount of output"` } +func (v Verbosity) Level() int { + return len(v.Verbose) +} + +func (v Verbosity) IsQuiet() bool { + return v.Quiet +} + // Struct holding connection-related options type Connection struct { Host string `short:"h" long:"host" description:"Specify a resolvable hostname to which to connect"` @@ -109,22 +128,30 @@ type EnabledOptions struct { // Ask for a new instance of tool options func New(appName, usageStr string, enabled EnabledOptions) *ToolOptions { + hiddenOpts := &HiddenOptions{ + BulkWriters: 1, + BulkBufferSize: 10000, + } + opts := &ToolOptions{ AppName: appName, VersionStr: VersionStr, UsageStr: usageStr, - General: &General{}, - Verbosity: &Verbosity{}, - Connection: &Connection{}, - SSL: &SSL{}, - Auth: &Auth{}, - Namespace: &Namespace{}, - Kerberos: &Kerberos{}, - parser: flags.NewNamedParser(appName, flags.None), + General: &General{}, + Verbosity: &Verbosity{}, + Connection: &Connection{}, + SSL: &SSL{}, + Auth: &Auth{}, + Namespace: &Namespace{}, + HiddenOptions: hiddenOpts, + Kerberos: &Kerberos{}, + parser: flags.NewNamedParser(appName, flags.None), } - opts.parser.Usage = usageStr + opts.parser.UnknownOptionHandler = func(option string, args []string) ([]string, error) { + return parseHiddenOption(hiddenOpts, option, args) + } if _, err := opts.parser.AddGroup("general options", "", opts.General); err != nil { panic(fmt.Errorf("couldn't register general options: %v", err)) @@ -156,6 +183,12 @@ func New(appName, usageStr string, enabled EnabledOptions) *ToolOptions { panic(fmt.Errorf("couldn't register namespace options")) } } + + if opts.MaxProcs <= 0 { + opts.MaxProcs = runtime.NumCPU() + } + log.Logf(log.Info, "Setting num cpus to %v", opts.MaxProcs) + runtime.GOMAXPROCS(opts.MaxProcs) return opts } @@ -211,12 +244,40 @@ func (self *ToolOptions) Parse() ([]string, error) { return self.parser.Parse() } -//Validate() runs validation checks that are global to all tools. -func (self *ToolOptions) Validate() error { - switch { - case self.DBPath != "" && self.Host != "": - return fmt.Errorf("--dbpath is not allowed when --host is specified") +func parseHiddenOption(opts *HiddenOptions, option string, args []string) ([]string, error) { + if option == "dbpath" || option == "directoryperdb" || option == "journal" { + return args, fmt.Errorf(`--dbpath and related flags are not supported in 2.8 tools. +See http://dochub.mongodb.org/core/tools-dbpath-deprecated for more information`) + } + var err error + optionValue, err := getInt(args) + switch option { + case "numThreads": + opts.MaxProcs = optionValue + case "numInsertionWorkersPerCollection": + opts.BulkWriters = optionValue + case "batchSize": + opts.BulkBufferSize = optionValue + case "numDecodingWorkers": + opts.NumDecodingWorkers = optionValue + case "numInsertionWorkers": + opts.NumInsertionWorkers = optionValue + default: + return args, fmt.Errorf(`unknown option "%v"`, option) + } + if err != nil { + return args, fmt.Errorf(`error parsing value for "%v": %v`, option, err) } + return args[1:], nil +} - return nil +func getInt(args []string) (int, error) { + if len(args) == 0 { + return 0, fmt.Errorf("no value specified") + } + val, err := strconv.Atoi(args[0]) + if err != nil { + return 0, fmt.Errorf("expected an integer value but got '%v'") + } + return val, nil } diff --git a/mongodump/main/mongodump.go b/mongodump/main/mongodump.go index 89c4d2deffb..ec8f5b128ff 100644 --- a/mongodump/main/mongodump.go +++ b/mongodump/main/mongodump.go @@ -6,7 +6,6 @@ import ( "github.com/mongodb/mongo-tools/mongodump" "github.com/mongodb/mongo-tools/mongodump/options" "os" - "runtime" ) func main() { @@ -38,11 +37,6 @@ func main() { // init logger log.SetVerbosity(opts.Verbosity) - if outputOpts.MaxProcs > 0 { - log.Logf(log.DebugHigh, "setting GOMAXPROCS to %v", outputOpts.MaxProcs) - runtime.GOMAXPROCS(outputOpts.MaxProcs) - } - // don't attempt to discover other members of a replica set opts.Direct = true diff --git a/mongodump/mongodump.go b/mongodump/mongodump.go index 1c52b4d61bd..abba8edd280 100644 --- a/mongodump/mongodump.go +++ b/mongodump/mongodump.go @@ -49,9 +49,6 @@ type MongoDump struct { // ValidateOptions checks for any incompatible sets of options func (dump *MongoDump) ValidateOptions() error { - if err := dump.ToolOptions.Validate(); err != nil { - return err - } switch { case dump.OutputOptions.Out == "-" && dump.ToolOptions.Namespace.Collection == "": return fmt.Errorf("can only dump a single collection to stdout") @@ -75,8 +72,6 @@ func (dump *MongoDump) ValidateOptions() error { return fmt.Errorf("--db is required when --excludeCollectionsWithPrefix is specified") case dump.OutputOptions.Repair && dump.InputOptions.Query != "": return fmt.Errorf("cannot run a query with --repair enabled") - case dump.OutputOptions.JobThreads < 1: - return fmt.Errorf("number of processing threads must be >= 1") } return nil } @@ -260,7 +255,10 @@ func (dump *MongoDump) Dump() error { func (dump *MongoDump) DumpIntents() error { resultChan := make(chan error) - jobs := dump.OutputOptions.JobThreads + jobs := dump.ToolOptions.HiddenOptions.MaxProcs + if jobs <= 0 { + jobs = 1 + } if jobs > 1 { dump.manager.Finalize(intents.LongestTaskFirst) } else { diff --git a/mongodump/mongodump_kerberos_test.go b/mongodump/mongodump_kerberos_test.go index e5105cf0b57..822bf680b14 100644 --- a/mongodump/mongodump_kerberos_test.go +++ b/mongodump/mongodump_kerberos_test.go @@ -25,9 +25,6 @@ func TestMongoDumpKerberos(t *testing.T) { mongoDump := MongoDump{ ToolOptions: opts, InputOptions: &options.InputOptions{}, - OutputOptions: &options.OutputOptions{ - JobThreads: 1, - }, } mongoDump.OutputOptions.Out = KERBEROS_DUMP_DIRECTORY diff --git a/mongodump/mongodump_test.go b/mongodump/mongodump_test.go index f93615accaf..dd61c7c599a 100644 --- a/mongodump/mongodump_test.go +++ b/mongodump/mongodump_test.go @@ -43,15 +43,14 @@ func simpleMongoDumpInstance() *MongoDump { Port: testPort, } toolOptions := &commonOpts.ToolOptions{ - SSL: &ssl, - Namespace: namespace, - Connection: connection, - Auth: &auth, - Verbosity: &commonOpts.Verbosity{}, - } - outputOptions := &options.OutputOptions{ - JobThreads: 1, + SSL: &ssl, + Namespace: namespace, + Connection: connection, + Auth: &auth, + HiddenOptions: &commonOpts.HiddenOptions{}, + Verbosity: &commonOpts.Verbosity{}, } + outputOptions := &options.OutputOptions{} inputOptions := &options.InputOptions{} log.SetVerbosity(toolOptions.Verbosity) diff --git a/mongodump/options/options.go b/mongodump/options/options.go index 05f6fa6a0b2..b4f0d1086dd 100644 --- a/mongodump/options/options.go +++ b/mongodump/options/options.go @@ -19,8 +19,6 @@ type OutputOptions struct { DumpDBUsersAndRoles bool `long:"dumpDbUsersAndRoles" description:"Dump user and role definitions for the given database"` ExcludedCollections []string `long:"excludeCollection" description:"Collections to exclude from the dump"` ExcludedCollectionPrefixes []string `long:"excludeCollectionsWithPrefix" description:"Exclude all collections from the dump that have the given prefix"` - JobThreads int `long:"numParallelCollections" short:"j" description:"Number of collections to dump in parallel" default:"4"` - MaxProcs int `long:"numCPUThreads" description:"GOMAXPROCS for testing"` // TODO: hide this option } func (self *OutputOptions) Name() string { diff --git a/mongoexport/mongoexport.go b/mongoexport/mongoexport.go index 2246b790adc..826aa3a8f63 100644 --- a/mongoexport/mongoexport.go +++ b/mongoexport/mongoexport.go @@ -65,13 +65,6 @@ type ExportOutput interface { // ValidateSettings returns an error if any settings specified on the command line // were invalid, or nil if they are valid. func (exp *MongoExport) ValidateSettings() error { - // TODO - on legacy mongoexport, if -d is blank, it assumes some default database. - // Do we want to use that same behavior? It seems very odd to assume the DB - // when only a collection is provided, but that's the behavior of the legacy tools. - if err := exp.ToolOptions.Validate(); err != nil { - return err - } - // Namespace must have a valid database if none is specified, // use 'test' if exp.ToolOptions.Namespace.DB == "" { diff --git a/mongoimport/common.go b/mongoimport/common.go index 8279cefbc9b..6ad3ff33958 100644 --- a/mongoimport/common.go +++ b/mongoimport/common.go @@ -194,13 +194,16 @@ func setNestedValue(key string, value interface{}, document *bson.D) { // channel in parallel and then sends over the processed data to the outputChan // channel - either in sequence or concurrently (depending on the value of // ordered) - in which the data was received -func streamDocuments(ordered bool, inputChan chan ConvertibleDoc, outputChan chan bson.D, errChan chan error) { +func streamDocuments(ordered bool, numDecoders int, inputChan chan ConvertibleDoc, outputChan chan bson.D, errChan chan error) { + if numDecoders == 0 { + numDecoders = 1 + } var importWorkers []*ImportWorker // initialize all our concurrent processing threads wg := &sync.WaitGroup{} inChan := inputChan outChan := outputChan - for i := 0; i < numDecodingWorkers; i++ { + for i := 0; i < numDecoders; i++ { if ordered { // TODO: experiment with buffered channel size; the buffer size of // inChan should always be the same as that of outChan @@ -234,7 +237,7 @@ func streamDocuments(ordered bool, inputChan chan ConvertibleDoc, outputChan cha // tokensToBSON reads in slice of records - along with ordered fields names - // and returns a BSON document for the record. func tokensToBSON(fields, tokens []string, numProcessed uint64) (bson.D, error) { - log.Logf(log.DebugLow, "got line: %v", tokens) + log.Logf(log.DebugHigh, "got line: %v", tokens) var parsedValue interface{} document := bson.D{} for index, token := range tokens { diff --git a/mongoimport/common_test.go b/mongoimport/common_test.go index ab262abd785..23740ffb8f9 100644 --- a/mongoimport/common_test.go +++ b/mongoimport/common_test.go @@ -102,51 +102,51 @@ func TestValidateHeaders(t *testing.T) { So(err, ShouldBeNil) Convey("if headerLine is true, the first line in the input should be used", func() { - headers, err := validateHeaders(NewCSVInputReader(fields, fileHandle), true) + headers, err := validateHeaders(NewCSVInputReader(fields, fileHandle, 1), true) So(err, ShouldBeNil) So(len(headers), ShouldEqual, 2) // spaces are trimed in the header So(headers, ShouldResemble, strings.Split(strings.Replace(contents, " ", "", -1), ",")) }) Convey("if headerLine is false, the fields passed in should be used", func() { - headers, err := validateHeaders(NewCSVInputReader(fields, fileHandle), false) + headers, err := validateHeaders(NewCSVInputReader(fields, fileHandle, 1), false) So(err, ShouldBeNil) So(len(headers), ShouldEqual, 3) // spaces are trimed in the header So(headers, ShouldResemble, fields) }) Convey("if the fields contain '..', an error should be thrown", func() { - _, err := validateHeaders(NewCSVInputReader([]string{"a..a"}, fileHandle), false) + _, err := validateHeaders(NewCSVInputReader([]string{"a..a"}, fileHandle, 1), false) So(err, ShouldNotBeNil) }) Convey("if the fields start/end in a '.', an error should be thrown", func() { - _, err := validateHeaders(NewCSVInputReader([]string{".a"}, fileHandle), false) + _, err := validateHeaders(NewCSVInputReader([]string{".a"}, fileHandle, 1), false) So(err, ShouldNotBeNil) - _, err = validateHeaders(NewCSVInputReader([]string{"a."}, fileHandle), false) + _, err = validateHeaders(NewCSVInputReader([]string{"a."}, fileHandle, 1), false) So(err, ShouldNotBeNil) }) Convey("if the fields collide, an error should be thrown", func() { - _, err := validateHeaders(NewCSVInputReader([]string{"a", "a.a"}, fileHandle), false) + _, err := validateHeaders(NewCSVInputReader([]string{"a", "a.a"}, fileHandle, 1), false) So(err, ShouldNotBeNil) - _, err = validateHeaders(NewCSVInputReader([]string{"a", "a.ba", "b.a"}, fileHandle), false) + _, err = validateHeaders(NewCSVInputReader([]string{"a", "a.ba", "b.a"}, fileHandle, 1), false) So(err, ShouldNotBeNil) - _, err = validateHeaders(NewCSVInputReader([]string{"a", "a.b.c"}, fileHandle), false) + _, err = validateHeaders(NewCSVInputReader([]string{"a", "a.b.c"}, fileHandle, 1), false) So(err, ShouldNotBeNil) }) Convey("if the fields don't collide, no error should be thrown", func() { - _, err := validateHeaders(NewCSVInputReader([]string{"a", "aa"}, fileHandle), false) + _, err := validateHeaders(NewCSVInputReader([]string{"a", "aa"}, fileHandle, 1), false) So(err, ShouldBeNil) - _, err = validateHeaders(NewCSVInputReader([]string{"a", "aa", "b.a", "b.c"}, fileHandle), false) + _, err = validateHeaders(NewCSVInputReader([]string{"a", "aa", "b.a", "b.c"}, fileHandle, 1), false) So(err, ShouldBeNil) - _, err = validateHeaders(NewCSVInputReader([]string{"a", "ba", "ab", "b.a"}, fileHandle), false) + _, err = validateHeaders(NewCSVInputReader([]string{"a", "ba", "ab", "b.a"}, fileHandle, 1), false) So(err, ShouldBeNil) - _, err = validateHeaders(NewCSVInputReader([]string{"a", "ba", "ab", "b.a", "b.c.d"}, fileHandle), false) + _, err = validateHeaders(NewCSVInputReader([]string{"a", "ba", "ab", "b.a", "b.c.d"}, fileHandle, 1), false) So(err, ShouldBeNil) - _, err = validateHeaders(NewCSVInputReader([]string{"a", "ab.c"}, fileHandle), false) + _, err = validateHeaders(NewCSVInputReader([]string{"a", "ab.c"}, fileHandle, 1), false) So(err, ShouldBeNil) }) Convey("if the fields contain the same keys, an error should be thrown", func() { - _, err := validateHeaders(NewCSVInputReader([]string{"a", "ba", "a"}, fileHandle), false) + _, err := validateHeaders(NewCSVInputReader([]string{"a", "ba", "a"}, fileHandle, 1), false) So(err, ShouldNotBeNil) }) }) @@ -509,7 +509,7 @@ func TestStreamDocuments(t *testing.T) { inputChannel <- csvConvertibleDoc } close(inputChannel) - streamDocuments(true, inputChannel, outputChannel, errorChannel) + streamDocuments(true, 3, inputChannel, outputChannel, errorChannel) // ensure documents are streamed out and processed in the correct manner for _, expectedDocument := range expectedDocuments { So(<-outputChannel, ShouldResemble, expectedDocument) @@ -525,7 +525,7 @@ func TestStreamDocuments(t *testing.T) { } inputChannel <- csvConvertibleDoc close(inputChannel) - go streamDocuments(true, inputChannel, outputChannel, errorChannel) + go streamDocuments(true, 3, inputChannel, outputChannel, errorChannel) // ensure that an error is returned on the error channel So(<-errorChannel, ShouldNotBeNil) }) diff --git a/mongoimport/csv.go b/mongoimport/csv.go index 82acb0ce207..caddac19496 100644 --- a/mongoimport/csv.go +++ b/mongoimport/csv.go @@ -10,16 +10,20 @@ import ( // CSVInputReader is a struct that implements the InputReader interface for a // CSV input source type CSVInputReader struct { + // Fields is a list of field names in the BSON documents to be imported Fields []string - // csvReader is the underlying reader used to read data in from the CSV - // or CSV file + + // csvReader is the underlying reader used to read data in from the CSV or CSV file csvReader *csv.Reader + // csvRecord stores each line of input we read from the underlying reader csvRecord []string - // numProcessed tracks the number of CSV records processed by the underlying - // reader + + // numProcessed tracks how many the number of CSV records processed by the underlying reader numProcessed uint64 + + numDecoders int } // CSVConvertibleDoc implements the ConvertibleDoc interface for CSV input @@ -30,7 +34,7 @@ type CSVConvertibleDoc struct { // NewCSVInputReader returns a CSVInputReader configured to read input from the // given io.Reader, extracting the specified fields only. -func NewCSVInputReader(fields []string, in io.Reader) *CSVInputReader { +func NewCSVInputReader(fields []string, in io.Reader, numDecoders int) *CSVInputReader { csvReader := csv.NewReader(in) // allow variable number of fields in document csvReader.FieldsPerRecord = -1 @@ -39,6 +43,7 @@ func NewCSVInputReader(fields []string, in io.Reader) *CSVInputReader { Fields: fields, csvReader: csvReader, numProcessed: uint64(0), + numDecoders: numDecoders, } } @@ -68,7 +73,7 @@ func (csvInputReader *CSVInputReader) ReadHeadersFromSource() ([]string, error) // hits EOF or an error. If ordered is true, it streams the documents in which // the documents are read func (csvInputReader *CSVInputReader) StreamDocument(ordered bool, readChan chan bson.D, errChan chan error) { - csvRecordChan := make(chan ConvertibleDoc, numDecodingWorkers) + csvRecordChan := make(chan ConvertibleDoc, csvInputReader.numDecoders) var err error go func() { @@ -92,7 +97,7 @@ func (csvInputReader *CSVInputReader) StreamDocument(ordered bool, readChan chan csvInputReader.numProcessed++ } }() - streamDocuments(ordered, csvRecordChan, readChan, errChan) + streamDocuments(ordered, csvInputReader.numDecoders, csvRecordChan, readChan, errChan) } // This is required to satisfy the ConvertibleDoc interface for CSV input. It diff --git a/mongoimport/csv_test.go b/mongoimport/csv_test.go index 03e81bbab82..9a6808cd418 100644 --- a/mongoimport/csv_test.go +++ b/mongoimport/csv_test.go @@ -25,7 +25,7 @@ func TestCSVStreamDocument(t *testing.T) { Convey("badly encoded CSV should result in a parsing error", func() { contents := `1, 2, foo"bar` fields := []string{"a", "b", "c"} - csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents))) + csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go csvInputReader.StreamDocument(true, docChan, errChan) @@ -34,7 +34,7 @@ func TestCSVStreamDocument(t *testing.T) { Convey("escaped quotes are parsed correctly", func() { contents := `1, 2, "foo""bar"` fields := []string{"a", "b", "c"} - csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents))) + csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go csvInputReader.StreamDocument(true, docChan, errChan) @@ -43,7 +43,7 @@ func TestCSVStreamDocument(t *testing.T) { Convey("whitespace separated quoted strings are still an error", func() { contents := `1, 2, "foo" "bar"` fields := []string{"a", "b", "c"} - csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents))) + csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go csvInputReader.StreamDocument(true, docChan, errChan) @@ -57,7 +57,7 @@ func TestCSVStreamDocument(t *testing.T) { bson.DocElem{"b", 2}, bson.DocElem{"c", `foo" "bar`}, } - csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents))) + csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go csvInputReader.StreamDocument(true, docChan, errChan) @@ -72,7 +72,7 @@ func TestCSVStreamDocument(t *testing.T) { bson.DocElem{"b", 2}, bson.DocElem{"c", " 3e"}, } - csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents))) + csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go csvInputReader.StreamDocument(true, docChan, errChan) @@ -89,7 +89,7 @@ func TestCSVStreamDocument(t *testing.T) { bson.DocElem{"c", " 3e"}, bson.DocElem{"field3", " may"}, } - csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents))) + csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go csvInputReader.StreamDocument(true, docChan, errChan) @@ -108,7 +108,7 @@ func TestCSVStreamDocument(t *testing.T) { bson.DocElem{"c", " 3e"}, bson.DocElem{"field3", " may"}, } - csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents))) + csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go csvInputReader.StreamDocument(true, docChan, errChan) @@ -124,7 +124,7 @@ func TestCSVStreamDocument(t *testing.T) { Convey("nested CSV fields causing header collisions should error", func() { contents := `1, 2f , " 3e" , " may", june` fields := []string{"a", "b.c", "field3"} - csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents))) + csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go csvInputReader.StreamDocument(true, docChan, errChan) @@ -145,7 +145,7 @@ func TestCSVStreamDocument(t *testing.T) { bson.DocElem{"b", 5}, bson.DocElem{"c", 6}, } - csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents))) + csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go csvInputReader.StreamDocument(true, docChan, errChan) @@ -164,7 +164,7 @@ func TestCSVSetHeader(t *testing.T) { func() { contents := "extraHeader1, extraHeader2, extraHeader3" fields := []string{} - csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents))) + csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) So(csvInputReader.SetHeader(true), ShouldBeNil) So(len(csvInputReader.Fields), ShouldEqual, 3) }) @@ -172,24 +172,24 @@ func TestCSVSetHeader(t *testing.T) { Convey("setting non-colliding nested CSV headers should not raise an error", func() { contents := "a, b, c" fields := []string{} - csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents))) + csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) So(csvInputReader.SetHeader(true), ShouldBeNil) So(len(csvInputReader.Fields), ShouldEqual, 3) contents = "a.b.c, a.b.d, c" fields = []string{} - csvInputReader = NewCSVInputReader(fields, bytes.NewReader([]byte(contents))) + csvInputReader = NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) So(csvInputReader.SetHeader(true), ShouldBeNil) So(len(csvInputReader.Fields), ShouldEqual, 3) contents = "a.b, ab, a.c" fields = []string{} - csvInputReader = NewCSVInputReader(fields, bytes.NewReader([]byte(contents))) + csvInputReader = NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) So(csvInputReader.SetHeader(true), ShouldBeNil) So(len(csvInputReader.Fields), ShouldEqual, 3) contents = "a, ab, ac, dd" fields = []string{} - csvInputReader = NewCSVInputReader(fields, bytes.NewReader([]byte(contents))) + csvInputReader = NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) So(csvInputReader.SetHeader(true), ShouldBeNil) So(len(csvInputReader.Fields), ShouldEqual, 4) }) @@ -197,17 +197,17 @@ func TestCSVSetHeader(t *testing.T) { Convey("setting colliding nested CSV headers should raise an error", func() { contents := "a, a.b, c" fields := []string{} - csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents))) + csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) So(csvInputReader.SetHeader(true), ShouldNotBeNil) contents = "a.b.c, a.b.d.c, a.b.d" fields = []string{} - csvInputReader = NewCSVInputReader(fields, bytes.NewReader([]byte(contents))) + csvInputReader = NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) So(csvInputReader.SetHeader(true), ShouldNotBeNil) contents = "a, a, a" fields = []string{} - csvInputReader = NewCSVInputReader(fields, bytes.NewReader([]byte(contents))) + csvInputReader = NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) So(csvInputReader.SetHeader(true), ShouldNotBeNil) }) @@ -216,32 +216,32 @@ func TestCSVSetHeader(t *testing.T) { contents := "c, a., b" fields := []string{} So(err, ShouldBeNil) - So(NewCSVInputReader(fields, bytes.NewReader([]byte(contents))).SetHeader(true), ShouldNotBeNil) + So(NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1).SetHeader(true), ShouldNotBeNil) }) Convey("setting the header that starts in a dot should error", func() { contents := "c, .a, b" fields := []string{} - So(NewCSVInputReader(fields, bytes.NewReader([]byte(contents))).SetHeader(true), ShouldNotBeNil) + So(NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1).SetHeader(true), ShouldNotBeNil) }) Convey("setting the header that contains multiple consecutive dots should error", func() { contents := "c, a..a, b" fields := []string{} - So(NewCSVInputReader(fields, bytes.NewReader([]byte(contents))).SetHeader(true), ShouldNotBeNil) + So(NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1).SetHeader(true), ShouldNotBeNil) contents = "c, a.a, b.b...b" fields = []string{} - So(NewCSVInputReader(fields, bytes.NewReader([]byte(contents))).SetHeader(true), ShouldNotBeNil) + So(NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1).SetHeader(true), ShouldNotBeNil) }) Convey("setting the header using an empty file should return EOF", func() { contents := "" fields := []string{} - csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents))) + csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) So(csvInputReader.SetHeader(true), ShouldEqual, io.EOF) So(len(csvInputReader.Fields), ShouldEqual, 0) }) @@ -250,7 +250,7 @@ func TestCSVSetHeader(t *testing.T) { func() { contents := "extraHeader1,extraHeader2,extraHeader3" fields := []string{"a", "b", "c"} - csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents))) + csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) So(csvInputReader.SetHeader(true), ShouldBeNil) // if SetHeader() is called with fields already passed in, // the header should be replaced with the read header line @@ -274,7 +274,7 @@ func TestCSVSetHeader(t *testing.T) { } fileHandle, err := os.Open("testdata/test.csv") So(err, ShouldBeNil) - csvInputReader := NewCSVInputReader(fields, fileHandle) + csvInputReader := NewCSVInputReader(fields, fileHandle, 1) errChan := make(chan error) docChan := make(chan bson.D) @@ -291,7 +291,7 @@ func TestCSVGetHeaders(t *testing.T) { Convey("With a CSV input reader", t, func() { Convey("getting the header should return any already set headers", func() { fields := []string{"extraHeader1", "extraHeader2", "extraHeader3"} - csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte{})) + csvInputReader := NewCSVInputReader(fields, bytes.NewReader([]byte{}), 1) So(csvInputReader.GetHeaders(), ShouldResemble, fields) }) }) @@ -304,7 +304,7 @@ func TestCSVReadHeadersFromSource(t *testing.T) { expectedHeaders := []string{"1", "2", "3"} fileHandle, err := os.Open("testdata/test.csv") So(err, ShouldBeNil) - csvInputReader := NewCSVInputReader([]string{}, fileHandle) + csvInputReader := NewCSVInputReader([]string{}, fileHandle, 1) headers, err := csvInputReader.ReadHeadersFromSource() So(err, ShouldBeNil) So(headers, ShouldResemble, expectedHeaders) diff --git a/mongoimport/json.go b/mongoimport/json.go index 0f95ab66136..4e44409da23 100644 --- a/mongoimport/json.go +++ b/mongoimport/json.go @@ -35,6 +35,10 @@ type JSONInputReader struct { // separator. It is a reader consisting of the decoder's buffer and the // underlying reader separatorReader io.Reader + + // numDecoders is the number of concurrent goroutines to allow to be used + // for decoding the input stream + numDecoders int } // JSONConvertibleDoc implements the ConvertibleDoc interface for JSON input @@ -60,12 +64,13 @@ var ( // NewJSONInputReader creates a new JSONInputReader in array mode if specified, // configured to read data to the given io.Reader -func NewJSONInputReader(isArray bool, in io.Reader) *JSONInputReader { +func NewJSONInputReader(isArray bool, in io.Reader, numDecoders int) *JSONInputReader { return &JSONInputReader{ IsArray: isArray, Decoder: json.NewDecoder(in), readOpeningBracket: false, bytesFromReader: make([]byte, 1), + numDecoders: numDecoders, } } @@ -90,7 +95,7 @@ func (jsonInputReader *JSONInputReader) ReadHeadersFromSource() ([]string, error // hits EOF or an error. If ordered is true, it streams the documents in which // the documents are read func (jsonInputReader *JSONInputReader) StreamDocument(ordered bool, readChan chan bson.D, errChan chan error) { - rawChan := make(chan ConvertibleDoc, numDecodingWorkers) + rawChan := make(chan ConvertibleDoc, jsonInputReader.numDecoders) var err error go func() { for { @@ -116,7 +121,7 @@ func (jsonInputReader *JSONInputReader) StreamDocument(ordered bool, readChan ch jsonInputReader.numProcessed++ } }() - streamDocuments(ordered, rawChan, readChan, errChan) + streamDocuments(ordered, jsonInputReader.numDecoders, rawChan, readChan, errChan) } // This is required to satisfy the ConvertibleDoc interface for JSON input. It @@ -126,7 +131,7 @@ func (jsonConvertibleDoc JSONConvertibleDoc) Convert() (bson.D, error) { if err != nil { return nil, fmt.Errorf("error unmarshaling bytes on document #%v: %v", 1, err) } - log.Logf(log.DebugLow, "got line: %v", document) + log.Logf(log.DebugHigh, "got line: %v", document) // TODO: perhaps move this to decode.go bsonD, err := bsonutil.GetExtendedBsonD(document) if err != nil { diff --git a/mongoimport/json_test.go b/mongoimport/json_test.go index 0d0cead462e..eaf15eef8a0 100644 --- a/mongoimport/json_test.go +++ b/mongoimport/json_test.go @@ -17,7 +17,7 @@ func TestJSONArrayStreamDocument(t *testing.T) { Convey("an error should be thrown if a plain JSON document is supplied", func() { contents := `{"a": "ae"}` - jsonInputReader := NewJSONInputReader(true, bytes.NewReader([]byte(contents))) + jsonInputReader := NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go jsonInputReader.StreamDocument(true, docChan, errChan) @@ -28,7 +28,7 @@ func TestJSONArrayStreamDocument(t *testing.T) { "error out", func() { contents := `{"a":3},{"b":4}]` - jsonInputReader := NewJSONInputReader(true, bytes.NewReader([]byte(contents))) + jsonInputReader := NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go jsonInputReader.StreamDocument(true, docChan, errChan) @@ -39,7 +39,7 @@ func TestJSONArrayStreamDocument(t *testing.T) { "error out", func() { contents := `[{"a": "ae"}` - jsonInputReader := NewJSONInputReader(true, bytes.NewReader([]byte(contents))) + jsonInputReader := NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go jsonInputReader.StreamDocument(true, docChan, errChan) @@ -54,7 +54,7 @@ func TestJSONArrayStreamDocument(t *testing.T) { func() { fileHandle, err := os.Open("testdata/test_plain.json") So(err, ShouldBeNil) - jsonInputReader := NewJSONInputReader(true, fileHandle) + jsonInputReader := NewJSONInputReader(true, fileHandle, 1) errChan := make(chan error) docChan := make(chan bson.D) go jsonInputReader.StreamDocument(true, docChan, errChan) @@ -77,7 +77,7 @@ func TestJSONArrayStreamDocument(t *testing.T) { } fileHandle, err := os.Open("testdata/test_array.json") So(err, ShouldBeNil) - jsonInputReader := NewJSONInputReader(true, fileHandle) + jsonInputReader := NewJSONInputReader(true, fileHandle, 1) errChan := make(chan error) docChan := make(chan bson.D) // TODO check error @@ -101,7 +101,7 @@ func TestJSONPlainStreamDocument(t *testing.T) { func() { contents := `{"a": "ae"}` expectedRead := bson.D{bson.DocElem{"a", "ae"}} - jsonInputReader := NewJSONInputReader(false, bytes.NewReader([]byte(contents))) + jsonInputReader := NewJSONInputReader(false, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go jsonInputReader.StreamDocument(true, docChan, errChan) @@ -114,7 +114,7 @@ func TestJSONPlainStreamDocument(t *testing.T) { contents := `{"a": "ae"}{"b": "dc"}` expectedReadOne := bson.D{bson.DocElem{"a", "ae"}} expectedReadTwo := bson.D{bson.DocElem{"b", "dc"}} - jsonInputReader := NewJSONInputReader(false, bytes.NewReader([]byte(contents))) + jsonInputReader := NewJSONInputReader(false, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go jsonInputReader.StreamDocument(true, docChan, errChan) @@ -127,7 +127,7 @@ func TestJSONPlainStreamDocument(t *testing.T) { func() { contents := `{"a": "ae", "b": 2.0}` expectedRead := bson.D{bson.DocElem{"a", "ae"}, bson.DocElem{"b", 2.0}} - jsonInputReader := NewJSONInputReader(false, bytes.NewReader([]byte(contents))) + jsonInputReader := NewJSONInputReader(false, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go jsonInputReader.StreamDocument(true, docChan, errChan) @@ -137,7 +137,7 @@ func TestJSONPlainStreamDocument(t *testing.T) { Convey("JSON arrays should return an error", func() { contents := `[{"a": "ae", "b": 2.0}]` - jsonInputReader := NewJSONInputReader(false, bytes.NewReader([]byte(contents))) + jsonInputReader := NewJSONInputReader(false, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go jsonInputReader.StreamDocument(true, docChan, errChan) @@ -166,7 +166,7 @@ func TestJSONPlainStreamDocument(t *testing.T) { } fileHandle, err := os.Open("testdata/test_plain.json") So(err, ShouldBeNil) - jsonInputReader := NewJSONInputReader(false, fileHandle) + jsonInputReader := NewJSONInputReader(false, fileHandle, 1) errChan := make(chan error) docChan := make(chan bson.D) go jsonInputReader.StreamDocument(true, docChan, errChan) @@ -192,7 +192,7 @@ func TestReadJSONArraySeparator(t *testing.T) { Convey("reading a JSON array separator should consume [", func() { contents := `[{"a": "ae"}` - jsonImporter := NewJSONInputReader(true, bytes.NewReader([]byte(contents))) + jsonImporter := NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1) So(jsonImporter.readJSONArraySeparator(), ShouldBeNil) // at this point it should have consumed all bytes up to `{` So(jsonImporter.readJSONArraySeparator(), ShouldNotBeNil) @@ -201,14 +201,14 @@ func TestReadJSONArraySeparator(t *testing.T) { "corresponding opening bracket should error out ", func() { contents := `]` - jsonImporter := NewJSONInputReader(true, bytes.NewReader([]byte(contents))) + jsonImporter := NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1) So(jsonImporter.readJSONArraySeparator(), ShouldNotBeNil) }) Convey("reading an opening JSON array separator without a "+ "corresponding closing bracket should error out ", func() { contents := `[` - jsonImporter := NewJSONInputReader(true, bytes.NewReader([]byte(contents))) + jsonImporter := NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1) So(jsonImporter.readJSONArraySeparator(), ShouldBeNil) So(jsonImporter.readJSONArraySeparator(), ShouldNotBeNil) }) @@ -216,7 +216,7 @@ func TestReadJSONArraySeparator(t *testing.T) { "closing bracket should return EOF", func() { contents := `[]` - jsonImporter := NewJSONInputReader(true, bytes.NewReader([]byte(contents))) + jsonImporter := NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1) So(jsonImporter.readJSONArraySeparator(), ShouldBeNil) So(jsonImporter.readJSONArraySeparator(), ShouldEqual, io.EOF) }) @@ -224,7 +224,7 @@ func TestReadJSONArraySeparator(t *testing.T) { "bracket but then additional characters after that, should error", func() { contents := `[]a` - jsonImporter := NewJSONInputReader(true, bytes.NewReader([]byte(contents))) + jsonImporter := NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1) So(jsonImporter.readJSONArraySeparator(), ShouldBeNil) So(jsonImporter.readJSONArraySeparator(), ShouldNotBeNil) }) @@ -232,7 +232,7 @@ func TestReadJSONArraySeparator(t *testing.T) { "error out", func() { contents := `[{"a":3}x{"b":4}]` - jsonInputReader := NewJSONInputReader(true, bytes.NewReader([]byte(contents))) + jsonInputReader := NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go jsonInputReader.StreamDocument(true, docChan, errChan) @@ -244,7 +244,7 @@ func TestReadJSONArraySeparator(t *testing.T) { "valid objects should error out", func() { contents := `[{"a":3},b{"b":4}]` - jsonInputReader := NewJSONInputReader(true, bytes.NewReader([]byte(contents))) + jsonInputReader := NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go jsonInputReader.StreamDocument(true, docChan, errChan) @@ -252,7 +252,7 @@ func TestReadJSONArraySeparator(t *testing.T) { So(jsonInputReader.readJSONArraySeparator(), ShouldNotBeNil) contents = `[{"a":3},,{"b":4}]` - jsonInputReader = NewJSONInputReader(true, bytes.NewReader([]byte(contents))) + jsonInputReader = NewJSONInputReader(true, bytes.NewReader([]byte(contents)), 1) errChan = make(chan error) docChan = make(chan bson.D) go jsonInputReader.StreamDocument(true, docChan, errChan) diff --git a/mongoimport/main/mongoimport.go b/mongoimport/main/mongoimport.go index 6b6fb316111..7309030b1bf 100644 --- a/mongoimport/main/mongoimport.go +++ b/mongoimport/main/mongoimport.go @@ -2,12 +2,12 @@ package main import ( "fmt" - "os" "github.com/mongodb/mongo-tools/common/db" "github.com/mongodb/mongo-tools/common/log" commonopts "github.com/mongodb/mongo-tools/common/options" "github.com/mongodb/mongo-tools/mongoimport" "github.com/mongodb/mongo-tools/mongoimport/options" + "os" ) func main() { diff --git a/mongoimport/mongoimport.go b/mongoimport/mongoimport.go index b3fd4b4ed66..c3a854c63d4 100644 --- a/mongoimport/mongoimport.go +++ b/mongoimport/mongoimport.go @@ -13,7 +13,6 @@ import ( "io" "os" "path/filepath" - "runtime" "strings" "sync" ) @@ -31,13 +30,6 @@ const ( maxMessageSizeBytes = 2 * maxBSONSize ) -// variables used by the input/ingestion goroutines -var ( - numDecodingWorkers = 1 // will be set to numCPUs at runtime - numInsertionWorkers = 1 - batchSize = 10000 -) - // Wrapper for MongoImport functionality type MongoImport struct { // generic mongo tool options @@ -95,16 +87,6 @@ type InputReader interface { // ValidateSettings ensures that the tool specific options supplied for // MongoImport are valid func (mongoImport *MongoImport) ValidateSettings(args []string) error { - if err := mongoImport.ToolOptions.Validate(); err != nil { - return err - } - - // TODO: move to common - // --dbpath is now deprecated for tools with version >= v2.8 - if mongoImport.ToolOptions.DBPath != "" { - return fmt.Errorf("--dbpath is now deprecated. start a mongod instead") - } - // Namespace must have a valid database if none is specified, // use 'test' if mongoImport.ToolOptions.Namespace.DB == "" { @@ -151,58 +133,27 @@ func (mongoImport *MongoImport) ValidateSettings(args []string) error { } } - numCPU := runtime.NumCPU() - - // set the number of operating system threads to use for imports - if mongoImport.IngestOptions.NumOSThreads == nil { - runtime.GOMAXPROCS(numCPU) - } else { - if *mongoImport.IngestOptions.NumOSThreads < 1 { - return fmt.Errorf("--numOSThreads argument must be > 0") - } - runtime.GOMAXPROCS(*mongoImport.IngestOptions.NumOSThreads) - } - // set the number of decoding workers to use for imports - if mongoImport.IngestOptions.NumDecodingWorkers != nil { - if *mongoImport.IngestOptions.NumDecodingWorkers < 1 { - return fmt.Errorf("--numDecodingWorkers argument must be > 0") - } - numDecodingWorkers = *mongoImport.IngestOptions.NumDecodingWorkers - } else { - mongoImport.IngestOptions.NumDecodingWorkers = &numCPU - numDecodingWorkers = numCPU + if mongoImport.ToolOptions.NumDecodingWorkers <= 0 { + mongoImport.ToolOptions.NumDecodingWorkers = mongoImport.ToolOptions.MaxProcs } + log.Logf(log.DebugLow, "Using %v decoding workers", mongoImport.ToolOptions.NumDecodingWorkers) // set the number of insertion workers to use for imports - if mongoImport.IngestOptions.NumInsertionWorkers != nil { - if *mongoImport.IngestOptions.NumInsertionWorkers < 1 { - return fmt.Errorf("--numInsertionThreads argument must be > 0") - } - numInsertionWorkers = *mongoImport.IngestOptions.NumInsertionWorkers - } else { - mongoImport.IngestOptions.NumInsertionWorkers = &numInsertionWorkers + if mongoImport.ToolOptions.NumInsertionWorkers <= 0 { + mongoImport.ToolOptions.NumInsertionWorkers = 1 } - // if maintain --maintainInsertionOrder is true, we can only have one - // insertion worker + log.Logf(log.DebugLow, "Using %v insert workers", mongoImport.ToolOptions.NumInsertionWorkers) + + // if --maintainInsertionOrder is set, we can only allow 1 insertion worker if mongoImport.IngestOptions.MaintainInsertionOrder { - if numInsertionWorkers > 1 { - return fmt.Errorf("cannot specify --maintainInsertionOrder with more than 1 insertionWorker") - } - mongoImport.IngestOptions.NumInsertionWorkers = &numInsertionWorkers + mongoImport.ToolOptions.NumInsertionWorkers = 1 } // get the number of documents per batch - if mongoImport.IngestOptions.BatchSize != nil { - if *mongoImport.IngestOptions.BatchSize < 1 { - return fmt.Errorf("--batchSize argument must be > 0") - } - batchSize = *mongoImport.IngestOptions.BatchSize - } else { - // TODO: TOOLS-335 replace use of global variables - batch size, - // numInsertionWorkers and numDecodingWorkers - mongoImport.IngestOptions.BatchSize = &batchSize + if mongoImport.ToolOptions.BulkBufferSize <= 0 { + mongoImport.ToolOptions.BulkBufferSize = 10000 } // ensure no more than one positional argument is supplied @@ -350,7 +301,10 @@ func (mongoImport *MongoImport) importDocuments(inputReader InputReader) (numImp ordered := mongoImport.IngestOptions.MaintainInsertionOrder // set the batch size for ingestion - readDocChanSize := batchSize * numDecodingWorkers + readDocChanSize := mongoImport.ToolOptions.BulkBufferSize * mongoImport.ToolOptions.NumDecodingWorkers + if readDocChanSize == 0 { + readDocChanSize = 1 + } // readDocChan is buffered with readDocChanSize to ensure we only block // accepting reads if processing is slow @@ -380,19 +334,22 @@ func (mongoImport *MongoImport) importDocuments(inputReader InputReader) (numImp // IngestDocuments takes a slice of documents and either inserts/upserts them - // based on whether an upsert is requested - into the given collection func (mongoImport *MongoImport) IngestDocuments(readChan chan bson.D) (err error) { - numDecodingWorkers := *mongoImport.IngestOptions.NumInsertionWorkers - // initialize the tomb where all goroutines go to die mongoImport.tomb = &tomb.Tomb{} + numInsertionWorkers := mongoImport.ToolOptions.NumInsertionWorkers + if numInsertionWorkers <= 0 { + numInsertionWorkers = 1 + } + // spawn all the worker goroutines, each in its own goroutine - for i := 0; i < numDecodingWorkers; i++ { + for i := 0; i < numInsertionWorkers; i++ { mongoImport.tomb.Go(func() error { // Each ingest worker will return an error which may // be nil or not. It will be not nil in any of this cases: // // 1. There is a problem connecting with the server - // 2. There server becomes unreachable + // 2. The server becomes unreachable // 3. There is an insertion/update error - e.g. duplicate key // error - and stopOnError is set to true return mongoImport.ingestDocs(readChan) @@ -450,7 +407,7 @@ readLoop: // limit so we self impose a limit by using maxMessageSizeBytes // and send documents over the wire when we hit the batch size // or when we're at/over the maximum message size threshold - if len(documents) == batchSize || numMessageBytes >= maxMessageSizeBytes { + if len(documents) == mongoImport.ToolOptions.BulkBufferSize || numMessageBytes >= maxMessageSizeBytes { if err = mongoImport.ingester(documents, collection); err != nil { return err } @@ -528,6 +485,9 @@ func (mongoImport *MongoImport) ingester(documents []bson.Raw, collection *mgo.C numInserted, err = mongoImport.handleUpsert(documents, collection) return err } else { + if len(documents) == 0 { + return + } bulk := collection.Bulk() for _, document := range documents { bulk.Insert(document) @@ -569,9 +529,9 @@ func (mongoImport *MongoImport) getInputReader(in io.Reader) (InputReader, error } } if mongoImport.InputOptions.Type == CSV { - return NewCSVInputReader(fields, in), nil + return NewCSVInputReader(fields, in, mongoImport.ToolOptions.NumDecodingWorkers), nil } else if mongoImport.InputOptions.Type == TSV { - return NewTSVInputReader(fields, in), nil + return NewTSVInputReader(fields, in, mongoImport.ToolOptions.NumDecodingWorkers), nil } - return NewJSONInputReader(mongoImport.InputOptions.JSONArray, in), nil + return NewJSONInputReader(mongoImport.InputOptions.JSONArray, in, mongoImport.ToolOptions.NumDecodingWorkers), nil } diff --git a/mongoimport/mongoimport_test.go b/mongoimport/mongoimport_test.go index 3d6fcc22bf1..4c18dc1d379 100644 --- a/mongoimport/mongoimport_test.go +++ b/mongoimport/mongoimport_test.go @@ -15,24 +15,21 @@ import ( "testing" ) -var ( - testDB = "db" - testCollection = "c" - testServer = "localhost" - testPort = "27017" - sessionProvider *db.SessionProvider +const ( + testDb = "db" + testCollection = "c" ) // checkOnlyHasDocuments returns an error if the documents in the test // collection don't exactly match those that are passed in -func checkOnlyHasDocuments(expectedDocuments []bson.M) error { +func checkOnlyHasDocuments(sessionProvider db.SessionProvider, expectedDocuments []bson.M) error { session, err := sessionProvider.GetSession() if err != nil { return err } defer session.Close() - collection := session.DB(testDB).C(testCollection) + collection := session.DB(testDb).C(testCollection) dbDocuments := []bson.M{} err = collection.Find(nil).Sort("_id").All(&dbDocuments) if err != nil { @@ -57,18 +54,31 @@ func getBasicToolOptions() *commonOpts.ToolOptions { ssl := testutil.GetSSLOptions() auth := testutil.GetAuthOptions() namespace := &commonOpts.Namespace{ - DB: testDB, + DB: testDb, Collection: testCollection, } connection := &commonOpts.Connection{ - Host: testServer, - Port: testPort, + Host: "localhost", + Port: "27017", } return &commonOpts.ToolOptions{ - SSL: &ssl, - Namespace: namespace, - Connection: connection, - Auth: &auth, + SSL: &ssl, + Namespace: namespace, + Connection: connection, + HiddenOptions: &commonOpts.HiddenOptions{}, + Auth: &auth, + } +} + +func NewMongoImport() *MongoImport { + toolOptions := getBasicToolOptions() + inputOptions := &options.InputOptions{} + ingestOptions := &options.IngestOptions{} + return &MongoImport{ + ToolOptions: toolOptions, + InputOptions: inputOptions, + IngestOptions: ingestOptions, + SessionProvider: db.NewSessionProvider(*toolOptions), } } @@ -76,198 +86,77 @@ func TestMongoImportValidateSettings(t *testing.T) { testutil.VerifyTestType(t, testutil.UNIT_TEST_TYPE) Convey("Given a mongoimport instance for validation, ", t, func() { - Convey("an error should be thrown if no database is given", func() { - namespace := &commonOpts.Namespace{} - toolOptions := &commonOpts.ToolOptions{ - Namespace: namespace, - } - inputOptions := &options.InputOptions{ - Type: CSV, - } - ingestOptions := &options.IngestOptions{} - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - } + Convey("an error should be thrown if no collection is given", func() { + mongoImport := NewMongoImport() + mongoImport.ToolOptions.Namespace.DB = "" + mongoImport.ToolOptions.Namespace.Collection = "" So(mongoImport.ValidateSettings([]string{}), ShouldNotBeNil) }) Convey("an error should be thrown if an invalid type is given", func() { - namespace := &commonOpts.Namespace{ - DB: testDB, - Collection: testCollection, - } - toolOptions := &commonOpts.ToolOptions{ - Namespace: namespace, - } - inputOptions := &options.InputOptions{ - Type: "invalid", - } - ingestOptions := &options.IngestOptions{} - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - } + mongoImport := NewMongoImport() + mongoImport.InputOptions.Type = "invalid" So(mongoImport.ValidateSettings([]string{}), ShouldNotBeNil) }) Convey("an error should be thrown if neither --headerline is supplied "+ "nor --fields/--fieldFile", func() { - namespace := &commonOpts.Namespace{ - DB: testDB, - Collection: testCollection, - } - toolOptions := &commonOpts.ToolOptions{ - Namespace: namespace, - } - inputOptions := &options.InputOptions{ - Type: CSV, - } - ingestOptions := &options.IngestOptions{} - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - } + mongoImport := NewMongoImport() + mongoImport.InputOptions.Type = CSV So(mongoImport.ValidateSettings([]string{}), ShouldNotBeNil) }) Convey("no error should be thrown if --headerline is not supplied "+ "but --fields is supplied", func() { - namespace := &commonOpts.Namespace{ - DB: testDB, - Collection: testCollection, - } - toolOptions := &commonOpts.ToolOptions{ - Namespace: namespace, - } - inputOptions := &options.InputOptions{ - Fields: "a,b,c", - Type: CSV, - } - ingestOptions := &options.IngestOptions{} - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - } + mongoImport := NewMongoImport() + mongoImport.InputOptions.Fields = "a,b,c" + mongoImport.InputOptions.Type = CSV So(mongoImport.ValidateSettings([]string{}), ShouldBeNil) }) - Convey("no error should be thrown if no input type is supplied", - func() { - namespace := &commonOpts.Namespace{ - DB: testDB, - Collection: testCollection, - } - toolOptions := &commonOpts.ToolOptions{ - Namespace: namespace, - } - inputOptions := &options.InputOptions{ - Fields: "a,b,c", - } - ingestOptions := &options.IngestOptions{} - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - } - So(mongoImport.ValidateSettings([]string{}), ShouldBeNil) - }) + Convey("no error should be thrown if no input type is supplied", func() { + mongoImport := NewMongoImport() + mongoImport.InputOptions.Fields = "a,b,c" + So(mongoImport.ValidateSettings([]string{}), ShouldBeNil) + }) Convey("no error should be thrown if --headerline is not supplied "+ "but --fieldFile is supplied", func() { - namespace := &commonOpts.Namespace{ - DB: testDB, - Collection: testCollection, - } - toolOptions := &commonOpts.ToolOptions{ - Namespace: namespace, - } - inputOptions := &options.InputOptions{ - FieldFile: "test.csv", - Type: CSV, - } - ingestOptions := &options.IngestOptions{} - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - } + mongoImport := NewMongoImport() + mongoImport.InputOptions.FieldFile = "test.csv" + mongoImport.InputOptions.Type = CSV So(mongoImport.ValidateSettings([]string{}), ShouldBeNil) }) - Convey("an error should be thrown if no collection and no file is "+ - "supplied", func() { - namespace := &commonOpts.Namespace{ - DB: testDB, - } - toolOptions := &commonOpts.ToolOptions{ - Namespace: namespace, - } - inputOptions := &options.InputOptions{ - FieldFile: "test.csv", - Type: CSV, - } - ingestOptions := &options.IngestOptions{} - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - } + Convey("an error should be thrown if no collection and no file is supplied", func() { + mongoImport := NewMongoImport() + mongoImport.InputOptions.FieldFile = "test.csv" + mongoImport.InputOptions.Type = CSV + mongoImport.ToolOptions.Namespace.Collection = "" So(mongoImport.ValidateSettings([]string{}), ShouldNotBeNil) }) Convey("no error should be thrown if no collection but a file is "+ - "supplied and the file name should be used as the collection "+ - "name", func() { - namespace := &commonOpts.Namespace{ - DB: testDB, - } - toolOptions := &commonOpts.ToolOptions{ - Namespace: namespace, - } - inputOptions := &options.InputOptions{ - File: "input", - FieldFile: "test.csv", - Type: CSV, - } - ingestOptions := &options.IngestOptions{} - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - } + "supplied and the file name should be used as the collection name", func() { + mongoImport := NewMongoImport() + mongoImport.InputOptions.File = "input" + mongoImport.InputOptions.FieldFile = "test.csv" + mongoImport.InputOptions.Type = CSV + mongoImport.ToolOptions.Namespace.Collection = "" So(mongoImport.ValidateSettings([]string{}), ShouldBeNil) So(mongoImport.ToolOptions.Namespace.Collection, ShouldEqual, mongoImport.InputOptions.File) }) Convey("with no collection name and a file name the base name of the "+ - "file (without the extension) should be used as the collection "+ - "name", func() { - namespace := &commonOpts.Namespace{ - DB: testDB, - } - toolOptions := &commonOpts.ToolOptions{ - Namespace: namespace, - } - inputOptions := &options.InputOptions{ - File: "/path/to/input/file/dot/input.txt", - FieldFile: "test.csv", - Type: CSV, - } - ingestOptions := &options.IngestOptions{} - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - } + "file (without the extension) should be used as the collection name", func() { + mongoImport := NewMongoImport() + mongoImport.InputOptions.File = "/path/to/input/file/dot/input.txt" + mongoImport.InputOptions.FieldFile = "test.csv" + mongoImport.InputOptions.Type = CSV + mongoImport.ToolOptions.Namespace.Collection = "" So(mongoImport.ValidateSettings([]string{}), ShouldBeNil) - So(mongoImport.ToolOptions.Namespace.Collection, ShouldEqual, - "input") + So(mongoImport.ToolOptions.Namespace.Collection, ShouldEqual, "input") }) }) } @@ -278,34 +167,25 @@ func TestGetSourceReader(t *testing.T) { func() { Convey("an error should be thrown if the given file referenced by "+ "the reader does not exist", func() { - inputOptions := &options.InputOptions{ - File: "/path/to/input/file/dot/input.txt", - } - mongoImport := MongoImport{ - InputOptions: inputOptions, - } + mongoImport := NewMongoImport() + mongoImport.InputOptions.File = "/path/to/input/file/dot/input.txt" + mongoImport.InputOptions.Type = CSV + mongoImport.ToolOptions.Namespace.Collection = "" _, err := mongoImport.getSourceReader() So(err, ShouldNotBeNil) }) Convey("no error should be thrown if the file exists", func() { - inputOptions := &options.InputOptions{ - File: "testdata/test_array.json", - } - mongoImport := MongoImport{ - InputOptions: inputOptions, - } + mongoImport := NewMongoImport() + mongoImport.InputOptions.File = "testdata/test_array.json" + mongoImport.InputOptions.Type = JSON _, err := mongoImport.getSourceReader() So(err, ShouldBeNil) }) Convey("no error should be thrown if stdin is used", func() { - inputOptions := &options.InputOptions{ - File: "", - } - mongoImport := MongoImport{ - InputOptions: inputOptions, - } + mongoImport := NewMongoImport() + mongoImport.InputOptions.File = "" _, err := mongoImport.getSourceReader() So(err, ShouldBeNil) }) @@ -317,75 +197,47 @@ func TestGetInputReader(t *testing.T) { Convey("Given a io.Reader on calling getInputReader", t, func() { Convey("no error should be thrown if neither --fields nor --fieldFile "+ "is used", func() { - inputOptions := &options.InputOptions{ - File: "/path/to/input/file/dot/input.txt", - } - mongoImport := MongoImport{ - InputOptions: inputOptions, - } + mongoImport := NewMongoImport() + mongoImport.InputOptions.File = "/path/to/input/file/dot/input.txt" _, err := mongoImport.getInputReader(&os.File{}) So(err, ShouldBeNil) }) Convey("no error should be thrown if --fields is used", func() { - inputOptions := &options.InputOptions{ - Fields: "a,b,c", - File: "/path/to/input/file/dot/input.txt", - } - mongoImport := MongoImport{ - InputOptions: inputOptions, - } + mongoImport := NewMongoImport() + mongoImport.InputOptions.Fields = "a,b,c" + mongoImport.InputOptions.File = "/path/to/input/file/dot/input.txt" _, err := mongoImport.getInputReader(&os.File{}) So(err, ShouldBeNil) }) Convey("no error should be thrown if --fieldFile is used and it "+ "references a valid file", func() { - inputOptions := &options.InputOptions{ - FieldFile: "testdata/test_array.json", - } - mongoImport := MongoImport{ - InputOptions: inputOptions, - } + mongoImport := NewMongoImport() + mongoImport.InputOptions.FieldFile = "testdata/test_array.json" _, err := mongoImport.getInputReader(&os.File{}) So(err, ShouldBeNil) }) Convey("an error should be thrown if --fieldFile is used and it "+ "references an invalid file", func() { - inputOptions := &options.InputOptions{ - FieldFile: "/path/to/input/file/dot/input.txt", - } - mongoImport := MongoImport{ - InputOptions: inputOptions, - } + mongoImport := NewMongoImport() + mongoImport.InputOptions.FieldFile = "/path/to/input/file/dot/input.txt" _, err := mongoImport.getInputReader(&os.File{}) So(err, ShouldNotBeNil) }) Convey("no error should be thrown for CSV import inputs", func() { - inputOptions := &options.InputOptions{ - Type: CSV, - } - mongoImport := MongoImport{ - InputOptions: inputOptions, - } + mongoImport := NewMongoImport() + mongoImport.InputOptions.Type = CSV _, err := mongoImport.getInputReader(&os.File{}) So(err, ShouldBeNil) }) Convey("no error should be thrown for TSV import inputs", func() { - inputOptions := &options.InputOptions{ - Type: TSV, - } - mongoImport := MongoImport{ - InputOptions: inputOptions, - } + mongoImport := NewMongoImport() + mongoImport.InputOptions.Type = TSV _, err := mongoImport.getInputReader(&os.File{}) So(err, ShouldBeNil) }) Convey("no error should be thrown for JSON import inputs", func() { - inputOptions := &options.InputOptions{ - Type: JSON, - } - mongoImport := MongoImport{ - InputOptions: inputOptions, - } + mongoImport := NewMongoImport() + mongoImport.InputOptions.Type = JSON _, err := mongoImport.getInputReader(&os.File{}) So(err, ShouldBeNil) }) @@ -394,151 +246,76 @@ func TestGetInputReader(t *testing.T) { func TestImportDocuments(t *testing.T) { testutil.VerifyTestType(t, testutil.INTEGRATION_TEST_TYPE) - var err error - Convey("Given a mongoimport instance with which to import documents, on "+ - "calling importDocuments", t, func() { - batchSize := 1 - NumDecodingWorkers := 1 - NumInsertionWorkers := 1 + Convey("With a mongoimport instance", t, func() { + Reset(func() { + sessionProvider := db.NewSessionProvider(*getBasicToolOptions()) + session, err := sessionProvider.GetSession() + if err != nil { + t.Fatalf("error getting session: %v", err) + } + defer session.Close() + session.DB(testDb).C(testCollection).DropCollection() + fmt.Println("droping", testDb, testCollection) + }) Convey("no error should be thrown for CSV import on test data and all "+ "CSV data lines should be imported correctly", func() { - toolOptions := getBasicToolOptions() - inputOptions := &options.InputOptions{ - Type: CSV, - File: "testdata/test.csv", - Fields: "a,b,c", - } - ingestOptions := &options.IngestOptions{ - BatchSize: &batchSize, - NumDecodingWorkers: &NumDecodingWorkers, - NumInsertionWorkers: &NumInsertionWorkers, - } - sessionProvider, err = db.InitSessionProvider(*toolOptions) - So(err, ShouldBeNil) - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - SessionProvider: sessionProvider, - } + mongoImport := NewMongoImport() + mongoImport.InputOptions.Type = CSV + mongoImport.InputOptions.File = "testdata/test.csv" + mongoImport.InputOptions.Fields = "a,b,c" + mongoImport.IngestOptions.WriteConcern = "majority" numImported, err := mongoImport.ImportDocuments() So(err, ShouldBeNil) So(numImported, ShouldEqual, 3) }) Convey("TOOLS-247: no error should be thrown for JSON import on test "+ "data and all documents should be imported correctly", func() { - toolOptions := getBasicToolOptions() - inputOptions := &options.InputOptions{ - File: "testdata/test_plain2.json", - } - ingestOptions := &options.IngestOptions{ - IgnoreBlanks: true, - BatchSize: &batchSize, - NumDecodingWorkers: &NumDecodingWorkers, - NumInsertionWorkers: &NumInsertionWorkers, - } - sessionProvider, err = db.InitSessionProvider(*toolOptions) - So(err, ShouldBeNil) - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - SessionProvider: sessionProvider, - } + mongoImport := NewMongoImport() + mongoImport.InputOptions.File = "testdata/test_plain2.json" + mongoImport.IngestOptions.WriteConcern = "majority" numImported, err := mongoImport.ImportDocuments() So(err, ShouldBeNil) So(numImported, ShouldEqual, 10) }) - Convey("no error should be thrown for CSV import on test data with "+ - "--ignoreBlanks only fields without blanks should be imported", - func() { - toolOptions := getBasicToolOptions() - inputOptions := &options.InputOptions{ - Type: CSV, - File: "testdata/test_blanks.csv", - Fields: "_id,b,c", - } - ingestOptions := &options.IngestOptions{ - IgnoreBlanks: true, - BatchSize: &batchSize, - NumDecodingWorkers: &NumDecodingWorkers, - NumInsertionWorkers: &NumInsertionWorkers, - MaintainInsertionOrder: true, - } - sessionProvider, err = db.InitSessionProvider(*toolOptions) - So(err, ShouldBeNil) - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - SessionProvider: sessionProvider, - } - numImported, err := mongoImport.ImportDocuments() - So(err, ShouldBeNil) - So(numImported, ShouldEqual, 3) - expectedDocuments := []bson.M{ - bson.M{"_id": 1, "b": 2}, - bson.M{"_id": 5, "c": "6e"}, - bson.M{"_id": 7, "b": 8, "c": 6}, - } - So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil) - }) - Convey("no error should be thrown for CSV import on test data without "+ - "--ignoreBlanks supplied - fields with blanks should be imported", - func() { - toolOptions := getBasicToolOptions() - inputOptions := &options.InputOptions{ - Type: CSV, - File: "testdata/test_blanks.csv", - Fields: "_id,b,c", - } - ingestOptions := &options.IngestOptions{ - BatchSize: &batchSize, - NumDecodingWorkers: &NumDecodingWorkers, - NumInsertionWorkers: &NumInsertionWorkers, - MaintainInsertionOrder: true, - } - sessionProvider, err = db.InitSessionProvider(*toolOptions) - So(err, ShouldBeNil) - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - SessionProvider: sessionProvider, - } - numImported, err := mongoImport.ImportDocuments() - So(err, ShouldBeNil) - So(numImported, ShouldEqual, 3) - expectedDocuments := []bson.M{ - bson.M{"_id": 1, "b": 2, "c": ""}, - bson.M{"_id": 5, "b": "", "c": "6e"}, - bson.M{"_id": 7, "b": 8, "c": 6}, - } - So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil) - }) - Convey("no error should be thrown for CSV import on test data with "+ - "--upsert", func() { - toolOptions := getBasicToolOptions() - inputOptions := &options.InputOptions{ - Type: CSV, - File: "testdata/test.csv", - Fields: "_id,b,c", - } - ingestOptions := &options.IngestOptions{ - Upsert: true, - BatchSize: &batchSize, - NumDecodingWorkers: &NumDecodingWorkers, - NumInsertionWorkers: &NumInsertionWorkers, - MaintainInsertionOrder: true, + Convey("CSV import with --ignoreBlanks should import only non-blank fields", func() { + mongoImport := NewMongoImport() + mongoImport.InputOptions.Type = CSV + mongoImport.InputOptions.File = "testdata/test_blanks.csv" + mongoImport.InputOptions.Fields = "_id,b,c" + mongoImport.IngestOptions.IgnoreBlanks = true + + numImported, err := mongoImport.ImportDocuments() + So(err, ShouldBeNil) + So(numImported, ShouldEqual, 3) + expectedDocuments := []bson.M{ + bson.M{"_id": 1, "b": 2}, + bson.M{"_id": 5, "c": "6e"}, + bson.M{"_id": 7, "b": 8, "c": 6}, } - sessionProvider, err = db.InitSessionProvider(*toolOptions) + So(checkOnlyHasDocuments(*mongoImport.SessionProvider, expectedDocuments), ShouldBeNil) + }) + Convey("CSV import without --ignoreBlanks should include blanks", func() { + mongoImport := NewMongoImport() + mongoImport.InputOptions.Type = CSV + mongoImport.InputOptions.File = "testdata/test_blanks.csv" + mongoImport.InputOptions.Fields = "_id,b,c" + numImported, err := mongoImport.ImportDocuments() So(err, ShouldBeNil) - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - SessionProvider: sessionProvider, + So(numImported, ShouldEqual, 3) + expectedDocuments := []bson.M{ + bson.M{"_id": 1, "b": 2, "c": ""}, + bson.M{"_id": 5, "b": "", "c": "6e"}, + bson.M{"_id": 7, "b": 8, "c": 6}, } + So(checkOnlyHasDocuments(*mongoImport.SessionProvider, expectedDocuments), ShouldBeNil) + }) + Convey("no error should be thrown for CSV import on test data with --upsert", func() { + mongoImport := NewMongoImport() + mongoImport.InputOptions.Type = CSV + mongoImport.InputOptions.File = "testdata/test.csv" + mongoImport.InputOptions.Fields = "_id,b,c" + mongoImport.IngestOptions.Upsert = true + mongoImport.IngestOptions.MaintainInsertionOrder = true numImported, err := mongoImport.ImportDocuments() So(err, ShouldBeNil) So(numImported, ShouldEqual, 3) @@ -547,99 +324,55 @@ func TestImportDocuments(t *testing.T) { bson.M{"_id": 3, "b": 5.4, "c": "string"}, bson.M{"_id": 5, "b": 6, "c": 6}, } - So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil) + So(checkOnlyHasDocuments(*mongoImport.SessionProvider, expectedDocuments), ShouldBeNil) }) Convey("no error should be thrown for CSV import on test data with "+ - "--stopOnError. Only documents before error should be imported", - func() { - toolOptions := getBasicToolOptions() - inputOptions := &options.InputOptions{ - Type: CSV, - File: "testdata/test.csv", - Fields: "_id,b,c", - } - ingestOptions := &options.IngestOptions{ - StopOnError: true, - BatchSize: &batchSize, - NumDecodingWorkers: &NumDecodingWorkers, - NumInsertionWorkers: &NumInsertionWorkers, - MaintainInsertionOrder: true, - } - sessionProvider, err = db.InitSessionProvider(*toolOptions) - So(err, ShouldBeNil) - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - SessionProvider: sessionProvider, - } - numImported, err := mongoImport.ImportDocuments() - So(err, ShouldBeNil) - So(numImported, ShouldEqual, 3) - expectedDocuments := []bson.M{ - bson.M{"_id": 1, "b": 2, "c": 3}, - bson.M{"_id": 3, "b": 5.4, "c": "string"}, - bson.M{"_id": 5, "b": 6, "c": 6}, - } - So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil) - }) - Convey("no error should be thrown for CSV import on test data with "+ - "duplicate _id if --stopOnError is not set", func() { - toolOptions := getBasicToolOptions() - inputOptions := &options.InputOptions{ - Type: CSV, - File: "testdata/test_duplicate.csv", - Fields: "_id,b,c", - } - ingestOptions := &options.IngestOptions{ - BatchSize: &batchSize, - NumDecodingWorkers: &NumDecodingWorkers, - NumInsertionWorkers: &NumInsertionWorkers, - } - sessionProvider, err = db.InitSessionProvider(*toolOptions) + "--stopOnError. Only documents before error should be imported", func() { + mongoImport := NewMongoImport() + mongoImport.InputOptions.Type = CSV + mongoImport.InputOptions.File = "testdata/test.csv" + mongoImport.InputOptions.Fields = "_id,b,c" + mongoImport.IngestOptions.StopOnError = true + mongoImport.IngestOptions.MaintainInsertionOrder = true + mongoImport.IngestOptions.WriteConcern = "majority" + numImported, err := mongoImport.ImportDocuments() So(err, ShouldBeNil) - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - SessionProvider: sessionProvider, + So(numImported, ShouldEqual, 3) + expectedDocuments := []bson.M{ + bson.M{"_id": 1, "b": 2, "c": 3}, + bson.M{"_id": 3, "b": 5.4, "c": "string"}, + bson.M{"_id": 5, "b": 6, "c": 6}, } - _, err = mongoImport.ImportDocuments() + So(checkOnlyHasDocuments(*mongoImport.SessionProvider, expectedDocuments), ShouldBeNil) + }) + Convey("CSV import with duplicate _id's should not error if --stopOnError is not set", func() { + mongoImport := NewMongoImport() + + mongoImport.InputOptions.Type = CSV + mongoImport.InputOptions.File = "testdata/test_duplicate.csv" + mongoImport.InputOptions.Fields = "_id,b,c" + mongoImport.IngestOptions.StopOnError = false + numImported, err := mongoImport.ImportDocuments() So(err, ShouldBeNil) + So(numImported, ShouldEqual, 5) + expectedDocuments := []bson.M{ bson.M{"_id": 1, "b": 2, "c": 3}, bson.M{"_id": 3, "b": 5.4, "c": "string"}, bson.M{"_id": 5, "b": 6, "c": 6}, bson.M{"_id": 8, "b": 6, "c": 6}, } - // all documents - except for the one with a duplicate _id - should - // be imported - So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil) + // all docs except the one with duplicate _id - should be imported + So(checkOnlyHasDocuments(*mongoImport.SessionProvider, expectedDocuments), ShouldBeNil) }) - Convey("no error should be thrown for CSV import on test data with "+ - "--drop", func() { - toolOptions := getBasicToolOptions() - inputOptions := &options.InputOptions{ - Type: CSV, - File: "testdata/test.csv", - Fields: "_id,b,c", - } - ingestOptions := &options.IngestOptions{ - Drop: true, - BatchSize: &batchSize, - NumDecodingWorkers: &NumDecodingWorkers, - NumInsertionWorkers: &NumInsertionWorkers, - MaintainInsertionOrder: true, - WriteConcern: "majority", - } - sessionProvider, err = db.InitSessionProvider(*toolOptions) - So(err, ShouldBeNil) - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - SessionProvider: sessionProvider, - } + Convey("no error should be thrown for CSV import on test data with --drop", func() { + mongoImport := NewMongoImport() + mongoImport.InputOptions.Type = CSV + mongoImport.InputOptions.File = "testdata/test.csv" + mongoImport.InputOptions.Fields = "_id,b,c" + mongoImport.IngestOptions.Drop = true + mongoImport.IngestOptions.MaintainInsertionOrder = true + mongoImport.IngestOptions.WriteConcern = "majority" numImported, err := mongoImport.ImportDocuments() So(err, ShouldBeNil) So(numImported, ShouldEqual, 3) @@ -648,85 +381,41 @@ func TestImportDocuments(t *testing.T) { bson.M{"_id": 3, "b": 5.4, "c": "string"}, bson.M{"_id": 5, "b": 6, "c": 6}, } - So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil) + So(checkOnlyHasDocuments(*mongoImport.SessionProvider, expectedDocuments), ShouldBeNil) }) - Convey("no error should be thrown for CSV import on test data with "+ - "--headerLine", func() { - toolOptions := getBasicToolOptions() - inputOptions := &options.InputOptions{ - Type: CSV, - File: "testdata/test.csv", - HeaderLine: true, - } - ingestOptions := &options.IngestOptions{ - BatchSize: &batchSize, - NumDecodingWorkers: &NumDecodingWorkers, - NumInsertionWorkers: &NumInsertionWorkers, - } - sessionProvider, err = db.InitSessionProvider(*toolOptions) - So(err, ShouldBeNil) - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - SessionProvider: sessionProvider, - } + Convey("CSV import on test data with --headerLine should succeed", func() { + mongoImport := NewMongoImport() + mongoImport.InputOptions.Type = CSV + mongoImport.InputOptions.File = "testdata/test.csv" + mongoImport.InputOptions.Fields = "_id,b,c" + mongoImport.InputOptions.HeaderLine = true numImported, err := mongoImport.ImportDocuments() So(err, ShouldBeNil) So(numImported, ShouldEqual, 2) }) - Convey("EOF should be thrown for CSV import on test data with "+ - "--headerLine if the input file is empty", func() { - toolOptions := getBasicToolOptions() + Convey("EOF should be thrown for CSV import with --headerLine if file is empty", func() { csvFile, err := ioutil.TempFile("", "mongoimport_") So(err, ShouldBeNil) csvFile.Close() - inputOptions := &options.InputOptions{ - Type: CSV, - File: csvFile.Name(), - HeaderLine: true, - } - ingestOptions := &options.IngestOptions{ - BatchSize: &batchSize, - NumDecodingWorkers: &NumDecodingWorkers, - NumInsertionWorkers: &NumInsertionWorkers, - } - sessionProvider, err = db.InitSessionProvider(*toolOptions) - So(err, ShouldBeNil) - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - SessionProvider: sessionProvider, - } + + mongoImport := NewMongoImport() + mongoImport.InputOptions.Type = CSV + mongoImport.InputOptions.File = csvFile.Name() + mongoImport.InputOptions.Fields = "_id,b,c" + mongoImport.InputOptions.HeaderLine = true numImported, err := mongoImport.ImportDocuments() So(err, ShouldEqual, io.EOF) So(numImported, ShouldEqual, 0) }) - Convey("no error should be thrown for CSV import on test data with "+ - "--upsert and --upsertFields", func() { - toolOptions := getBasicToolOptions() - inputOptions := &options.InputOptions{ - Type: CSV, - File: "testdata/test.csv", - Fields: "_id,c,b", - } - ingestOptions := &options.IngestOptions{ - Upsert: true, - UpsertFields: "_id", - BatchSize: &batchSize, - NumDecodingWorkers: &NumDecodingWorkers, - NumInsertionWorkers: &NumInsertionWorkers, - MaintainInsertionOrder: true, - } - sessionProvider, err = db.InitSessionProvider(*toolOptions) - So(err, ShouldBeNil) - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - SessionProvider: sessionProvider, - } + Convey("CSV import with --upsert and --upsertFields should succeed", func() { + mongoImport := NewMongoImport() + + mongoImport.InputOptions.Type = CSV + mongoImport.InputOptions.File = "testdata/test.csv" + mongoImport.InputOptions.Fields = "_id,c,b" + mongoImport.IngestOptions.Upsert = true + mongoImport.IngestOptions.UpsertFields = "_id" + mongoImport.IngestOptions.MaintainInsertionOrder = true numImported, err := mongoImport.ImportDocuments() So(err, ShouldBeNil) So(numImported, ShouldEqual, 3) @@ -735,33 +424,16 @@ func TestImportDocuments(t *testing.T) { bson.M{"_id": 3, "c": 5.4, "b": "string"}, bson.M{"_id": 5, "c": 6, "b": 6}, } - So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil) + So(checkOnlyHasDocuments(*mongoImport.SessionProvider, expectedDocuments), ShouldBeNil) }) - Convey("no error should be thrown for CSV import on test data with "+ - "--upsert and --upsertFields and duplicate _id if --stopOnError "+ - "is not set", func() { - inputOptions := &options.InputOptions{ - Type: CSV, - File: "testdata/test_duplicate.csv", - Fields: "_id,b,c", - } - ingestOptions := &options.IngestOptions{ - Upsert: true, - UpsertFields: "_id", - BatchSize: &batchSize, - NumDecodingWorkers: &NumDecodingWorkers, - NumInsertionWorkers: &NumInsertionWorkers, - MaintainInsertionOrder: true, - } - toolOptions := getBasicToolOptions() - sessionProvider, err = db.InitSessionProvider(*toolOptions) - So(err, ShouldBeNil) - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - SessionProvider: sessionProvider, - } + Convey("CSV import with --upsert/--upsertFields with duplicate id should succeed"+ + " if stopOnError is not set", func() { + mongoImport := NewMongoImport() + mongoImport.InputOptions.Type = CSV + mongoImport.InputOptions.File = "testdata/test_duplicate.csv" + mongoImport.InputOptions.Fields = "_id,b,c" + mongoImport.IngestOptions.Upsert = true + mongoImport.IngestOptions.UpsertFields = "_id" numImported, err := mongoImport.ImportDocuments() So(err, ShouldBeNil) So(numImported, ShouldEqual, 5) @@ -771,78 +443,41 @@ func TestImportDocuments(t *testing.T) { bson.M{"_id": 5, "b": 6, "c": 9}, bson.M{"_id": 8, "b": 6, "c": 6}, } - So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil) + So(checkOnlyHasDocuments(*mongoImport.SessionProvider, expectedDocuments), ShouldBeNil) }) Convey("an error should be thrown for CSV import on test data with "+ "duplicate _id if --stopOnError is set", func() { - toolOptions := getBasicToolOptions() - inputOptions := &options.InputOptions{ - Type: CSV, - File: "testdata/test_duplicate.csv", - Fields: "_id,b,c", - } - ingestOptions := &options.IngestOptions{ - StopOnError: true, - BatchSize: &batchSize, - NumDecodingWorkers: &NumDecodingWorkers, - NumInsertionWorkers: &NumInsertionWorkers, - WriteConcern: "1", - MaintainInsertionOrder: true, - } - sessionProvider, err = db.InitSessionProvider(*toolOptions) - So(err, ShouldBeNil) - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - SessionProvider: sessionProvider, - } - _, err = mongoImport.ImportDocuments() + + mongoImport := NewMongoImport() + mongoImport.InputOptions.Type = CSV + mongoImport.InputOptions.File = "testdata/test_duplicate.csv" + mongoImport.InputOptions.Fields = "_id,b,c" + mongoImport.IngestOptions.StopOnError = true + mongoImport.IngestOptions.WriteConcern = "1" + mongoImport.IngestOptions.MaintainInsertionOrder = true + _, err := mongoImport.ImportDocuments() So(err, ShouldNotBeNil) expectedDocuments := []bson.M{ bson.M{"_id": 1, "b": 2, "c": 3}, bson.M{"_id": 3, "b": 5.4, "c": "string"}, bson.M{"_id": 5, "b": 6, "c": 6}, } - So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil) - }) - Convey("an error should be thrown for invalid CSV import on test data", - func() { - inputOptions := &options.InputOptions{ - Type: CSV, - File: "testdata/test_bad.csv", - Fields: "_id,b,c", - } - toolOptions := getBasicToolOptions() - ingestOptions := &options.IngestOptions{ - StopOnError: true, - BatchSize: &batchSize, - NumDecodingWorkers: &NumDecodingWorkers, - NumInsertionWorkers: &NumInsertionWorkers, - MaintainInsertionOrder: true, - } - sessionProvider, err = db.InitSessionProvider(*toolOptions) - So(err, ShouldBeNil) - mongoImport := MongoImport{ - ToolOptions: toolOptions, - InputOptions: inputOptions, - IngestOptions: ingestOptions, - SessionProvider: sessionProvider, - } - _, err = mongoImport.ImportDocuments() - So(err, ShouldNotBeNil) - expectedDocuments := []bson.M{ - bson.M{"_id": 1, "b": 2, "c": 3}, - } - So(checkOnlyHasDocuments(expectedDocuments), ShouldBeNil) - }) - Reset(func() { - session, err := sessionProvider.GetSession() - if err != nil { - t.Fatalf("error getting session: %v", err) + So(checkOnlyHasDocuments(*mongoImport.SessionProvider, expectedDocuments), ShouldBeNil) + }) + Convey("an error should be thrown for invalid CSV import on test data", func() { + mongoImport := NewMongoImport() + mongoImport.InputOptions.Type = CSV + mongoImport.InputOptions.File = "testdata/test_bad.csv" + mongoImport.InputOptions.Fields = "_id,b,c" + mongoImport.IngestOptions.StopOnError = true + mongoImport.IngestOptions.WriteConcern = "1" + mongoImport.IngestOptions.MaintainInsertionOrder = true + _, err := mongoImport.ImportDocuments() + So(err, ShouldNotBeNil) + expectedDocuments := []bson.M{ + bson.M{"_id": 1, "b": 2, "c": 3}, } - defer session.Close() - session.DB(testDB).C(testCollection).DropCollection() + So(checkOnlyHasDocuments(*mongoImport.SessionProvider, expectedDocuments), ShouldBeNil) }) }) } diff --git a/mongoimport/options/options.go b/mongoimport/options/options.go index 0d883134e0d..f201b55aced 100644 --- a/mongoimport/options/options.go +++ b/mongoimport/options/options.go @@ -56,19 +56,6 @@ type IngestOptions struct { // Specifies the number of operating system threads to use during the import process MaintainInsertionOrder bool `long:"maintainInsertionOrder" description:"if given, documents should be inserted in the order of their appearance in the input source"` - // Specifies the number of operating system threads to use during the import process - // TODO: hide this option - NumOSThreads *int `long:"numOsThreads" description:"number of operating system threads to use (defaults to the number of logical CPUs)"` - - // Specifies the number of threads to use in processing data read from the input source - NumDecodingWorkers *int `long:"numDecodingWorkers" description:"number of goroutines to use for converting JSON to BSON (defaults to the number of logical CPUs)"` - - // Specifies the number of threads to use in sending processed data over to the server - NumInsertionWorkers *int `long:"numInsertionWorkers" description:"number of goroutines to use in ingesting data (defaults to 1)"` - - // Specifies the maximum number of documents in each batch sent over to the server - BatchSize *int `long:"batchSize" description:"number of documents to insert in a single batch"` - // Specifies the write concern for each write operation that mongoimport writes to the target database. // By default, mongoimport waits for a majority of members from the replica set to respond before returning. WriteConcern string `long:"writeConcern" default:"majority" description:"write concern options e.g. --writeConcern majority, --writeConcern '{w: 3, wtimeout: 500, fsync: true, j: true}'"` diff --git a/mongoimport/tsv.go b/mongoimport/tsv.go index 5309eb3913a..182ab904405 100644 --- a/mongoimport/tsv.go +++ b/mongoimport/tsv.go @@ -23,9 +23,11 @@ type TSVInputReader struct { tsvReader *bufio.Reader // tsvRecord stores each line of input we read from the underlying reader tsvRecord string - // numProcessed tracks the number of TSV records processed by the underlying - // reader + // numProcessed tracks the number of TSV records processed by the underlying reader numProcessed uint64 + + // numDecoders is the number of concurrent goroutines to use for decoding + numDecoders int } // TSVConvertibleDoc implements the ConvertibleDoc interface for TSV input @@ -37,11 +39,12 @@ type TSVConvertibleDoc struct { // NewTSVInputReader returns a TSVInputReader configured to read input from the // given io.Reader, extracting the specified fields only. -func NewTSVInputReader(fields []string, in io.Reader) *TSVInputReader { +func NewTSVInputReader(fields []string, in io.Reader, numDecoders int) *TSVInputReader { return &TSVInputReader{ Fields: fields, tsvReader: bufio.NewReader(in), numProcessed: uint64(0), + numDecoders: numDecoders, } } @@ -80,7 +83,7 @@ func (tsvInputReader *TSVInputReader) ReadHeadersFromSource() ([]string, error) // hits EOF or an error. If ordered is true, it streams the documents in which // the documents are read func (tsvInputReader *TSVInputReader) StreamDocument(ordered bool, readChan chan bson.D, errChan chan error) { - tsvRecordChan := make(chan ConvertibleDoc, numDecodingWorkers) + tsvRecordChan := make(chan ConvertibleDoc, tsvInputReader.numDecoders) var err error go func() { @@ -104,7 +107,7 @@ func (tsvInputReader *TSVInputReader) StreamDocument(ordered bool, readChan chan tsvInputReader.numProcessed++ } }() - streamDocuments(ordered, tsvRecordChan, readChan, errChan) + streamDocuments(ordered, tsvInputReader.numDecoders, tsvRecordChan, readChan, errChan) } // This is required to satisfy the ConvertibleDoc interface for TSV input. It diff --git a/mongoimport/tsv_test.go b/mongoimport/tsv_test.go index 54f3efb505e..105f9c35a07 100644 --- a/mongoimport/tsv_test.go +++ b/mongoimport/tsv_test.go @@ -23,7 +23,7 @@ func TestTSVStreamDocument(t *testing.T) { bson.DocElem{"b", 2}, bson.DocElem{"c", "3e"}, } - tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents))) + tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go tsvInputReader.StreamDocument(true, docChan, errChan) @@ -40,7 +40,7 @@ func TestTSVStreamDocument(t *testing.T) { bson.DocElem{"c", "3e"}, bson.DocElem{"field3", " may"}, } - tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents))) + tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go tsvInputReader.StreamDocument(true, docChan, errChan) @@ -57,7 +57,7 @@ func TestTSVStreamDocument(t *testing.T) { bson.DocElem{"c", "Inline"}, bson.DocElem{"d", 14}, } - tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents))) + tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go tsvInputReader.StreamDocument(true, docChan, errChan) @@ -81,7 +81,7 @@ func TestTSVStreamDocument(t *testing.T) { bson.DocElem{"c", 6}, }, } - tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents))) + tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go tsvInputReader.StreamDocument(true, docChan, errChan) @@ -108,7 +108,7 @@ func TestTSVStreamDocument(t *testing.T) { bson.DocElem{"b", `"`}, bson.DocElem{"c", 6}, } - tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents))) + tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) errChan := make(chan error) docChan := make(chan bson.D) go tsvInputReader.StreamDocument(true, docChan, errChan) @@ -133,7 +133,7 @@ func TestTSVStreamDocument(t *testing.T) { } fileHandle, err := os.Open("testdata/test.tsv") So(err, ShouldBeNil) - tsvInputReader := NewTSVInputReader(fields, fileHandle) + tsvInputReader := NewTSVInputReader(fields, fileHandle, 1) errChan := make(chan error) docChan := make(chan bson.D) go tsvInputReader.StreamDocument(true, docChan, errChan) @@ -150,7 +150,7 @@ func TestTSVSetHeader(t *testing.T) { Convey("setting the header should read the first line of the TSV", func() { contents := "extraHeader1\textraHeader2\textraHeader3\n" fields := []string{} - tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents))) + tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) So(tsvInputReader.SetHeader(true), ShouldBeNil) So(len(tsvInputReader.Fields), ShouldEqual, 3) }) @@ -158,7 +158,7 @@ func TestTSVSetHeader(t *testing.T) { "the header line with the existing fields", func() { contents := "extraHeader\textraHeader2\textraHeader3\n\n" fields := []string{"a", "b", "c"} - tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents))) + tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte(contents)), 1) // if SetHeader() is called with fields already passed in, // the header should be replaced with the read header line So(tsvInputReader.SetHeader(true), ShouldBeNil) @@ -173,7 +173,7 @@ func TestTSVGetHeaders(t *testing.T) { Convey("With a TSV input reader", t, func() { Convey("getting the header should return any already set headers", func() { fields := []string{"extraHeader1", "extraHeader2", "extraHeader3"} - tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte{})) + tsvInputReader := NewTSVInputReader(fields, bytes.NewReader([]byte{}), 1) So(tsvInputReader.GetHeaders(), ShouldResemble, fields) }) }) @@ -186,7 +186,7 @@ func TestTSVReadHeadersFromSource(t *testing.T) { expectedHeaders := []string{"1", "2", "3"} fileHandle, err := os.Open("testdata/test.tsv") So(err, ShouldBeNil) - tsvInputReader := NewTSVInputReader([]string{}, fileHandle) + tsvInputReader := NewTSVInputReader([]string{}, fileHandle, 1) headers, err := tsvInputReader.ReadHeadersFromSource() So(err, ShouldBeNil) So(headers, ShouldResemble, expectedHeaders) diff --git a/mongorestore/main/mongorestore.go b/mongorestore/main/mongorestore.go index 7516da7aa44..1011610522b 100644 --- a/mongorestore/main/mongorestore.go +++ b/mongorestore/main/mongorestore.go @@ -9,7 +9,6 @@ import ( "github.com/mongodb/mongo-tools/mongorestore" "github.com/mongodb/mongo-tools/mongorestore/options" "os" - "runtime" ) func main() { @@ -37,9 +36,6 @@ func main() { log.SetVerbosity(opts.Verbosity) - runtime.GOMAXPROCS(runtime.NumCPU()) - log.Logf(log.Info, "running mongorestore with %v job threads", outputOpts.JobThreads) - targetDir, err := getTargetDirFromArgs(extraArgs, os.Args, inputOpts.Directory) if err != nil { fmt.Printf("error parsing command line options: %v\n", err) diff --git a/mongorestore/mongorestore.go b/mongorestore/mongorestore.go index 299f518e050..c868eaaad73 100644 --- a/mongorestore/mongorestore.go +++ b/mongorestore/mongorestore.go @@ -38,12 +38,8 @@ type MongoRestore struct { func (restore *MongoRestore) ParseAndValidateOptions() error { // Can't use option pkg defaults for --objcheck because it's two separate flags, // and we need to be able to see if they're both being used. We default to - // true here and then see if noobjcheck is enable. + // true here and then see if noobjcheck is enabled. log.Log(log.DebugHigh, "checking options") - err := restore.ToolOptions.Validate() - if err != nil { - return err - } restore.objCheck = true if restore.InputOptions.NoObjcheck { restore.objCheck = false @@ -74,6 +70,7 @@ func (restore *MongoRestore) ParseAndValidateOptions() error { if !restore.InputOptions.OplogReplay { return fmt.Errorf("cannot use --oplogLimit without --oplogReplay enabled") } + var err error restore.oplogLimit, err = ParseTimestampFlag(restore.InputOptions.OplogLimit) if err != nil { return fmt.Errorf("error parsing timestamp argument to --oplogLimit: %v", err) @@ -97,7 +94,7 @@ func (restore *MongoRestore) ParseAndValidateOptions() error { restore.tempRolesCol = "temproles" } - if restore.OutputOptions.BulkWriters < 0 { + if restore.ToolOptions.HiddenOptions.BulkWriters < 0 { return fmt.Errorf( "cannot specify a negative number of insertion workers per collection") } @@ -147,7 +144,7 @@ func (restore *MongoRestore) Restore() error { } // 2. Restore them... - if restore.OutputOptions.JobThreads > 0 { + if restore.OutputOptions.NumParallelCollections > 0 { restore.manager.Finalize(intents.MultiDatabaseLTF) } else { // use legacy restoration order if we are single-threaded diff --git a/mongorestore/options/options.go b/mongorestore/options/options.go index 5746727ab60..3172db4dc5b 100644 --- a/mongorestore/options/options.go +++ b/mongorestore/options/options.go @@ -14,17 +14,13 @@ func (self *InputOptions) Name() string { } type OutputOptions struct { - Drop bool `long:"drop" description:"Drop each collection before import"` - WriteConcern string `long:"writeConcern" default:"majority" description:"Write concern options e.g. --writeConcern majority, --writeConcern '{w: 3, wtimeout: 500, fsync: true, j: true}'"` - NoIndexRestore bool `long:"noIndexRestore" description:"Don't restore indexes"` - NoOptionsRestore bool `long:"noOptionsRestore" description:"Don't restore options"` - KeepIndexVersion bool `long:"keepIndexVersion" description:"Don't update index version"` - - JobThreads int `long:"numParallelCollections" short:"j" description:"Number of collections to restore in parallel" default:"4"` - BulkWriters int `long:"numInsertionWorkersPerCollection" description:"Number of insert connections per collection" default:"1"` - BulkBufferSize int `long:"batchSize" description:"Maximum number of documents to coalesce into a single bulk insertion" default:"10000"` - PreserveDocOrder bool `long:"preserveOrder" description:"Preserve order of documents during restoration"` - // TODO: add hidden option for NumOSThreads to set GOMAXPROCS on CLI + Drop bool `long:"drop" description:"Drop each collection before import"` + WriteConcern string `long:"writeConcern" default:"majority" description:"Write concern options e.g. --writeConcern majority, --writeConcern '{w: 3, wtimeout: 500, fsync: true, j: true}'"` + NoIndexRestore bool `long:"noIndexRestore" description:"Don't restore indexes"` + NoOptionsRestore bool `long:"noOptionsRestore" description:"Don't restore options"` + KeepIndexVersion bool `long:"keepIndexVersion" description:"Don't update index version"` + MaintainInsertionOrder bool `long:"maintainInsertionOrder" description:"Preserve order of documents during restoration"` + NumParallelCollections int `long:"numParallelCollections" short:"j" description:"Number of collections to restore in parallel" default:"4"` } func (self *OutputOptions) Name() string { diff --git a/mongorestore/restore.go b/mongorestore/restore.go index 81272992cf2..f99d0864339 100644 --- a/mongorestore/restore.go +++ b/mongorestore/restore.go @@ -26,11 +26,11 @@ func (restore *MongoRestore) RestoreIntents() error { restore.progressManager.Start() defer restore.progressManager.Stop() - if restore.OutputOptions.JobThreads > 0 { + if restore.OutputOptions.NumParallelCollections > 0 { resultChan := make(chan error) // start a goroutine for each job thread - for i := 0; i < restore.OutputOptions.JobThreads; i++ { + for i := 0; i < restore.OutputOptions.NumParallelCollections; i++ { go func(id int) { log.Logf(log.DebugHigh, "starting restore routine with id=%v", id) for { @@ -51,7 +51,7 @@ func (restore *MongoRestore) RestoreIntents() error { } // wait until all goroutines are done or one of them errors out - for i := 0; i < restore.OutputOptions.JobThreads; i++ { + for i := 0; i < restore.OutputOptions.NumParallelCollections; i++ { select { case err := <-resultChan: if err != nil { @@ -234,18 +234,18 @@ func (restore *MongoRestore) RestoreCollectionToDB(dbName, colName string, defer restore.progressManager.Detach(bar) } - MaxInsertThreads := restore.OutputOptions.BulkWriters - if restore.OutputOptions.PreserveDocOrder { + MaxInsertThreads := restore.ToolOptions.BulkWriters + if restore.OutputOptions.MaintainInsertionOrder { MaxInsertThreads = 1 } - docChan := make(chan bson.Raw, restore.OutputOptions.BulkBufferSize*MaxInsertThreads) + docChan := make(chan bson.Raw, restore.ToolOptions.BulkBufferSize*MaxInsertThreads) resultChan := make(chan error, MaxInsertThreads) killChan := make(chan struct{}) // make sure goroutines clean up on error defer close(killChan) // start a goroutine for adding up the number of bytes read - bytesReadChan := make(chan int64, restore.OutputOptions.BulkBufferSize*MaxInsertThreads) + bytesReadChan := make(chan int64, restore.ToolOptions.BulkBufferSize*MaxInsertThreads) go func() { for { select { @@ -272,7 +272,7 @@ func (restore *MongoRestore) RestoreCollectionToDB(dbName, colName string, for i := 0; i < MaxInsertThreads; i++ { go func() { - bulk := db.NewBufferedBulkInserter(collection, restore.OutputOptions.BulkBufferSize, false) + bulk := db.NewBufferedBulkInserter(collection, restore.ToolOptions.BulkBufferSize, false) for { select { case rawDoc, alive := <-docChan: diff --git a/vendor/src/github.com/jessevdk/go-flags/.travis.yml b/vendor/src/github.com/jessevdk/go-flags/.travis.yml new file mode 100644 index 00000000000..485d3183902 --- /dev/null +++ b/vendor/src/github.com/jessevdk/go-flags/.travis.yml @@ -0,0 +1,35 @@ +language: go + +install: + # go-flags + - go get -d -v ./... + - go build -v ./... + + # linting + - go get code.google.com/p/go.tools/cmd/vet + - go get github.com/golang/lint + - go install github.com/golang/lint/golint + + # code coverage + - go get code.google.com/p/go.tools/cmd/cover + - go get github.com/onsi/ginkgo/ginkgo + - go get github.com/modocache/gover + - if [ "$TRAVIS_SECURE_ENV_VARS" = "true" ]; then go get github.com/mattn/goveralls; fi + +script: + # go-flags + - $(exit $(gofmt -l . | wc -l)) + - go test -v ./... + + # linting + - go tool vet -all=true -v=true . || true + - $(go env GOPATH | awk 'BEGIN{FS=":"} {print $1}')/bin/golint ./... + + # code coverage + - $(go env GOPATH | awk 'BEGIN{FS=":"} {print $1}')/bin/ginkgo -r -cover + - $(go env GOPATH | awk 'BEGIN{FS=":"} {print $1}')/bin/gover + - if [ "$TRAVIS_SECURE_ENV_VARS" = "true" ]; then $(go env GOPATH | awk 'BEGIN{FS=":"} {print $1}')/bin/goveralls -coverprofile=gover.coverprofile -service=travis-ci -repotoken $COVERALLS_TOKEN; fi + +env: + # coveralls.io + secure: "RCYbiB4P0RjQRIoUx/vG/AjP3mmYCbzOmr86DCww1Z88yNcy3hYr3Cq8rpPtYU5v0g7wTpu4adaKIcqRE9xknYGbqj3YWZiCoBP1/n4Z+9sHW3Dsd9D/GRGeHUus0laJUGARjWoCTvoEtOgTdGQDoX7mH+pUUY0FBltNYUdOiiU=" diff --git a/vendor/src/github.com/jessevdk/go-flags/README.md b/vendor/src/github.com/jessevdk/go-flags/README.md index 0e31cb1789b..b6faef6d38a 100644 --- a/vendor/src/github.com/jessevdk/go-flags/README.md +++ b/vendor/src/github.com/jessevdk/go-flags/README.md @@ -1,6 +1,8 @@ go-flags: a go library for parsing command line arguments ========================================================= +[![GoDoc](https://godoc.org/github.com/jessevdk/go-flags?status.png)](https://godoc.org/github.com/jessevdk/go-flags) [![Build Status](https://travis-ci.org/jessevdk/go-flags.svg?branch=master)](https://travis-ci.org/jessevdk/go-flags) [![Coverage Status](https://img.shields.io/coveralls/jessevdk/go-flags.svg)](https://coveralls.io/r/jessevdk/go-flags?branch=master) + This library provides similar functionality to the builtin flag library of go, but provides much more functionality and nicer formatting. From the documentation: @@ -25,6 +27,7 @@ Supported features: * Supports same option multiple times (can store in slice or last option counts) * Supports maps * Supports function callbacks +* Supports namespaces for (nested) option groups The flags package uses structs, reflection and struct field tags to allow users to specify command line options. This results in very simple diff --git a/vendor/src/github.com/jessevdk/go-flags/arg.go b/vendor/src/github.com/jessevdk/go-flags/arg.go new file mode 100644 index 00000000000..fd8db9c777c --- /dev/null +++ b/vendor/src/github.com/jessevdk/go-flags/arg.go @@ -0,0 +1,21 @@ +package flags + +import ( + "reflect" +) + +// Arg represents a positional argument on the command line. +type Arg struct { + // The name of the positional argument (used in the help) + Name string + + // A description of the positional argument (used in the help) + Description string + + value reflect.Value + tag multiTag +} + +func (a *Arg) isRemaining() bool { + return a.value.Type().Kind() == reflect.Slice +} diff --git a/vendor/src/github.com/jessevdk/go-flags/arg_test.go b/vendor/src/github.com/jessevdk/go-flags/arg_test.go new file mode 100644 index 00000000000..faea28093a6 --- /dev/null +++ b/vendor/src/github.com/jessevdk/go-flags/arg_test.go @@ -0,0 +1,53 @@ +package flags + +import ( + "testing" +) + +func TestPositional(t *testing.T) { + var opts = struct { + Value bool `short:"v"` + + Positional struct { + Command int + Filename string + Rest []string + } `positional-args:"yes" required:"yes"` + }{} + + p := NewParser(&opts, Default) + ret, err := p.ParseArgs([]string{"10", "arg_test.go", "a", "b"}) + + if err != nil { + t.Fatalf("Unexpected error: %v", err) + return + } + + if opts.Positional.Command != 10 { + t.Fatalf("Expected opts.Positional.Command to be 10, but got %v", opts.Positional.Command) + } + + if opts.Positional.Filename != "arg_test.go" { + t.Fatalf("Expected opts.Positional.Filename to be \"arg_test.go\", but got %v", opts.Positional.Filename) + } + + assertStringArray(t, opts.Positional.Rest, []string{"a", "b"}) + assertStringArray(t, ret, []string{}) +} + +func TestPositionalRequired(t *testing.T) { + var opts = struct { + Value bool `short:"v"` + + Positional struct { + Command int + Filename string + Rest []string + } `positional-args:"yes" required:"yes"` + }{} + + p := NewParser(&opts, None) + _, err := p.ParseArgs([]string{"10"}) + + assertError(t, err, ErrRequired, "the required argument `Filename` was not provided") +} diff --git a/vendor/src/github.com/jessevdk/go-flags/assert_test.go b/vendor/src/github.com/jessevdk/go-flags/assert_test.go index 14949be56c3..8e06636b66d 100644 --- a/vendor/src/github.com/jessevdk/go-flags/assert_test.go +++ b/vendor/src/github.com/jessevdk/go-flags/assert_test.go @@ -1,23 +1,70 @@ package flags import ( + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path" + "runtime" "testing" ) +func assertCallerInfo() (string, int) { + ptr := make([]uintptr, 15) + n := runtime.Callers(1, ptr) + + if n == 0 { + return "", 0 + } + + mef := runtime.FuncForPC(ptr[0]) + mefile, meline := mef.FileLine(ptr[0]) + + for i := 2; i < n; i++ { + f := runtime.FuncForPC(ptr[i]) + file, line := f.FileLine(ptr[i]) + + if file != mefile { + return file, line + } + } + + return mefile, meline +} + +func assertErrorf(t *testing.T, format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + + file, line := assertCallerInfo() + + t.Errorf("%s:%d: %s", path.Base(file), line, msg) +} + +func assertFatalf(t *testing.T, format string, args ...interface{}) { + msg := fmt.Sprintf(format, args...) + + file, line := assertCallerInfo() + + t.Fatalf("%s:%d: %s", path.Base(file), line, msg) +} + func assertString(t *testing.T, a string, b string) { if a != b { - t.Errorf("Expected %#v, but got %#v", b, a) + assertErrorf(t, "Expected %#v, but got %#v", b, a) } } + func assertStringArray(t *testing.T, a []string, b []string) { if len(a) != len(b) { - t.Errorf("Expected %#v, but got %#v", b, a) + assertErrorf(t, "Expected %#v, but got %#v", b, a) return } for i, v := range a { if b[i] != v { - t.Errorf("Expected %#v, but got %#v", b, a) + assertErrorf(t, "Expected %#v, but got %#v", b, a) return } } @@ -25,13 +72,13 @@ func assertStringArray(t *testing.T, a []string, b []string) { func assertBoolArray(t *testing.T, a []bool, b []bool) { if len(a) != len(b) { - t.Errorf("Expected %#v, but got %#v", b, a) + assertErrorf(t, "Expected %#v, but got %#v", b, a) return } for i, v := range a { if b[i] != v { - t.Errorf("Expected %#v, but got %#v", b, a) + assertErrorf(t, "Expected %#v, but got %#v", b, a) return } } @@ -56,19 +103,19 @@ func assertParseSuccess(t *testing.T, data interface{}, args ...string) []string func assertError(t *testing.T, err error, typ ErrorType, msg string) { if err == nil { - t.Fatalf("Expected error: %s", msg) + assertFatalf(t, "Expected error: %s", msg) return } if e, ok := err.(*Error); !ok { - t.Fatalf("Expected Error type, but got %#v", err) + assertFatalf(t, "Expected Error type, but got %#v", err) } else { if e.Type != typ { - t.Errorf("Expected error type {%s}, but got {%s}", typ, e.Type) + assertErrorf(t, "Expected error type {%s}, but got {%s}", typ, e.Type) } if e.Message != msg { - t.Errorf("Expected error message %#v, but got %#v", msg, e.Message) + assertErrorf(t, "Expected error message %#v, but got %#v", msg, e.Message) } } } @@ -80,3 +127,51 @@ func assertParseFail(t *testing.T, typ ErrorType, msg string, data interface{}, assertError(t, err, typ, msg) return ret } + +func diff(a, b string) (string, error) { + atmp, err := ioutil.TempFile("", "help-diff") + + if err != nil { + return "", err + } + + btmp, err := ioutil.TempFile("", "help-diff") + + if err != nil { + return "", err + } + + if _, err := io.WriteString(atmp, a); err != nil { + return "", err + } + + if _, err := io.WriteString(btmp, b); err != nil { + return "", err + } + + ret, err := exec.Command("diff", "-u", "-d", "--label", "got", atmp.Name(), "--label", "expected", btmp.Name()).Output() + + os.Remove(atmp.Name()) + os.Remove(btmp.Name()) + + if err.Error() == "exit status 1" { + return string(ret), nil + } + + return string(ret), err +} + +func assertDiff(t *testing.T, actual, expected, msg string) { + if actual == expected { + return + } + + ret, err := diff(actual, expected) + + if err != nil { + assertErrorf(t, "Unexpected diff error: %s", err) + assertErrorf(t, "Unexpected %s, expected:\n\n%s\n\nbut got\n\n%s", msg, expected, actual) + } else { + assertErrorf(t, "Unexpected %s:\n\n%s", msg, ret) + } +} diff --git a/vendor/src/github.com/jessevdk/go-flags/closest.go b/vendor/src/github.com/jessevdk/go-flags/closest.go index 14f58d55110..3b518757c43 100644 --- a/vendor/src/github.com/jessevdk/go-flags/closest.go +++ b/vendor/src/github.com/jessevdk/go-flags/closest.go @@ -9,35 +9,33 @@ func levenshtein(s string, t string) int { return len(s) } - var l1, l2, l3 int - - if len(s) == 1 { - l1 = len(t) + 1 - } else { - l1 = levenshtein(s[1:len(s)-1], t) + 1 - } - - if len(t) == 1 { - l2 = len(s) + 1 - } else { - l2 = levenshtein(t[1:len(t)-1], s) + 1 - } - - l3 = levenshtein(s[1:len(s)], t[1:len(t)]) - - if s[0] != t[0] { - l3++ - } - - if l2 < l1 { - l1 = l2 - } - - if l1 < l3 { - return l1 + dists := make([][]int, len(s)+1) + for i := range dists { + dists[i] = make([]int, len(t)+1) + dists[i][0] = i + } + + for j := range t { + dists[0][j] = j + } + + for i, sc := range s { + for j, tc := range t { + if sc == tc { + dists[i+1][j+1] = dists[i][j] + } else { + dists[i+1][j+1] = dists[i][j] + 1 + if dists[i+1][j] < dists[i+1][j+1] { + dists[i+1][j+1] = dists[i+1][j] + 1 + } + if dists[i][j+1] < dists[i+1][j+1] { + dists[i+1][j+1] = dists[i][j+1] + 1 + } + } + } } - return l3 + return dists[len(s)][len(t)] } func closestChoice(cmd string, choices []string) (string, int) { diff --git a/vendor/src/github.com/jessevdk/go-flags/command.go b/vendor/src/github.com/jessevdk/go-flags/command.go index 9bdac581b01..13332ae331a 100644 --- a/vendor/src/github.com/jessevdk/go-flags/command.go +++ b/vendor/src/github.com/jessevdk/go-flags/command.go @@ -20,8 +20,12 @@ type Command struct { // Aliases for the command Aliases []string + // Whether positional arguments are required + ArgsRequired bool + commands []*Command hasBuiltinHelpGroup bool + args []*Arg } // Commander is an interface which can be implemented by any command added in @@ -50,6 +54,8 @@ type Usage interface { func (c *Command) AddCommand(command string, shortDescription string, longDescription string, data interface{}) (*Command, error) { cmd := newCommand(command, shortDescription, longDescription, data) + cmd.parent = c + if err := cmd.scan(); err != nil { return nil, err } @@ -64,6 +70,8 @@ func (c *Command) AddCommand(command string, shortDescription string, longDescri func (c *Command) AddGroup(shortDescription string, longDescription string, data interface{}) (*Group, error) { group := newGroup(shortDescription, longDescription, data) + group.parent = c + if err := group.scanType(c.scanSubcommandHandler(group)); err != nil { return nil, err } @@ -88,3 +96,11 @@ func (c *Command) Find(name string) *Command { return nil } + +// Args returns a list of positional arguments associated with this command. +func (c *Command) Args() []*Arg { + ret := make([]*Arg, len(c.args)) + copy(ret, c.args) + + return ret +} diff --git a/vendor/src/github.com/jessevdk/go-flags/command_private.go b/vendor/src/github.com/jessevdk/go-flags/command_private.go index bd97cc70a33..1727a3066ce 100644 --- a/vendor/src/github.com/jessevdk/go-flags/command_private.go +++ b/vendor/src/github.com/jessevdk/go-flags/command_private.go @@ -11,7 +11,6 @@ type lookup struct { shortNames map[string]*Option longNames map[string]*Option - required map[*Option]bool commands map[string]*Command } @@ -30,6 +29,44 @@ func (c *Command) scanSubcommandHandler(parentg *Group) scanHandler { return true, err } + positional := mtag.Get("positional-args") + + if len(positional) != 0 { + stype := realval.Type() + + for i := 0; i < stype.NumField(); i++ { + field := stype.Field(i) + + m := newMultiTag((string(field.Tag))) + + if err := m.Parse(); err != nil { + return true, err + } + + name := m.Get("name") + + if len(name) == 0 { + name = field.Name + } + + arg := &Arg{ + Name: name, + Description: m.Get("description"), + + value: realval.Field(i), + tag: m, + } + + c.args = append(c.args, arg) + + if len(mtag.Get("required")) != 0 { + c.ArgsRequired = true + } + } + + return true, nil + } + subcommand := mtag.Get("command") if len(subcommand) != 0 { @@ -104,23 +141,17 @@ func (c *Command) makeLookup() lookup { ret := lookup{ shortNames: make(map[string]*Option), longNames: make(map[string]*Option), - - required: make(map[*Option]bool), - commands: make(map[string]*Command), + commands: make(map[string]*Command), } c.eachGroup(func(g *Group) { for _, option := range g.options { - if option.Required && option.canCli() { - ret.required[option] = true - } - if option.ShortName != 0 { ret.shortNames[string(option.ShortName)] = option } if len(option.LongName) > 0 { - ret.longNames[option.LongName] = option + ret.longNames[option.LongNameWithNamespace()] = option } } }) @@ -209,3 +240,11 @@ func (c *Command) hasCliOptions() bool { return ret } + +func (c *Command) fillParseState(s *parseState) { + s.positional = make([]*Arg, len(c.args)) + copy(s.positional, c.args) + + s.lookup = c.makeLookup() + s.command = c +} diff --git a/vendor/src/github.com/jessevdk/go-flags/command_test.go b/vendor/src/github.com/jessevdk/go-flags/command_test.go index b2df7e3dc6a..a093e1588e2 100644 --- a/vendor/src/github.com/jessevdk/go-flags/command_test.go +++ b/vendor/src/github.com/jessevdk/go-flags/command_test.go @@ -1,6 +1,7 @@ package flags import ( + "fmt" "testing" ) @@ -114,6 +115,23 @@ func TestCommandEstimate(t *testing.T) { assertError(t, err, ErrCommandRequired, "Please specify one command of: add or remove") } +func TestCommandEstimate2(t *testing.T) { + var opts = struct { + Value bool `short:"v"` + + Cmd1 struct { + } `command:"remove"` + + Cmd2 struct { + } `command:"add"` + }{} + + p := NewParser(&opts, None) + _, err := p.ParseArgs([]string{"rmive"}) + + assertError(t, err, ErrUnknownCommand, "Unknown command `rmive', did you mean `remove'?") +} + type testCommand struct { G bool `short:"g"` Executed bool @@ -265,7 +283,7 @@ func TestRequiredOnCommand(t *testing.T) { } `command:"cmd"` }{} - assertParseFail(t, ErrRequired, "the required flag `-v' was not specified", &opts, "cmd") + assertParseFail(t, ErrRequired, fmt.Sprintf("the required flag `%cv' was not specified", defaultShortOptDelimiter), &opts, "cmd") } func TestRequiredAllOnCommand(t *testing.T) { @@ -278,7 +296,7 @@ func TestRequiredAllOnCommand(t *testing.T) { } `command:"cmd"` }{} - assertParseFail(t, ErrRequired, "the required flags `-v' and `--missing' were not specified", &opts, "cmd") + assertParseFail(t, ErrRequired, fmt.Sprintf("the required flags `%smissing' and `%cv' were not specified", defaultLongOptDelimiter, defaultShortOptDelimiter), &opts, "cmd") } func TestDefaultOnCommand(t *testing.T) { diff --git a/vendor/src/github.com/jessevdk/go-flags/completion.go b/vendor/src/github.com/jessevdk/go-flags/completion.go new file mode 100644 index 00000000000..cb7aed6a725 --- /dev/null +++ b/vendor/src/github.com/jessevdk/go-flags/completion.go @@ -0,0 +1,304 @@ +package flags + +import ( + "fmt" + "path/filepath" + "reflect" + "sort" + "strings" + "unicode/utf8" +) + +// Completion is a type containing information of a completion. +type Completion struct { + // The completed item + Item string + + // A description of the completed item (optional) + Description string +} + +type completions []Completion + +func (c completions) Len() int { + return len(c) +} + +func (c completions) Less(i, j int) bool { + return c[i].Item < c[j].Item +} + +func (c completions) Swap(i, j int) { + c[i], c[j] = c[j], c[i] +} + +// Completer is an interface which can be implemented by types +// to provide custom command line argument completion. +type Completer interface { + // Complete receives a prefix representing a (partial) value + // for its type and should provide a list of possible valid + // completions. + Complete(match string) []Completion +} + +type completion struct { + parser *Parser + + ShowDescriptions bool +} + +// Filename is a string alias which provides filename completion. +type Filename string + +func completionsWithoutDescriptions(items []string) []Completion { + ret := make([]Completion, len(items)) + + for i, v := range items { + ret[i].Item = v + } + + return ret +} + +// Complete returns a list of existing files with the given +// prefix. +func (f *Filename) Complete(match string) []Completion { + ret, _ := filepath.Glob(match + "*") + return completionsWithoutDescriptions(ret) +} + +func (c *completion) skipPositional(s *parseState, n int) { + if n >= len(s.positional) { + s.positional = nil + } else { + s.positional = s.positional[n:] + } +} + +func (c *completion) completeOptionNames(names map[string]*Option, prefix string, match string) []Completion { + n := make([]Completion, 0, len(names)) + + for k, opt := range names { + if strings.HasPrefix(k, match) { + n = append(n, Completion{ + Item: prefix + k, + Description: opt.Description, + }) + } + } + + return n +} + +func (c *completion) completeLongNames(s *parseState, prefix string, match string) []Completion { + return c.completeOptionNames(s.lookup.longNames, prefix, match) +} + +func (c *completion) completeShortNames(s *parseState, prefix string, match string) []Completion { + if len(match) != 0 { + return []Completion{ + Completion{ + Item: prefix + match, + }, + } + } + + return c.completeOptionNames(s.lookup.shortNames, prefix, match) +} + +func (c *completion) completeCommands(s *parseState, match string) []Completion { + n := make([]Completion, 0, len(s.command.commands)) + + for _, cmd := range s.command.commands { + if cmd.data != c && strings.HasPrefix(cmd.Name, match) { + n = append(n, Completion{ + Item: cmd.Name, + Description: cmd.ShortDescription, + }) + } + } + + return n +} + +func (c *completion) completeValue(value reflect.Value, prefix string, match string) []Completion { + i := value.Interface() + + var ret []Completion + + if cmp, ok := i.(Completer); ok { + ret = cmp.Complete(match) + } else if value.CanAddr() { + if cmp, ok = value.Addr().Interface().(Completer); ok { + ret = cmp.Complete(match) + } + } + + for i, v := range ret { + ret[i].Item = prefix + v.Item + } + + return ret +} + +func (c *completion) completeArg(arg *Arg, prefix string, match string) []Completion { + if arg.isRemaining() { + // For remaining positional args (that are parsed into a slice), complete + // based on the element type. + return c.completeValue(reflect.New(arg.value.Type().Elem()), prefix, match) + } + + return c.completeValue(arg.value, prefix, match) +} + +func (c *completion) complete(args []string) []Completion { + if len(args) == 0 { + args = []string{""} + } + + s := &parseState{ + args: args, + } + + c.parser.fillParseState(s) + + var opt *Option + + for len(s.args) > 1 { + arg := s.pop() + + if (c.parser.Options&PassDoubleDash) != None && arg == "--" { + opt = nil + c.skipPositional(s, len(s.args)-1) + + break + } + + if argumentIsOption(arg) { + prefix, optname, islong := stripOptionPrefix(arg) + optname, _, argument := splitOption(prefix, optname, islong) + + if argument == nil { + var o *Option + canarg := true + + if islong { + o = s.lookup.longNames[optname] + } else { + for i, r := range optname { + sname := string(r) + o = s.lookup.shortNames[sname] + + if o == nil { + break + } + + if i == 0 && o.canArgument() && len(optname) != len(sname) { + canarg = false + break + } + } + } + + if o == nil && (c.parser.Options&PassAfterNonOption) != None { + opt = nil + c.skipPositional(s, len(s.args)-1) + + break + } else if o != nil && o.canArgument() && !o.OptionalArgument && canarg { + if len(s.args) > 1 { + s.pop() + } else { + opt = o + } + } + } + } else { + if len(s.positional) > 0 { + if !s.positional[0].isRemaining() { + // Don't advance beyond a remaining positional arg (because + // it consumes all subsequent args). + s.positional = s.positional[1:] + } + } else if cmd, ok := s.lookup.commands[arg]; ok { + cmd.fillParseState(s) + } + + opt = nil + } + } + + lastarg := s.args[len(s.args)-1] + var ret []Completion + + if opt != nil { + // Completion for the argument of 'opt' + ret = c.completeValue(opt.value, "", lastarg) + } else if argumentIsOption(lastarg) { + // Complete the option + prefix, optname, islong := stripOptionPrefix(lastarg) + optname, split, argument := splitOption(prefix, optname, islong) + + if argument == nil && !islong { + rname, n := utf8.DecodeRuneInString(optname) + sname := string(rname) + + if opt := s.lookup.shortNames[sname]; opt != nil && opt.canArgument() { + ret = c.completeValue(opt.value, prefix+sname, optname[n:]) + } else { + ret = c.completeShortNames(s, prefix, optname) + } + } else if argument != nil { + if islong { + opt = s.lookup.longNames[optname] + } else { + opt = s.lookup.shortNames[optname] + } + + if opt != nil { + ret = c.completeValue(opt.value, prefix+optname+split, *argument) + } + } else if islong { + ret = c.completeLongNames(s, prefix, optname) + } else { + ret = c.completeShortNames(s, prefix, optname) + } + } else if len(s.positional) > 0 { + // Complete for positional argument + ret = c.completeArg(s.positional[0], "", lastarg) + } else if len(s.command.commands) > 0 { + // Complete for command + ret = c.completeCommands(s, lastarg) + } + + sort.Sort(completions(ret)) + return ret +} + +func (c *completion) execute(args []string) { + ret := c.complete(args) + + if c.ShowDescriptions && len(ret) > 1 { + maxl := 0 + + for _, v := range ret { + if len(v.Item) > maxl { + maxl = len(v.Item) + } + } + + for _, v := range ret { + fmt.Printf("%s", v.Item) + + if len(v.Description) > 0 { + fmt.Printf("%s # %s", strings.Repeat(" ", maxl-len(v.Item)), v.Description) + } + + fmt.Printf("\n") + } + } else { + for _, v := range ret { + fmt.Println(v.Item) + } + } +} diff --git a/vendor/src/github.com/jessevdk/go-flags/completion_test.go b/vendor/src/github.com/jessevdk/go-flags/completion_test.go new file mode 100644 index 00000000000..2d5a97f5976 --- /dev/null +++ b/vendor/src/github.com/jessevdk/go-flags/completion_test.go @@ -0,0 +1,289 @@ +package flags + +import ( + "bytes" + "io" + "os" + "path" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" +) + +type TestComplete struct { +} + +func (t *TestComplete) Complete(match string) []Completion { + options := []string{ + "hello world", + "hello universe", + "hello multiverse", + } + + ret := make([]Completion, 0, len(options)) + + for _, o := range options { + if strings.HasPrefix(o, match) { + ret = append(ret, Completion{ + Item: o, + }) + } + } + + return ret +} + +var completionTestOptions struct { + Verbose bool `short:"v" long:"verbose" description:"Verbose messages"` + Debug bool `short:"d" long:"debug" description:"Enable debug"` + Version bool `long:"version" description:"Show version"` + Required bool `long:"required" required:"true" description:"This is required"` + + AddCommand struct { + Positional struct { + Filename Filename + } `positional-args:"yes"` + } `command:"add" description:"add an item"` + + AddMultiCommand struct { + Positional struct { + Filename []Filename + } `positional-args:"yes"` + } `command:"add-multi" description:"add multiple items"` + + RemoveCommand struct { + Other bool `short:"o"` + File Filename `short:"f" long:"filename"` + } `command:"rm" description:"remove an item"` + + RenameCommand struct { + Completed TestComplete `short:"c" long:"completed"` + } `command:"rename" description:"rename an item"` +} + +type completionTest struct { + Args []string + Completed []string + ShowDescriptions bool +} + +var completionTests []completionTest + +func init() { + _, sourcefile, _, _ := runtime.Caller(0) + completionTestSourcedir := filepath.Join(filepath.SplitList(path.Dir(sourcefile))...) + + completionTestFilename := []string{filepath.Join(completionTestSourcedir, "completion.go"), filepath.Join(completionTestSourcedir, "completion_test.go")} + + completionTests = []completionTest{ + { + // Short names + []string{"-"}, + []string{"-d", "-v"}, + false, + }, + + { + // Short names concatenated + []string{"-dv"}, + []string{"-dv"}, + false, + }, + + { + // Long names + []string{"--"}, + []string{"--debug", "--required", "--verbose", "--version"}, + false, + }, + + { + // Long names with descriptions + []string{"--"}, + []string{ + "--debug # Enable debug", + "--required # This is required", + "--verbose # Verbose messages", + "--version # Show version", + }, + true, + }, + + { + // Long names partial + []string{"--ver"}, + []string{"--verbose", "--version"}, + false, + }, + + { + // Commands + []string{""}, + []string{"add", "add-multi", "rename", "rm"}, + false, + }, + + { + // Commands with descriptions + []string{""}, + []string{ + "add # add an item", + "add-multi # add multiple items", + "rename # rename an item", + "rm # remove an item", + }, + true, + }, + + { + // Commands partial + []string{"r"}, + []string{"rename", "rm"}, + false, + }, + + { + // Positional filename + []string{"add", filepath.Join(completionTestSourcedir, "completion")}, + completionTestFilename, + false, + }, + + { + // Multiple positional filename (1 arg) + []string{"add-multi", filepath.Join(completionTestSourcedir, "completion")}, + completionTestFilename, + false, + }, + { + // Multiple positional filename (2 args) + []string{"add-multi", filepath.Join(completionTestSourcedir, "completion.go"), filepath.Join(completionTestSourcedir, "completion")}, + completionTestFilename, + false, + }, + { + // Multiple positional filename (3 args) + []string{"add-multi", filepath.Join(completionTestSourcedir, "completion.go"), filepath.Join(completionTestSourcedir, "completion.go"), filepath.Join(completionTestSourcedir, "completion")}, + completionTestFilename, + false, + }, + + { + // Flag filename + []string{"rm", "-f", path.Join(completionTestSourcedir, "completion")}, + completionTestFilename, + false, + }, + + { + // Flag short concat last filename + []string{"rm", "-of", path.Join(completionTestSourcedir, "completion")}, + completionTestFilename, + false, + }, + + { + // Flag concat filename + []string{"rm", "-f" + path.Join(completionTestSourcedir, "completion")}, + []string{"-f" + completionTestFilename[0], "-f" + completionTestFilename[1]}, + false, + }, + + { + // Flag equal concat filename + []string{"rm", "-f=" + path.Join(completionTestSourcedir, "completion")}, + []string{"-f=" + completionTestFilename[0], "-f=" + completionTestFilename[1]}, + false, + }, + + { + // Flag concat long filename + []string{"rm", "--filename=" + path.Join(completionTestSourcedir, "completion")}, + []string{"--filename=" + completionTestFilename[0], "--filename=" + completionTestFilename[1]}, + false, + }, + + { + // Flag long filename + []string{"rm", "--filename", path.Join(completionTestSourcedir, "completion")}, + completionTestFilename, + false, + }, + + { + // Custom completed + []string{"rename", "-c", "hello un"}, + []string{"hello universe"}, + false, + }, + } +} + +func TestCompletion(t *testing.T) { + p := NewParser(&completionTestOptions, Default) + c := &completion{parser: p} + + for _, test := range completionTests { + if test.ShowDescriptions { + continue + } + + ret := c.complete(test.Args) + items := make([]string, len(ret)) + + for i, v := range ret { + items[i] = v.Item + } + + if !reflect.DeepEqual(items, test.Completed) { + t.Errorf("Args: %#v, %#v\n Expected: %#v\n Got: %#v", test.Args, test.ShowDescriptions, test.Completed, items) + } + } +} + +func TestParserCompletion(t *testing.T) { + for _, test := range completionTests { + if test.ShowDescriptions { + os.Setenv("GO_FLAGS_COMPLETION", "verbose") + } else { + os.Setenv("GO_FLAGS_COMPLETION", "1") + } + + tmp := os.Stdout + + r, w, _ := os.Pipe() + os.Stdout = w + + out := make(chan string) + + go func() { + var buf bytes.Buffer + + io.Copy(&buf, r) + + out <- buf.String() + }() + + p := NewParser(&completionTestOptions, None) + + _, err := p.ParseArgs(test.Args) + + w.Close() + + os.Stdout = tmp + + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + got := strings.Split(strings.Trim(<-out, "\n"), "\n") + + if !reflect.DeepEqual(got, test.Completed) { + t.Errorf("Expected: %#v\nGot: %#v", test.Completed, got) + } + } + + os.Setenv("GO_FLAGS_COMPLETION", "") +} diff --git a/vendor/src/github.com/jessevdk/go-flags/convert.go b/vendor/src/github.com/jessevdk/go-flags/convert.go index be8de393d9c..191b5f4cd1a 100644 --- a/vendor/src/github.com/jessevdk/go-flags/convert.go +++ b/vendor/src/github.com/jessevdk/go-flags/convert.go @@ -294,6 +294,32 @@ func convert(val string, retval reflect.Value, options multiTag) error { return nil } +func isPrint(s string) bool { + for _, c := range s { + if !strconv.IsPrint(c) { + return false + } + } + + return true +} + +func quoteIfNeeded(s string) string { + if !isPrint(s) { + return strconv.Quote(s) + } + + return s +} + +func unquoteIfPossible(s string) (string, error) { + if len(s) == 0 || s[0] != '"' { + return s, nil + } + + return strconv.Unquote(s) +} + func wrapText(s string, l int, prefix string) string { // Basic text wrapping of s at spaces to fit in l var ret string diff --git a/vendor/src/github.com/jessevdk/go-flags/convert_test.go b/vendor/src/github.com/jessevdk/go-flags/convert_test.go index 4a409111ace..0de0eea7a01 100644 --- a/vendor/src/github.com/jessevdk/go-flags/convert_test.go +++ b/vendor/src/github.com/jessevdk/go-flags/convert_test.go @@ -71,7 +71,7 @@ func TestConvertToString(t *testing.T) { true, []int{-3, 4, -2}, - map[int]float64{-2: 4.5, -3: 0.1}, + map[int]float64{-2: 4.5}, new(bool), float32(5.2), @@ -104,7 +104,7 @@ func TestConvertToString(t *testing.T) { "true", "[-3, 4, -2]", - "{-2:4.5, -3:0.1}", + "{-2:4.5}", "false", "5.2", @@ -171,13 +171,5 @@ func TestWrapText(t *testing.T) { occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.` - if got != expected { - ret, err := helpDiff(got, expected) - - if err != nil { - t.Errorf("Unexpected wrapped text, expected:\n\n%s\n\nbut got\n\n%s", expected, got) - } else { - t.Errorf("Unexpected wrapped text:\n\n%s", ret) - } - } + assertDiff(t, got, expected, "wrapped text") } diff --git a/vendor/src/github.com/jessevdk/go-flags/error.go b/vendor/src/github.com/jessevdk/go-flags/error.go index 3a676932f62..fce9d312128 100644 --- a/vendor/src/github.com/jessevdk/go-flags/error.go +++ b/vendor/src/github.com/jessevdk/go-flags/error.go @@ -2,7 +2,6 @@ package flags import ( "fmt" - "reflect" ) // ErrorType represents the type of error. @@ -55,7 +54,36 @@ const ( ) func (e ErrorType) String() string { - return reflect.TypeOf(e).Name() + switch e { + case ErrUnknown: + return "unknown" + case ErrExpectedArgument: + return "expected argument" + case ErrUnknownFlag: + return "unknown flag" + case ErrUnknownGroup: + return "unknown group" + case ErrMarshal: + return "marshal" + case ErrHelp: + return "help" + case ErrNoArgumentForBool: + return "no argument for bool" + case ErrRequired: + return "required" + case ErrShortNameTooLong: + return "short name too long" + case ErrDuplicatedFlag: + return "duplicated flag" + case ErrTag: + return "tag" + case ErrCommandRequired: + return "command required" + case ErrUnknownCommand: + return "unknown command" + } + + return "unrecognized error type" } // Error represents a parser error. The error returned from Parse is of this diff --git a/vendor/src/github.com/jessevdk/go-flags/example_test.go b/vendor/src/github.com/jessevdk/go-flags/example_test.go index b06dd27ac70..f7be2bb14f2 100644 --- a/vendor/src/github.com/jessevdk/go-flags/example_test.go +++ b/vendor/src/github.com/jessevdk/go-flags/example_test.go @@ -4,7 +4,6 @@ package flags import ( "fmt" "os/exec" - "strings" ) func Example() { @@ -36,6 +35,16 @@ func Example() { // Example of a map IntMap map[string]int `long:"intmap" description:"A map from string to int"` + + // Example of a filename (useful for completion) + Filename Filename `long:"filename" description:"A filename"` + + // Example of positional arguments + Args struct { + Id string + Num int + Rest []string + } `positional-args:"yes" required:"yes"` } // Callback which will invoke callto:<argument> to call a number. @@ -59,15 +68,17 @@ func Example() { "--ptrslice", "world", "--intmap", "a:1", "--intmap", "b:5", - "arg1", - "arg2", - "arg3", + "--filename", "hello.go", + "id", + "10", + "remaining1", + "remaining2", } // Parse flags from `args'. Note that here we use flags.ParseArgs for // the sake of making a working example. Normally, you would simply use // flags.Parse(&opts) which uses os.Args - args, err := ParseArgs(&opts, args) + _, err := ParseArgs(&opts, args) if err != nil { panic(err) @@ -80,7 +91,10 @@ func Example() { fmt.Printf("StringSlice: %v\n", opts.StringSlice) fmt.Printf("PtrSlice: [%v %v]\n", *opts.PtrSlice[0], *opts.PtrSlice[1]) fmt.Printf("IntMap: [a:%v b:%v]\n", opts.IntMap["a"], opts.IntMap["b"]) - fmt.Printf("Remaining args: %s\n", strings.Join(args, " ")) + fmt.Printf("Filename: %v\n", opts.Filename) + fmt.Printf("Args.Id: %s\n", opts.Args.Id) + fmt.Printf("Args.Num: %d\n", opts.Args.Num) + fmt.Printf("Args.Rest: %v\n", opts.Args.Rest) // Output: Verbosity: [true true] // Offset: 5 @@ -89,5 +103,8 @@ func Example() { // StringSlice: [hello world] // PtrSlice: [hello world] // IntMap: [a:1 b:5] - // Remaining args: arg1 arg2 arg3 + // Filename: hello.go + // Args.Id: id + // Args.Num: 10 + // Args.Rest: [remaining1 remaining2] } diff --git a/vendor/src/github.com/jessevdk/go-flags/examples/bash-completion b/vendor/src/github.com/jessevdk/go-flags/examples/bash-completion new file mode 100644 index 00000000000..974f52ad43f --- /dev/null +++ b/vendor/src/github.com/jessevdk/go-flags/examples/bash-completion @@ -0,0 +1,9 @@ +_examples() { + args=("${COMP_WORDS[@]:1:$COMP_CWORD}") + + local IFS=$'\n' + COMPREPLY=($(GO_FLAGS_COMPLETION=1 ${COMP_WORDS[0]} "${args[@]}")) + return 1 +} + +complete -F _examples examples diff --git a/vendor/src/github.com/jessevdk/go-flags/examples/main.go b/vendor/src/github.com/jessevdk/go-flags/examples/main.go index 53369b0f604..4a22be6e86d 100644 --- a/vendor/src/github.com/jessevdk/go-flags/examples/main.go +++ b/vendor/src/github.com/jessevdk/go-flags/examples/main.go @@ -10,8 +10,8 @@ import ( ) type EditorOptions struct { - Input string `short:"i" long:"input" description:"Input file" default:"-"` - Output string `short:"o" long:"output" description:"Output file" default:"-"` + Input flags.Filename `short:"i" long:"input" description:"Input file" default:"-"` + Output flags.Filename `short:"o" long:"output" description:"Output file" default:"-"` } type Point struct { diff --git a/vendor/src/github.com/jessevdk/go-flags/flags.go b/vendor/src/github.com/jessevdk/go-flags/flags.go index 34adf1aee66..e3e72a32680 100644 --- a/vendor/src/github.com/jessevdk/go-flags/flags.go +++ b/vendor/src/github.com/jessevdk/go-flags/flags.go @@ -2,156 +2,237 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package flags provides an extensive command line option parser. -// The flags package is similar in functionality to the go built-in flag package -// but provides more options and uses reflection to provide a convenient and -// succinct way of specifying command line options. -// -// Supported features: -// Options with short names (-v) -// Options with long names (--verbose) -// Options with and without arguments (bool v.s. other type) -// Options with optional arguments and default values -// Multiple option groups each containing a set of options -// Generate and print well-formatted help message -// Passing remaining command line arguments after -- (optional) -// Ignoring unknown command line options (optional) -// Supports -I/usr/include -I=/usr/include -I /usr/include option argument specification -// Supports multiple short options -aux -// Supports all primitive go types (string, int{8..64}, uint{8..64}, float) -// Supports same option multiple times (can store in slice or last option counts) -// Supports maps -// Supports function callbacks -// -// Additional features specific to Windows: -// Options with short names (/v) -// Options with long names (/verbose) -// Windows-style options with arguments use a colon as the delimiter -// Modify generated help message with Windows-style / options -// -// The flags package uses structs, reflection and struct field tags -// to allow users to specify command line options. This results in very simple -// and concise specification of your application options. For example: -// -// type Options struct { -// Verbose []bool `short:"v" long:"verbose" description:"Show verbose debug information"` -// } -// -// This specifies one option with a short name -v and a long name --verbose. -// When either -v or --verbose is found on the command line, a 'true' value -// will be appended to the Verbose field. e.g. when specifying -vvv, the -// resulting value of Verbose will be {[true, true, true]}. -// -// Slice options work exactly the same as primitive type options, except that -// whenever the option is encountered, a value is appended to the slice. -// -// Map options from string to primitive type are also supported. On the command -// line, you specify the value for such an option as key:value. For example -// -// type Options struct { -// AuthorInfo string[string] `short:"a"` -// } -// -// Then, the AuthorInfo map can be filled with something like -// -a name:Jesse -a "surname:van den Kieboom". -// -// Finally, for full control over the conversion between command line argument -// values and options, user defined types can choose to implement the Marshaler -// and Unmarshaler interfaces. -// -// Available field tags: -// short: the short name of the option (single character) -// long: the long name of the option -// required: whether an option is required to appear on the command -// line. If a required option is not present, the parser will -// return ErrRequired (optional) -// description: the description of the option (optional) -// long-description: the long description of the option. Currently only -// displayed in generated man pages (optional) -// no-flag: if non-empty this field is ignored as an option (optional) -// -// optional: whether an argument of the option is optional (optional) -// optional-value: the value of an optional option when the option occurs -// without an argument. This tag can be specified multiple -// times in the case of maps or slices (optional) -// default: the default value of an option. This tag can be specified -// multiple times in the case of slices or maps (optional) -// default-mask: when specified, this value will be displayed in the help -// instead of the actual default value. This is useful -// mostly for hiding otherwise sensitive information from -// showing up in the help. If default-mask takes the special -// value "-", then no default value will be shown at all -// (optional) -// value-name: the name of the argument value (to be shown in the help, -// (optional) -// -// base: a base (radix) used to convert strings to integer values, the -// default base is 10 (i.e. decimal) (optional) -// -// ini-name: the explicit ini option name (optional) -// no-ini: if non-empty this field is ignored as an ini option -// (optional) -// -// group: when specified on a struct field, makes the struct -// field a separate group with the given name (optional) -// command: when specified on a struct field, makes the struct -// field a (sub)command with the given name (optional) -// subcommands-optional: when specified on a command struct field, makes -// any subcommands of that command optional (optional) -// alias: when specified on a command struct field, adds the -// specified name as an alias for the command. Can be -// be specified multiple times to add more than one -// alias (optional) -// -// Either short: or long: must be specified to make the field eligible as an -// option. -// -// -// Option groups: -// -// Option groups are a simple way to semantically separate your options. The -// only real difference is in how your options will appear in the built-in -// generated help. All options in a particular group are shown together in the -// help under the name of the group. -// -// There are currently three ways to specify option groups. -// -// 1. Use NewNamedParser specifying the various option groups. -// 2. Use AddGroup to add a group to an existing parser. -// 3. Add a struct field to the top-level options annotated with the -// group:"group-name" tag. -// -// -// -// Commands: -// -// The flags package also has basic support for commands. Commands are often -// used in monolithic applications that support various commands or actions. -// Take git for example, all of the add, commit, checkout, etc. are called -// commands. Using commands you can easily separate multiple functions of your -// application. -// -// There are currently two ways to specify a command. -// -// 1. Use AddCommand on an existing parser. -// 2. Add a struct field to your options struct annotated with the -// command:"command-name" tag. -// -// The most common, idiomatic way to implement commands is to define a global -// parser instance and implement each command in a separate file. These -// command files should define a go init function which calls AddCommand on -// the global parser. -// -// When parsing ends and there is an active command and that command implements -// the Commander interface, then its Execute method will be run with the -// remaining command line arguments. -// -// Command structs can have options which become valid to parse after the -// command has been specified on the command line. It is currently not valid -// to specify options from the parent level of the command after the command -// name has occurred. Thus, given a top-level option "-v" and a command "add": -// -// Valid: ./app -v add -// Invalid: ./app add -v -// +/* +Package flags provides an extensive command line option parser. +The flags package is similar in functionality to the go built-in flag package +but provides more options and uses reflection to provide a convenient and +succinct way of specifying command line options. + + +Supported features + +The following features are supported in go-flags: + + Options with short names (-v) + Options with long names (--verbose) + Options with and without arguments (bool v.s. other type) + Options with optional arguments and default values + Option default values from ENVIRONMENT_VARIABLES, including slice and map values + Multiple option groups each containing a set of options + Generate and print well-formatted help message + Passing remaining command line arguments after -- (optional) + Ignoring unknown command line options (optional) + Supports -I/usr/include -I=/usr/include -I /usr/include option argument specification + Supports multiple short options -aux + Supports all primitive go types (string, int{8..64}, uint{8..64}, float) + Supports same option multiple times (can store in slice or last option counts) + Supports maps + Supports function callbacks + Supports namespaces for (nested) option groups + +Additional features specific to Windows: + Options with short names (/v) + Options with long names (/verbose) + Windows-style options with arguments use a colon as the delimiter + Modify generated help message with Windows-style / options + + +Basic usage + +The flags package uses structs, reflection and struct field tags +to allow users to specify command line options. This results in very simple +and concise specification of your application options. For example: + + type Options struct { + Verbose []bool `short:"v" long:"verbose" description:"Show verbose debug information"` + } + +This specifies one option with a short name -v and a long name --verbose. +When either -v or --verbose is found on the command line, a 'true' value +will be appended to the Verbose field. e.g. when specifying -vvv, the +resulting value of Verbose will be {[true, true, true]}. + +Slice options work exactly the same as primitive type options, except that +whenever the option is encountered, a value is appended to the slice. + +Map options from string to primitive type are also supported. On the command +line, you specify the value for such an option as key:value. For example + + type Options struct { + AuthorInfo string[string] `short:"a"` + } + +Then, the AuthorInfo map can be filled with something like +-a name:Jesse -a "surname:van den Kieboom". + +Finally, for full control over the conversion between command line argument +values and options, user defined types can choose to implement the Marshaler +and Unmarshaler interfaces. + + +Available field tags + +The following is a list of tags for struct fields supported by go-flags: + + short: the short name of the option (single character) + long: the long name of the option + required: whether an option is required to appear on the command + line. If a required option is not present, the parser will + return ErrRequired (optional) + description: the description of the option (optional) + long-description: the long description of the option. Currently only + displayed in generated man pages (optional) + no-flag: if non-empty this field is ignored as an option (optional) + + optional: whether an argument of the option is optional (optional) + optional-value: the value of an optional option when the option occurs + without an argument. This tag can be specified multiple + times in the case of maps or slices (optional) + default: the default value of an option. This tag can be specified + multiple times in the case of slices or maps (optional) + default-mask: when specified, this value will be displayed in the help + instead of the actual default value. This is useful + mostly for hiding otherwise sensitive information from + showing up in the help. If default-mask takes the special + value "-", then no default value will be shown at all + (optional) + env: the default value of the option is overridden from the + specified environment variable, if one has been defined. + (optional) + env-delim: the 'env' default value from environment is split into + multiple values with the given delimiter string, use with + slices and maps (optional) + value-name: the name of the argument value (to be shown in the help, + (optional) + + base: a base (radix) used to convert strings to integer values, the + default base is 10 (i.e. decimal) (optional) + + ini-name: the explicit ini option name (optional) + no-ini: if non-empty this field is ignored as an ini option + (optional) + + group: when specified on a struct field, makes the struct + field a separate group with the given name (optional) + namespace: when specified on a group struct field, the namespace + gets prepended to every option's long name and + subgroup's namespace of this group, separated by + the parser's namespace delimiter (optional) + command: when specified on a struct field, makes the struct + field a (sub)command with the given name (optional) + subcommands-optional: when specified on a command struct field, makes + any subcommands of that command optional (optional) + alias: when specified on a command struct field, adds the + specified name as an alias for the command. Can be + be specified multiple times to add more than one + alias (optional) + positional-args: when specified on a field with a struct type, + uses the fields of that struct to parse remaining + positional command line arguments into (in order + of the fields). If a field has a slice type, + then all remaining arguments will be added to it. + Positional arguments are optional by default, + unless the "required" tag is specified together + with the "positional-args" tag (optional) + +Either the `short:` tag or the `long:` must be specified to make the field eligible as an +option. + + +Option groups + +Option groups are a simple way to semantically separate your options. All +options in a particular group are shown together in the help under the name +of the group. Namespaces can be used to specify option long names more +precisely and emphasize the options affiliation to their group. + +There are currently three ways to specify option groups. + + 1. Use NewNamedParser specifying the various option groups. + 2. Use AddGroup to add a group to an existing parser. + 3. Add a struct field to the top-level options annotated with the + group:"group-name" tag. + + + +Commands + +The flags package also has basic support for commands. Commands are often +used in monolithic applications that support various commands or actions. +Take git for example, all of the add, commit, checkout, etc. are called +commands. Using commands you can easily separate multiple functions of your +application. + +There are currently two ways to specify a command. + + 1. Use AddCommand on an existing parser. + 2. Add a struct field to your options struct annotated with the + command:"command-name" tag. + +The most common, idiomatic way to implement commands is to define a global +parser instance and implement each command in a separate file. These +command files should define a go init function which calls AddCommand on +the global parser. + +When parsing ends and there is an active command and that command implements +the Commander interface, then its Execute method will be run with the +remaining command line arguments. + +Command structs can have options which become valid to parse after the +command has been specified on the command line. It is currently not valid +to specify options from the parent level of the command after the command +name has occurred. Thus, given a top-level option "-v" and a command "add": + + Valid: ./app -v add + Invalid: ./app add -v + + +Completion + +go-flags has builtin support to provide bash completion of flags, commands +and argument values. To use completion, the binary which uses go-flags +can be invoked in a special environment to list completion of the current +command line argument. It should be noted that this `executes` your application, +and it is up to the user to make sure there are no negative side effects (for +example from init functions). + +Setting the environment variable `GO_FLAGS_COMPLETION=1` enables completion +by replacing the argument parsing routine with the completion routine which +outputs completions for the passed arguments. The basic invocation to +complete a set of arguments is therefore: + + GO_FLAGS_COMPLETION=1 ./completion-example arg1 arg2 arg3 + +where `completion-example` is the binary, `arg1` and `arg2` are +the current arguments, and `arg3` (the last argument) is the argument +to be completed. If the GO_FLAGS_COMPLETION is set to "verbose", then +descriptions of possible completion items will also be shown, if there +are more than 1 completion items. + +To use this with bash completion, a simple file can be written which +calls the binary which supports go-flags completion: + + _completion_example() { + # All arguments except the first one + args=("${COMP_WORDS[@]:1:$COMP_CWORD}") + + # Only split on newlines + local IFS=$'\n' + + # Call completion (note that the first element of COMP_WORDS is + # the executable itself) + COMPREPLY=($(GO_FLAGS_COMPLETION=1 ${COMP_WORDS[0]} "${args[@]}")) + return 0 + } + + complete -F _completion_example completion-example + +Completion requires the parser option PassDoubleDash and is therefore enforced if the environment variable GO_FLAGS_COMPLETION is set. + +Customized completion for argument values is supported by implementing +the flags.Completer interface for the argument value type. An example +of a type which does so is the flags.Filename type, an alias of string +allowing simple filename completion. A slice or array argument value +whose element type implements flags.Completer will also be completed. +*/ package flags diff --git a/vendor/src/github.com/jessevdk/go-flags/group.go b/vendor/src/github.com/jessevdk/go-flags/group.go index a2b368d1937..8b609a3af7b 100644 --- a/vendor/src/github.com/jessevdk/go-flags/group.go +++ b/vendor/src/github.com/jessevdk/go-flags/group.go @@ -29,6 +29,12 @@ type Group struct { // (Command embeds Group) in the built-in generated help and man pages. LongDescription string + // The namespace of the group + Namespace string + + // The parent of the group or nil if it has no parent + parent interface{} + // All the options in the group options []*Option @@ -47,6 +53,8 @@ type Group struct { func (g *Group) AddGroup(shortDescription string, longDescription string, data interface{}) (*Group, error) { group := newGroup(shortDescription, longDescription, data) + group.parent = g + if err := group.scan(); err != nil { return nil, err } diff --git a/vendor/src/github.com/jessevdk/go-flags/group_private.go b/vendor/src/github.com/jessevdk/go-flags/group_private.go index 5242f5dae8b..15251ce39c3 100644 --- a/vendor/src/github.com/jessevdk/go-flags/group_private.go +++ b/vendor/src/github.com/jessevdk/go-flags/group_private.go @@ -32,7 +32,7 @@ func (g *Group) optionByName(name string, namematch func(*Option, string) bool) prio = 3 } - if name == opt.LongName && prio < 2 { + if name == opt.LongNameWithNamespace() && prio < 2 { retopt = opt prio = 2 } @@ -46,33 +46,6 @@ func (g *Group) optionByName(name string, namematch func(*Option, string) bool) return retopt } -func (g *Group) storeDefaults() { - for _, option := range g.options { - // First. empty out the value - if len(option.Default) > 0 { - option.clear() - } - - for _, d := range option.Default { - option.set(&d) - } - - if !option.value.CanSet() { - continue - } - - if option.value.Kind() == reflect.Map { - option.defaultValue = reflect.MakeMap(option.value.Type()) - - for _, k := range option.value.MapKeys() { - option.defaultValue.SetMapIndex(k, option.value.MapIndex(k)) - } - } else { - option.defaultValue = reflect.ValueOf(option.value.Interface()) - } - } -} - func (g *Group) eachGroup(f func(*Group)) { f(g) @@ -151,6 +124,7 @@ func (g *Group) scanStruct(realval reflect.Value, sfield *reflect.StructField, h description := mtag.Get("description") def := mtag.GetMany("default") + optionalValue := mtag.GetMany("optional-value") valueName := mtag.Get("value-name") defaultMask := mtag.Get("default-mask") @@ -163,12 +137,16 @@ func (g *Group) scanStruct(realval reflect.Value, sfield *reflect.StructField, h ShortName: short, LongName: longname, Default: def, + EnvDefaultKey: mtag.Get("env"), + EnvDefaultDelim: mtag.Get("env-delim"), OptionalArgument: optional, OptionalValue: optionalValue, Required: required, ValueName: valueName, DefaultMask: defaultMask, + group: g, + field: field, value: realval.Field(i), tag: mtag, @@ -189,11 +167,13 @@ func (g *Group) checkForDuplicateFlags() *Error { g.eachGroup(func(g *Group) { for _, option := range g.options { if option.LongName != "" { - if otherOption, ok := longNames[option.LongName]; ok { + longName := option.LongNameWithNamespace() + + if otherOption, ok := longNames[longName]; ok { duplicateError = newErrorf(ErrDuplicatedFlag, "option `%s' uses the same long name as option `%s'", option, otherOption) return } - longNames[option.LongName] = option + longNames[longName] = option } if option.ShortName != 0 { if otherOption, ok := shortNames[option.ShortName]; ok { @@ -221,10 +201,13 @@ func (g *Group) scanSubGroupHandler(realval reflect.Value, sfield *reflect.Struc ptrval := reflect.NewAt(realval.Type(), unsafe.Pointer(realval.UnsafeAddr())) description := mtag.Get("description") - if _, err := g.AddGroup(subgroup, description, ptrval.Interface()); err != nil { + group, err := g.AddGroup(subgroup, description, ptrval.Interface()) + if err != nil { return true, err } + group.Namespace = mtag.Get("namespace") + return true, nil } diff --git a/vendor/src/github.com/jessevdk/go-flags/group_test.go b/vendor/src/github.com/jessevdk/go-flags/group_test.go index 35d0767aca6..b5ed9d492d5 100644 --- a/vendor/src/github.com/jessevdk/go-flags/group_test.go +++ b/vendor/src/github.com/jessevdk/go-flags/group_test.go @@ -113,6 +113,33 @@ func TestGroupNestedInline(t *testing.T) { } } +func TestGroupNestedInlineNamespace(t *testing.T) { + var opts = struct { + Opt string `long:"opt"` + + Group struct { + Opt string `long:"opt"` + Group struct { + Opt string `long:"opt"` + } `group:"Subsubgroup" namespace:"sap"` + } `group:"Subgroup" namespace:"sip"` + }{} + + p, ret := assertParserSuccess(t, &opts, "--opt", "a", "--sip.opt", "b", "--sip.sap.opt", "c", "rest") + + assertStringArray(t, ret, []string{"rest"}) + + assertString(t, opts.Opt, "a") + assertString(t, opts.Group.Opt, "b") + assertString(t, opts.Group.Group.Opt, "c") + + for _, name := range []string{"Subgroup", "Subsubgroup"} { + if p.Command.Group.Find(name) == nil { + t.Errorf("Expected to find group '%s'", name) + } + } +} + func TestDuplicateShortFlags(t *testing.T) { var opts struct { Verbose []bool `short:"v" long:"verbose" description:"Show verbose debug information"` diff --git a/vendor/src/github.com/jessevdk/go-flags/help.go b/vendor/src/github.com/jessevdk/go-flags/help.go index 4ded286d03b..e26fcd01cd4 100644 --- a/vendor/src/github.com/jessevdk/go-flags/help.go +++ b/vendor/src/github.com/jessevdk/go-flags/help.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "reflect" + "runtime" "strings" "unicode/utf8" ) @@ -22,6 +23,41 @@ type alignmentInfo struct { indent bool } +const ( + paddingBeforeOption = 2 + distanceBetweenOptionAndDescription = 2 +) + +func (a *alignmentInfo) descriptionStart() int { + ret := a.maxLongLen + distanceBetweenOptionAndDescription + + if a.hasShort { + ret += 2 + } + + if a.maxLongLen > 0 { + ret += 4 + } + + if a.hasValueName { + ret += 3 + } + + return ret +} + +func (a *alignmentInfo) updateLen(name string, indent bool) { + l := utf8.RuneCountInString(name) + + if indent { + l = l + 4 + } + + if l > a.maxLongLen { + a.maxLongLen = l + } +} + func (p *Parser) getAlignmentInfo() alignmentInfo { ret := alignmentInfo{ maxLongLen: 0, @@ -34,7 +70,15 @@ func (p *Parser) getAlignmentInfo() alignmentInfo { ret.terminalColumns = 80 } + var prevcmd *Command + p.eachActiveGroup(func(c *Command, grp *Group) { + if c != prevcmd { + for _, arg := range c.args { + ret.updateLen(arg.Name, c != p.Command) + } + } + for _, info := range grp.options { if !info.canCli() { continue @@ -44,22 +88,11 @@ func (p *Parser) getAlignmentInfo() alignmentInfo { ret.hasShort = true } - lv := utf8.RuneCountInString(info.ValueName) - - if lv != 0 { + if len(info.ValueName) > 0 { ret.hasValueName = true } - l := utf8.RuneCountInString(info.LongName) + lv - - if c != p.Command { - // for indenting - l = l + 4 - } - - if l > ret.maxLongLen { - ret.maxLongLen = l - } + ret.updateLen(info.LongNameWithNamespace()+info.ValueName, c != p.Command) } }) @@ -69,9 +102,6 @@ func (p *Parser) getAlignmentInfo() alignmentInfo { func (p *Parser) writeHelpOption(writer *bufio.Writer, option *Option, info alignmentInfo) { line := &bytes.Buffer{} - distanceBetweenOptionAndDescription := 2 - paddingBeforeOption := 2 - prefix := paddingBeforeOption if info.indent { @@ -87,19 +117,7 @@ func (p *Parser) writeHelpOption(writer *bufio.Writer, option *Option, info alig line.WriteString(" ") } - descstart := info.maxLongLen + paddingBeforeOption + distanceBetweenOptionAndDescription - - if info.hasShort { - descstart += 2 - } - - if info.maxLongLen > 0 { - descstart += 4 - } - - if info.hasValueName { - descstart += 3 - } + descstart := info.descriptionStart() + paddingBeforeOption if len(option.LongName) > 0 { if option.ShortName != 0 { @@ -109,7 +127,7 @@ func (p *Parser) writeHelpOption(writer *bufio.Writer, option *Option, info alig } line.WriteString(defaultLongOptDelimiter) - line.WriteString(option.LongName) + line.WriteString(option.LongNameWithNamespace()) } if option.canArgument() { @@ -153,15 +171,32 @@ func (p *Parser) writeHelpOption(writer *bufio.Writer, option *Option, info alig def, _ = convertToString(option.value, option.tag) } } else if len(defs) != 0 { - def = strings.Join(defs, ", ") + l := len(defs) - 1 + + for i := 0; i < l; i++ { + def += quoteIfNeeded(defs[i]) + ", " + } + + def += quoteIfNeeded(defs[l]) + } + + var envDef string + if option.EnvDefaultKey != "" { + var envPrintable string + if runtime.GOOS == "windows" { + envPrintable = "%" + option.EnvDefaultKey + "%" + } else { + envPrintable = "$" + option.EnvDefaultKey + } + envDef = fmt.Sprintf(" [%s]", envPrintable) } var desc string if def != "" { - desc = fmt.Sprintf("%s (%v)", option.Description, def) + desc = fmt.Sprintf("%s (%v)%s", option.Description, def, envDef) } else { - desc = option.Description + desc = option.Description + envDef } writer.WriteString(wrapText(desc, @@ -236,6 +271,28 @@ func (p *Parser) WriteHelp(writer io.Writer) { fmt.Fprintf(wr, " %s", allcmd.Name) } + if len(allcmd.args) > 0 { + fmt.Fprintf(wr, " ") + } + + for i, arg := range allcmd.args { + if i != 0 { + fmt.Fprintf(wr, " ") + } + + name := arg.Name + + if arg.isRemaining() { + name = name + "..." + } + + if !allcmd.ArgsRequired { + fmt.Fprintf(wr, "[%s]", name) + } else { + fmt.Fprintf(wr, "%s", name) + } + } + if allcmd.Active == nil && len(allcmd.commands) > 0 { var co, cc string @@ -275,26 +332,32 @@ func (p *Parser) WriteHelp(writer io.Writer) { } } - prevcmd := p.Command + c := p.Command - p.eachActiveGroup(func(c *Command, grp *Group) { - first := true + for c != nil { + printcmd := c != p.Command - // Skip built-in help group for all commands except the top-level - // parser - if grp.isBuiltinHelp && c != p.Command { - return - } + c.eachGroup(func(grp *Group) { + first := true - for _, info := range grp.options { - if info.canCli() { - if prevcmd != c { + // Skip built-in help group for all commands except the top-level + // parser + if grp.isBuiltinHelp && c != p.Command { + return + } + + for _, info := range grp.options { + if !info.canCli() { + continue + } + + if printcmd { fmt.Fprintf(wr, "\n[%s command options]\n", c.Name) - prevcmd = c aligninfo.indent = true + printcmd = false } - if first && prevcmd.Group != grp { + if first && cmd.Group != grp { fmt.Fprintln(wr) if aligninfo.indent { @@ -307,8 +370,32 @@ func (p *Parser) WriteHelp(writer io.Writer) { p.writeHelpOption(wr, info, aligninfo) } + }) + + if len(c.args) > 0 { + if c == p.Command { + fmt.Fprintf(wr, "\nArguments:\n") + } else { + fmt.Fprintf(wr, "\n[%s command arguments]\n", c.Name) + } + + maxlen := aligninfo.descriptionStart() + + for _, arg := range c.args { + prefix := strings.Repeat(" ", paddingBeforeOption) + fmt.Fprintf(wr, "%s%s", prefix, arg.Name) + + if len(arg.Description) > 0 { + align := strings.Repeat(" ", maxlen-len(arg.Name)-1) + fmt.Fprintf(wr, ":%s%s", align, arg.Description) + } + + fmt.Fprintln(wr) + } } - }) + + c = c.Active + } scommands := cmd.sortedCommands() diff --git a/vendor/src/github.com/jessevdk/go-flags/help_test.go b/vendor/src/github.com/jessevdk/go-flags/help_test.go index e3e84d8dc26..32220fbe48e 100644 --- a/vendor/src/github.com/jessevdk/go-flags/help_test.go +++ b/vendor/src/github.com/jessevdk/go-flags/help_test.go @@ -3,52 +3,23 @@ package flags import ( "bytes" "fmt" - "io" - "io/ioutil" "os" - "os/exec" + "runtime" "testing" "time" ) -func helpDiff(a, b string) (string, error) { - atmp, err := ioutil.TempFile("", "help-diff") - - if err != nil { - return "", err - } - - btmp, err := ioutil.TempFile("", "help-diff") - - if err != nil { - return "", err - } - - if _, err := io.WriteString(atmp, a); err != nil { - return "", err - } - - if _, err := io.WriteString(btmp, b); err != nil { - return "", err - } - - ret, err := exec.Command("diff", "-u", "-d", "--label", "got", atmp.Name(), "--label", "expected", btmp.Name()).Output() - - os.Remove(atmp.Name()) - os.Remove(btmp.Name()) - - return string(ret), nil -} - type helpOptions struct { Verbose []bool `short:"v" long:"verbose" description:"Show verbose debug information" ini-name:"verbose"` Call func(string) `short:"c" description:"Call phone number" ini-name:"call"` PtrSlice []*string `long:"ptrslice" description:"A slice of pointers to string"` EmptyDescription bool `long:"empty-description"` - Default string `long:"default" default:"Some value" description:"Test default value"` - DefaultArray []string `long:"default-array" default:"Some value" default:"Another value" description:"Test default array value"` + Default string `long:"default" default:"Some\nvalue" description:"Test default value"` + DefaultArray []string `long:"default-array" default:"Some value" default:"Other\tvalue" description:"Test default array value"` DefaultMap map[string]string `long:"default-map" default:"some:value" default:"another:value" description:"Testdefault map value"` + EnvDefault1 string `long:"env-default1" default:"Some value" env:"ENV_DEFAULT" description:"Test env-default1 value"` + EnvDefault2 string `long:"env-default2" env:"ENV_DEFAULT" description:"Test env-default2 value"` OnlyIni string `ini-name:"only-ini" description:"Option only available in ini"` @@ -57,14 +28,30 @@ type helpOptions struct { IntMap map[string]int `long:"intmap" default:"a:1" description:"A map from string to int" ini-name:"int-map"` } `group:"Other Options"` + Group struct { + Opt string `long:"opt" description:"This is a subgroup option"` + + Group struct { + Opt string `long:"opt" description:"This is a subsubgroup option"` + } `group:"Subsubgroup" namespace:"sap"` + } `group:"Subgroup" namespace:"sip"` + Command struct { ExtraVerbose []bool `long:"extra-verbose" description:"Use for extra verbosity"` } `command:"command" alias:"cm" alias:"cmd" description:"A command"` + + Args struct { + Filename string `name:"filename" description:"A filename"` + Num int `name:"num" description:"A number"` + } `positional-args:"yes"` } func TestHelp(t *testing.T) { - var opts helpOptions + oldEnv := EnvSnapshot() + defer oldEnv.Restore() + os.Setenv("ENV_DEFAULT", "env-def") + var opts helpOptions p := NewNamedParser("TestHelp", HelpFlag) p.AddGroup("Application Options", "The application options", &opts) @@ -81,45 +68,91 @@ func TestHelp(t *testing.T) { t.Errorf("Expected flags.ErrHelp type, but got %s", e.Type) } - expected := `Usage: - TestHelp [OPTIONS] <command> + var expected string + + if runtime.GOOS == "windows" { + expected = `Usage: + TestHelp [OPTIONS] [filename] [num] <command> + +Application Options: + /v, /verbose Show verbose debug information + /c: Call phone number + /ptrslice: A slice of pointers to string + /empty-description + /default: Test default value ("Some\nvalue") + /default-array: Test default array value (Some value, "Other\tvalue") + /default-map: Testdefault map value (some:value, another:value) + /env-default1: Test env-default1 value (Some value) [%ENV_DEFAULT%] + /env-default2: Test env-default2 value [%ENV_DEFAULT%] + +Other Options: + /s: A slice of strings (some, value) + /intmap: A map from string to int (a:1) + +Subgroup: + /sip.opt: This is a subgroup option + +Subsubgroup: + /sip.sap.opt: This is a subsubgroup option + +Help Options: + /? Show this help message + /h, /help Show this help message + +Arguments: + filename: A filename + num: A number + +Available commands: + command A command (aliases: cm, cmd) +` + } else { + expected = `Usage: + TestHelp [OPTIONS] [filename] [num] <command> Application Options: -v, --verbose Show verbose debug information -c= Call phone number --ptrslice= A slice of pointers to string --empty-description - --default= Test default value (Some value) - --default-array= Test default array value (Some value, Another value) + --default= Test default value ("Some\nvalue") + --default-array= Test default array value (Some value, "Other\tvalue") --default-map= Testdefault map value (some:value, another:value) + --env-default1= Test env-default1 value (Some value) [$ENV_DEFAULT] + --env-default2= Test env-default2 value [$ENV_DEFAULT] Other Options: -s= A slice of strings (some, value) --intmap= A map from string to int (a:1) +Subgroup: + --sip.opt= This is a subgroup option + +Subsubgroup: + --sip.sap.opt= This is a subsubgroup option + Help Options: -h, --help Show this help message +Arguments: + filename: A filename + num: A number + Available commands: command A command (aliases: cm, cmd) ` - - if e.Message != expected { - ret, err := helpDiff(e.Message, expected) - - if err != nil { - t.Errorf("Unexpected diff error: %s", err) - t.Errorf("Unexpected help message, expected:\n\n%s\n\nbut got\n\n%s", expected, e.Message) - } else { - t.Errorf("Unexpected help message:\n\n%s", ret) - } } + + assertDiff(t, e.Message, expected, "help message") } } func TestMan(t *testing.T) { - var opts helpOptions + oldEnv := EnvSnapshot() + defer oldEnv.Restore() + os.Setenv("ENV_DEFAULT", "env-def") + var opts helpOptions p := NewNamedParser("TestMan", HelpFlag) p.ShortDescription = "Test manpage generation" p.LongDescription = "This is a somewhat `longer' description of what this does" @@ -163,17 +196,32 @@ Test default array value \fB--default-map\fP Testdefault map value .TP +\fB--env-default1\fP +Test env-default1 value +.TP +\fB--env-default2\fP +Test env-default2 value +.TP \fB-s\fP A slice of strings .TP \fB--intmap\fP A map from string to int +.TP +\fB--sip.opt\fP +This is a subgroup option +.TP +\fB--sip.sap.opt\fP +This is a subsubgroup option .SH COMMANDS .SS command A command Longer \fBcommand\fP description +\fBUsage\fP: TestMan [OPTIONS] command [command-OPTIONS] + + \fBAliases\fP: cm, cmd .TP @@ -181,15 +229,7 @@ Longer \fBcommand\fP description Use for extra verbosity `, tt.Format("2 January 2006")) - if got != expected { - ret, err := helpDiff(got, expected) - - if err != nil { - t.Errorf("Unexpected man page, expected:\n\n%s\n\nbut got\n\n%s", expected, got) - } else { - t.Errorf("Unexpected man page:\n\n%s", ret) - } - } + assertDiff(t, got, expected, "man page") } type helpCommandNoOptions struct { @@ -198,8 +238,11 @@ type helpCommandNoOptions struct { } func TestHelpCommand(t *testing.T) { - var opts helpCommandNoOptions + oldEnv := EnvSnapshot() + defer oldEnv.Restore() + os.Setenv("ENV_DEFAULT", "env-def") + var opts helpCommandNoOptions p := NewNamedParser("TestHelpCommand", HelpFlag) p.AddGroup("Application Options", "The application options", &opts) @@ -216,22 +259,25 @@ func TestHelpCommand(t *testing.T) { t.Errorf("Expected flags.ErrHelp type, but got %s", e.Type) } - expected := `Usage: + var expected string + + if runtime.GOOS == "windows" { + expected = `Usage: TestHelpCommand [OPTIONS] command Help Options: - -h, --help Show this help message + /? Show this help message + /h, /help Show this help message ` + } else { + expected = `Usage: + TestHelpCommand [OPTIONS] command - if e.Message != expected { - ret, err := helpDiff(e.Message, expected) - - if err != nil { - t.Errorf("Unexpected diff error: %s", err) - t.Errorf("Unexpected help message, expected:\n\n%s\n\nbut got\n\n%s", expected, e.Message) - } else { - t.Errorf("Unexpected help message:\n\n%s", ret) - } +Help Options: + -h, --help Show this help message +` } + + assertDiff(t, e.Message, expected, "help message") } } diff --git a/vendor/src/github.com/jessevdk/go-flags/ini.go b/vendor/src/github.com/jessevdk/go-flags/ini.go index f92faff482e..72250522559 100644 --- a/vendor/src/github.com/jessevdk/go-flags/ini.go +++ b/vendor/src/github.com/jessevdk/go-flags/ini.go @@ -76,7 +76,7 @@ func IniParse(filename string, data interface{}) error { // information on the ini file format. The returned errors can be of the type // flags.Error or flags.IniError. func (i *IniParser) ParseFile(filename string) error { - i.parser.storeDefaults() + i.parser.clearIsSet() ini, err := readIniFromFile(filename) @@ -112,7 +112,7 @@ func (i *IniParser) ParseFile(filename string) error { // // The returned errors can be of the type flags.Error or flags.IniError. func (i *IniParser) Parse(reader io.Reader) error { - i.parser.storeDefaults() + i.parser.clearIsSet() ini, err := readIni(reader, "") diff --git a/vendor/src/github.com/jessevdk/go-flags/ini_private.go b/vendor/src/github.com/jessevdk/go-flags/ini_private.go index 64f8fb8d950..887aa767984 100644 --- a/vendor/src/github.com/jessevdk/go-flags/ini_private.go +++ b/vendor/src/github.com/jessevdk/go-flags/ini_private.go @@ -6,16 +6,23 @@ import ( "io" "os" "reflect" + "sort" + "strconv" "strings" ) type iniValue struct { - Name string - Value string + Name string + Value string + Quoted bool + LineNumber uint } type iniSection []iniValue -type ini map[string]iniSection +type ini struct { + File string + Sections map[string]iniSection +} func readFullLine(reader *bufio.Reader) (string, error) { var line []byte @@ -57,13 +64,19 @@ func optionIniName(option *Option) string { return option.field.Name } -func writeGroupIni(group *Group, namespace string, writer io.Writer, options IniOptions) { +func writeGroupIni(cmd *Command, group *Group, namespace string, writer io.Writer, options IniOptions) { var sname string if len(namespace) != 0 { - sname = namespace + "." + group.ShortDescription - } else { - sname = group.ShortDescription + sname = namespace + } + + if cmd.Group != group && len(group.ShortDescription) != 0 { + if len(sname) != 0 { + sname += "." + } + + sname += group.ShortDescription } sectionwritten := false @@ -80,7 +93,7 @@ func writeGroupIni(group *Group, namespace string, writer io.Writer, options Ini val := option.value - if (options&IniIncludeDefaults) == IniNone && reflect.DeepEqual(val.Interface(), option.defaultValue.Interface()) { + if (options&IniIncludeDefaults) == IniNone && option.valueIsDefault() { continue } @@ -95,40 +108,49 @@ func writeGroupIni(group *Group, namespace string, writer io.Writer, options Ini oname := optionIniName(option) - commentOption := "" - if (options&(IniIncludeDefaults|IniCommentDefaults)) == IniIncludeDefaults|IniCommentDefaults && reflect.DeepEqual(val.Interface(), option.defaultValue.Interface()) { - commentOption = "; " - } + commentOption := (options&(IniIncludeDefaults|IniCommentDefaults)) == IniIncludeDefaults|IniCommentDefaults && option.valueIsDefault() - switch val.Type().Kind() { + kind := val.Type().Kind() + switch kind { case reflect.Slice: - for idx := 0; idx < val.Len(); idx++ { - v, _ := convertToString(val.Index(idx), option.tag) - fmt.Fprintf(writer, "%s%s = %s\n", commentOption, oname, v) - } + kind = val.Type().Elem().Kind() if val.Len() == 0 { - fmt.Fprintf(writer, "; %s =\n", oname) - } - case reflect.Map: - for _, key := range val.MapKeys() { - k, _ := convertToString(key, option.tag) - v, _ := convertToString(val.MapIndex(key), option.tag) + writeOption(writer, oname, kind, "", "", true, option.iniQuote) + } else { + for idx := 0; idx < val.Len(); idx++ { + v, _ := convertToString(val.Index(idx), option.tag) - fmt.Fprintf(writer, "%s%s = %s:%s\n", commentOption, oname, k, v) + writeOption(writer, oname, kind, "", v, commentOption, option.iniQuote) + } } + case reflect.Map: + kind = val.Type().Elem().Kind() if val.Len() == 0 { - fmt.Fprintf(writer, "; %s =\n", oname) + writeOption(writer, oname, kind, "", "", true, option.iniQuote) + } else { + mkeys := val.MapKeys() + keys := make([]string, len(val.MapKeys())) + kkmap := make(map[string]reflect.Value) + + for i, k := range mkeys { + keys[i], _ = convertToString(k, option.tag) + kkmap[keys[i]] = k + } + + sort.Strings(keys) + + for _, k := range keys { + v, _ := convertToString(val.MapIndex(kkmap[k]), option.tag) + + writeOption(writer, oname, kind, k, v, commentOption, option.iniQuote) + } } default: v, _ := convertToString(val, option.tag) - if len(v) != 0 { - fmt.Fprintf(writer, "%s%s = %s\n", commentOption, oname, v) - } else { - fmt.Fprintf(writer, "%s%s =\n", commentOption, oname) - } + writeOption(writer, oname, kind, "", v, commentOption, option.iniQuote) } if comments { @@ -141,9 +163,30 @@ func writeGroupIni(group *Group, namespace string, writer io.Writer, options Ini } } +func writeOption(writer io.Writer, optionName string, optionType reflect.Kind, optionKey string, optionValue string, commentOption bool, forceQuote bool) { + if forceQuote || (optionType == reflect.String && !isPrint(optionValue)) { + optionValue = strconv.Quote(optionValue) + } + + comment := "" + if commentOption { + comment = "; " + } + + fmt.Fprintf(writer, "%s%s =", comment, optionName) + + if optionKey != "" { + fmt.Fprintf(writer, " %s:%s", optionKey, optionValue) + } else if optionValue != "" { + fmt.Fprintf(writer, " %s", optionValue) + } + + fmt.Fprintln(writer) +} + func writeCommandIni(command *Command, namespace string, writer io.Writer, options IniOptions) { command.eachGroup(func(group *Group) { - writeGroupIni(group, namespace, writer, options) + writeGroupIni(command, group, namespace, writer, options) }) for _, c := range command.commands { @@ -177,7 +220,7 @@ func writeIniToFile(parser *IniParser, filename string, options IniOptions) erro return nil } -func readIniFromFile(filename string) (ini, error) { +func readIniFromFile(filename string) (*ini, error) { file, err := os.Open(filename) if err != nil { @@ -189,8 +232,11 @@ func readIniFromFile(filename string) (ini, error) { return readIni(file, filename) } -func readIni(contents io.Reader, filename string) (ini, error) { - ret := make(ini) +func readIni(contents io.Reader, filename string) (*ini, error) { + ret := &ini{ + File: filename, + Sections: make(map[string]iniSection), + } reader := bufio.NewReader(contents) @@ -198,7 +244,7 @@ func readIni(contents io.Reader, filename string) (ini, error) { section := make(iniSection, 0, 10) sectionname := "" - ret[sectionname] = section + ret.Sections[sectionname] = section var lineno uint @@ -239,11 +285,11 @@ func readIni(contents io.Reader, filename string) (ini, error) { } sectionname = name - section = ret[name] + section = ret.Sections[name] if section == nil { section = make(iniSection, 0, 10) - ret[name] = section + ret.Sections[name] = section } continue @@ -262,13 +308,30 @@ func readIni(contents io.Reader, filename string) (ini, error) { name := strings.TrimSpace(keyval[0]) value := strings.TrimSpace(keyval[1]) + quoted := false + + if len(value) != 0 && value[0] == '"' { + if v, err := strconv.Unquote(value); err == nil { + value = v + + quoted = true + } else { + return nil, &IniError{ + Message: err.Error(), + File: filename, + LineNumber: lineno, + } + } + } section = append(section, iniValue{ - Name: name, - Value: value, + Name: name, + Value: value, + Quoted: quoted, + LineNumber: lineno, }) - ret[sectionname] = section + ret.Sections[sectionname] = section } return ret, nil @@ -294,17 +357,16 @@ func (i *IniParser) matchingGroups(name string) []*Group { return nil } -func (i *IniParser) parse(ini ini) error { +func (i *IniParser) parse(ini *ini) error { p := i.parser - for name, section := range ini { + var quotesLookup = make(map[*Option]bool) + + for name, section := range ini.Sections { groups := i.matchingGroups(name) if len(groups) == 0 { - return newError( - ErrUnknownGroup, - fmt.Sprintf("could not find option group `%s'", name), - ) + return newErrorf(ErrUnknownGroup, "could not find option group `%s'", name) } for _, inival := range section { @@ -326,10 +388,11 @@ func (i *IniParser) parse(ini ini) error { if opt == nil { if (p.Options & IgnoreUnknown) == None { - return newError( - ErrUnknownFlag, - fmt.Sprintf("unknown option: %s", inival.Name), - ) + return &IniError{ + Message: fmt.Sprintf("unknown option: %s", inival.Name), + File: ini.File, + LineNumber: inival.LineNumber, + } } continue @@ -339,15 +402,51 @@ func (i *IniParser) parse(ini ini) error { if !opt.canArgument() && len(inival.Value) == 0 { pval = nil + } else { + if opt.value.Type().Kind() == reflect.Map { + parts := strings.SplitN(inival.Value, ":", 2) + + // only handle unquoting + if len(parts) == 2 && parts[1][0] == '"' { + if v, err := strconv.Unquote(parts[1]); err == nil { + parts[1] = v + + inival.Quoted = true + } else { + return &IniError{ + Message: err.Error(), + File: ini.File, + LineNumber: inival.LineNumber, + } + } + + s := parts[0] + ":" + parts[1] + + pval = &s + } + } } if err := opt.set(pval); err != nil { - return wrapError(err) + return &IniError{ + Message: err.Error(), + File: ini.File, + LineNumber: inival.LineNumber, + } + } + + // either all INI values are quoted or only values who need quoting + if _, ok := quotesLookup[opt]; !inival.Quoted || !ok { + quotesLookup[opt] = inival.Quoted } opt.tag.Set("_read-ini-name", inival.Name) } } + for opt, quoted := range quotesLookup { + opt.iniQuote = quoted + } + return nil } diff --git a/vendor/src/github.com/jessevdk/go-flags/ini_test.go b/vendor/src/github.com/jessevdk/go-flags/ini_test.go index b8fcd267e5a..8c3176c9489 100644 --- a/vendor/src/github.com/jessevdk/go-flags/ini_test.go +++ b/vendor/src/github.com/jessevdk/go-flags/ini_test.go @@ -2,26 +2,38 @@ package flags import ( "bytes" + "fmt" "io/ioutil" "os" + "reflect" "strings" "testing" ) func TestWriteIni(t *testing.T) { + oldEnv := EnvSnapshot() + defer oldEnv.Restore() + os.Setenv("ENV_DEFAULT", "env-def") + var tests = []struct { args []string options IniOptions expected string }{ { - []string{"-vv", "--intmap=a:2", "--intmap", "b:3", "command"}, + []string{"-vv", "--intmap=a:2", "--intmap", "b:3", "filename", "0", "command"}, IniDefault, `[Application Options] ; Show verbose debug information verbose = true verbose = true +; Test env-default1 value +EnvDefault1 = env-def + +; Test env-default2 value +EnvDefault2 = env-def + [Other Options] ; A map from string to int int-map = a:2 @@ -30,7 +42,7 @@ int-map = b:3 `, }, { - []string{"-vv", "--intmap=a:2", "--intmap", "b:3", "command"}, + []string{"-vv", "--intmap=a:2", "--intmap", "b:3", "filename", "0", "command"}, IniDefault | IniIncludeDefaults, `[Application Options] ; Show verbose debug information @@ -43,15 +55,21 @@ verbose = true EmptyDescription = false ; Test default value -Default = Some value +Default = "Some\nvalue" ; Test default array value DefaultArray = Some value -DefaultArray = Another value +DefaultArray = "Other\tvalue" ; Testdefault map value -DefaultMap = some:value DefaultMap = another:value +DefaultMap = some:value + +; Test env-default1 value +EnvDefault1 = env-def + +; Test env-default2 value +EnvDefault2 = env-def ; Option only available in ini only-ini = @@ -65,14 +83,22 @@ StringSlice = value int-map = a:2 int-map = b:3 -[command.A command] +[Subgroup] +; This is a subgroup option +Opt = + +[Subsubgroup] +; This is a subsubgroup option +Opt = + +[command] ; Use for extra verbosity ; ExtraVerbose = `, }, { - []string{"command"}, + []string{"filename", "0", "command"}, IniDefault | IniIncludeDefaults | IniCommentDefaults, `[Application Options] ; Show verbose debug information @@ -84,15 +110,21 @@ int-map = b:3 ; EmptyDescription = false ; Test default value -; Default = Some value +; Default = "Some\nvalue" ; Test default array value ; DefaultArray = Some value -; DefaultArray = Another value +; DefaultArray = "Other\tvalue" ; Testdefault map value -; DefaultMap = some:value ; DefaultMap = another:value +; DefaultMap = some:value + +; Test env-default1 value +EnvDefault1 = env-def + +; Test env-default2 value +EnvDefault2 = env-def ; Option only available in ini ; only-ini = @@ -105,14 +137,22 @@ int-map = b:3 ; A map from string to int ; int-map = a:1 -[command.A command] +[Subgroup] +; This is a subgroup option +; Opt = + +[Subsubgroup] +; This is a subsubgroup option +; Opt = + +[command] ; Use for extra verbosity ; ExtraVerbose = `, }, { - []string{"--default=New value", "--default-array=New value", "--default-map=new:value", "command"}, + []string{"--default=New value", "--default-array=New value", "--default-map=new:value", "filename", "0", "command"}, IniDefault | IniIncludeDefaults | IniCommentDefaults, `[Application Options] ; Show verbose debug information @@ -132,6 +172,12 @@ DefaultArray = New value ; Testdefault map value DefaultMap = new:value +; Test env-default1 value +EnvDefault1 = env-def + +; Test env-default2 value +EnvDefault2 = env-def + ; Option only available in ini ; only-ini = @@ -143,7 +189,15 @@ DefaultMap = new:value ; A map from string to int ; int-map = a:1 -[command.A command] +[Subgroup] +; This is a subgroup option +; Opt = + +[Subsubgroup] +; This is a subsubgroup option +; Opt = + +[command] ; Use for extra verbosity ; ExtraVerbose = @@ -171,15 +225,8 @@ DefaultMap = new:value got := b.String() expected := test.expected - if got != expected { - ret, err := helpDiff(got, expected) - - if err != nil { - t.Errorf("Unexpected ini with arguments %+v and ini options %b, expected:\n\n%s\n\nbut got\n\n%s", test.args, test.options, expected, got) - } else { - t.Errorf("Unexpected ini with arguments %+v and ini options %b:\n\n%s", test.args, test.options, ret) - } - } + msg := fmt.Sprintf("with arguments %+v and ini options %b", test.args, test.options) + assertDiff(t, got, expected, msg) } } @@ -196,16 +243,23 @@ func TestReadIni(t *testing.T) { verbose = true verbose = true +DefaultMap = another:"value\n1" +DefaultMap = some:value 2 + [Application Options] ; A slice of pointers to string ; PtrSlice = ; Test default value -Default = Some value +Default = "New\nvalue" + +; Test env-default1 value +EnvDefault1 = New value [Other Options] # A slice of strings -# StringSlice = +StringSlice = "some\nvalue" +StringSlice = another value ; A map from string to int int-map = a:2 @@ -222,6 +276,16 @@ int-map = b:3 assertBoolArray(t, opts.Verbose, []bool{true, true}) + if v := map[string]string{"another": "value\n1", "some": "value 2"}; !reflect.DeepEqual(opts.DefaultMap, v) { + t.Fatalf("Expected %#v for DefaultMap but got %#v", v, opts.DefaultMap) + } + + assertString(t, opts.Default, "New\nvalue") + + assertString(t, opts.EnvDefault1, "New value") + + assertStringArray(t, opts.Other.StringSlice, []string{"some\nvalue", "another value"}) + if v, ok := opts.Other.IntMap["a"]; !ok { t.Errorf("Expected \"a\" in Other.IntMap") } else if v != 2 { @@ -235,6 +299,222 @@ int-map = b:3 } } +func TestReadAndWriteIni(t *testing.T) { + var tests = []struct { + options IniOptions + read string + write string + }{ + { + IniIncludeComments, + `[Application Options] +; Show verbose debug information +verbose = true +verbose = true + +; Test default value +Default = "quote me" + +; Test default array value +DefaultArray = 1 +DefaultArray = "2" +DefaultArray = 3 + +; Testdefault map value +; DefaultMap = + +; Test env-default1 value +EnvDefault1 = env-def + +; Test env-default2 value +EnvDefault2 = env-def + +[Other Options] +; A slice of strings +; StringSlice = + +; A map from string to int +int-map = a:2 +int-map = b:"3" + +`, + `[Application Options] +; Show verbose debug information +verbose = true +verbose = true + +; Test default value +Default = "quote me" + +; Test default array value +DefaultArray = 1 +DefaultArray = 2 +DefaultArray = 3 + +; Testdefault map value +; DefaultMap = + +; Test env-default1 value +EnvDefault1 = env-def + +; Test env-default2 value +EnvDefault2 = env-def + +[Other Options] +; A slice of strings +; StringSlice = + +; A map from string to int +int-map = a:2 +int-map = b:3 + +`, + }, + { + IniIncludeComments, + `[Application Options] +; Show verbose debug information +verbose = true +verbose = true + +; Test default value +Default = "quote me" + +; Test default array value +DefaultArray = "1" +DefaultArray = "2" +DefaultArray = "3" + +; Testdefault map value +; DefaultMap = + +; Test env-default1 value +EnvDefault1 = env-def + +; Test env-default2 value +EnvDefault2 = env-def + +[Other Options] +; A slice of strings +; StringSlice = + +; A map from string to int +int-map = a:"2" +int-map = b:"3" + +`, + `[Application Options] +; Show verbose debug information +verbose = true +verbose = true + +; Test default value +Default = "quote me" + +; Test default array value +DefaultArray = "1" +DefaultArray = "2" +DefaultArray = "3" + +; Testdefault map value +; DefaultMap = + +; Test env-default1 value +EnvDefault1 = env-def + +; Test env-default2 value +EnvDefault2 = env-def + +[Other Options] +; A slice of strings +; StringSlice = + +; A map from string to int +int-map = a:"2" +int-map = b:"3" + +`, + }, + } + + for _, test := range tests { + var opts helpOptions + + p := NewNamedParser("TestIni", Default) + p.AddGroup("Application Options", "The application options", &opts) + + inip := NewIniParser(p) + + read := strings.NewReader(test.read) + err := inip.Parse(read) + if err != nil { + t.Fatalf("Unexpected error: %s", err) + } + + var write bytes.Buffer + inip.Write(&write, test.options) + + got := write.String() + + msg := fmt.Sprintf("with ini options %b", test.options) + assertDiff(t, got, test.write, msg) + } +} + +func TestReadIniWrongQuoting(t *testing.T) { + var tests = []struct { + iniFile string + lineNumber uint + }{ + { + iniFile: `Default = "New\nvalue`, + lineNumber: 1, + }, + { + iniFile: `StringSlice = "New\nvalue`, + lineNumber: 1, + }, + { + iniFile: `StringSlice = "New\nvalue" + StringSlice = "Second\nvalue`, + lineNumber: 2, + }, + { + iniFile: `DefaultMap = some:"value`, + lineNumber: 1, + }, + { + iniFile: `DefaultMap = some:value + DefaultMap = another:"value`, + lineNumber: 2, + }, + } + + for _, test := range tests { + var opts helpOptions + + p := NewNamedParser("TestIni", Default) + p.AddGroup("Application Options", "The application options", &opts) + + inip := NewIniParser(p) + + inic := test.iniFile + + b := strings.NewReader(inic) + err := inip.Parse(b) + + if err == nil { + t.Fatalf("Expect error") + } + + iniError := err.(*IniError) + + if iniError.LineNumber != test.lineNumber { + t.Fatalf("Expect error on line %d", test.lineNumber) + } + } +} + func TestIniCommands(t *testing.T) { var opts struct { Value string `short:"v" long:"value"` @@ -261,6 +541,7 @@ AliasName = 5 [add.Other Options] other = subgroup + ` b := strings.NewReader(inic) @@ -277,6 +558,13 @@ other = subgroup } assertString(t, opts.Add.Other.O, "subgroup") + + // Test writing it back + buf := &bytes.Buffer{} + + inip.Write(buf, IniDefault) + + assertDiff(t, buf.String(), inic, "ini contents") } func TestIniNoIni(t *testing.T) { @@ -300,7 +588,15 @@ value = some value t.Fatalf("Expected error") } - assertError(t, err, ErrUnknownFlag, "unknown option: value") + iniError := err.(*IniError) + + if v := uint(2); iniError.LineNumber != v { + t.Errorf("Expected opts.Add.Name to be %d, but got %d", v, iniError.LineNumber) + } + + if v := "unknown option: value"; iniError.Message != v { + t.Errorf("Expected opts.Add.Name to be %s, but got %s", v, iniError.Message) + } } func TestIniParse(t *testing.T) { @@ -359,7 +655,77 @@ func TestWriteFile(t *testing.T) { expected := "[Application Options]\nValue = 123\n\n" - if string(found) != expected { - t.Fatalf("Expected file content to be \"%s\" but was \"%s\"", expected, found) + assertDiff(t, string(found), expected, "ini content") +} + +func TestOverwriteRequiredOptions(t *testing.T) { + var tests = []struct { + args []string + expected []string + }{ + { + args: []string{"--value", "from CLI"}, + expected: []string{ + "from CLI", + "from default", + }, + }, + { + args: []string{"--value", "from CLI", "--default", "from CLI"}, + expected: []string{ + "from CLI", + "from CLI", + }, + }, + { + args: []string{"--config", "no file name"}, + expected: []string{ + "from INI", + "from INI", + }, + }, + { + args: []string{"--value", "from CLI before", "--default", "from CLI before", "--config", "no file name"}, + expected: []string{ + "from INI", + "from INI", + }, + }, + { + args: []string{"--value", "from CLI before", "--default", "from CLI before", "--config", "no file name", "--value", "from CLI after", "--default", "from CLI after"}, + expected: []string{ + "from CLI after", + "from CLI after", + }, + }, + } + + for _, test := range tests { + var opts struct { + Config func(s string) error `long:"config" no-ini:"true"` + Value string `long:"value" required:"true"` + Default string `long:"default" required:"true" default:"from default"` + } + + p := NewParser(&opts, Default) + + opts.Config = func(s string) error { + ini := NewIniParser(p) + + return ini.Parse(bytes.NewBufferString("value = from INI\ndefault = from INI")) + } + + _, err := p.ParseArgs(test.args) + if err != nil { + t.Fatalf("Unexpected error %s with args %+v", err, test.args) + } + + if opts.Value != test.expected[0] { + t.Fatalf("Expected Value to be \"%s\" but was \"%s\" with args %+v", test.expected[0], opts.Value, test.args) + } + + if opts.Default != test.expected[1] { + t.Fatalf("Expected Default to be \"%s\" but was \"%s\" with args %+v", test.expected[1], opts.Default, test.args) + } } } diff --git a/vendor/src/github.com/jessevdk/go-flags/man.go b/vendor/src/github.com/jessevdk/go-flags/man.go index 9e1f0369f6c..e8e5916c061 100644 --- a/vendor/src/github.com/jessevdk/go-flags/man.go +++ b/vendor/src/github.com/jessevdk/go-flags/man.go @@ -50,7 +50,7 @@ func writeManPageOptions(wr io.Writer, grp *Group) { fmt.Fprintf(wr, ", ") } - fmt.Fprintf(wr, "--%s", opt.LongName) + fmt.Fprintf(wr, "--%s", opt.LongNameWithNamespace()) } fmt.Fprintln(wr, "\\fP") @@ -74,11 +74,11 @@ func writeManPageSubcommands(wr io.Writer, name string, root *Command) { nn = c.Name } - writeManPageCommand(wr, nn, c) + writeManPageCommand(wr, nn, root, c) } } -func writeManPageCommand(wr io.Writer, name string, command *Command) { +func writeManPageCommand(wr io.Writer, name string, root *Command, command *Command) { fmt.Fprintf(wr, ".SS %s\n", name) fmt.Fprintln(wr, command.ShortDescription) @@ -98,6 +98,24 @@ func writeManPageCommand(wr io.Writer, name string, command *Command) { } } + var usage string + if us, ok := command.data.(Usage); ok { + usage = us.Usage() + } else if command.hasCliOptions() { + usage = fmt.Sprintf("[%s-OPTIONS]", command.Name) + } + + var pre string + if root.hasCliOptions() { + pre = fmt.Sprintf("%s [OPTIONS] %s", root.Name, command.Name) + } else { + pre = fmt.Sprintf("%s %s", root.Name, command.Name) + } + + if len(usage) > 0 { + fmt.Fprintf(wr, "\n\\fBUsage\\fP: %s %s\n\n", pre, usage) + } + if len(command.Aliases) > 0 { fmt.Fprintf(wr, "\n\\fBAliases\\fP: %s\n\n", strings.Join(command.Aliases, ", ")) } diff --git a/vendor/src/github.com/jessevdk/go-flags/marshal_test.go b/vendor/src/github.com/jessevdk/go-flags/marshal_test.go index 362d0f285b3..59c9ccefb96 100644 --- a/vendor/src/github.com/jessevdk/go-flags/marshal_test.go +++ b/vendor/src/github.com/jessevdk/go-flags/marshal_test.go @@ -80,7 +80,7 @@ func TestUnmarshalError(t *testing.T) { Value marshalled `short:"v"` }{} - assertParseFail(t, ErrMarshal, "invalid argument for flag `-v' (expected flags.marshalled): `invalid' is not a valid value, please specify `yes' or `no'", &opts, "-vinvalid") + assertParseFail(t, ErrMarshal, fmt.Sprintf("invalid argument for flag `%cv' (expected flags.marshalled): `invalid' is not a valid value, please specify `yes' or `no'", defaultShortOptDelimiter), &opts, "-vinvalid") } func TestMarshalError(t *testing.T) { diff --git a/vendor/src/github.com/jessevdk/go-flags/option.go b/vendor/src/github.com/jessevdk/go-flags/option.go index f4041d694e1..29e702c1960 100644 --- a/vendor/src/github.com/jessevdk/go-flags/option.go +++ b/vendor/src/github.com/jessevdk/go-flags/option.go @@ -27,6 +27,12 @@ type Option struct { // The default value of the option. Default []string + // The optional environment default value key name. + EnvDefaultKey string + + // The optional delimiter string for EnvDefaultKey values. + EnvDefaultDelim string + // If true, specifies that the argument to an option flag is optional. // When no argument to the flag is specified on the command line, the // value of Default will be set in the field this option represents. @@ -53,15 +59,71 @@ type Option struct { // passwords. DefaultMask string + // The group which the option belongs to + group *Group + // The struct field which the option represents. field reflect.StructField // The struct field value which the option represents. value reflect.Value - defaultValue reflect.Value - iniUsedName string - tag multiTag + // Determines if the option will be always quoted in the INI output + iniQuote bool + + tag multiTag + isSet bool +} + +// LongNameWithNamespace returns the option's long name with the group namespaces +// prepended by walking up the option's group tree. Namespaces and the long name +// itself are separated by the parser's namespace delimiter. If the long name is +// empty an empty string is returned. +func (option *Option) LongNameWithNamespace() string { + if len(option.LongName) == 0 { + return "" + } + + // fetch the namespace delimiter from the parser which is always at the + // end of the group hierarchy + namespaceDelimiter := "" + g := option.group + + for { + if p, ok := g.parent.(*Parser); ok { + namespaceDelimiter = p.NamespaceDelimiter + + break + } + + switch i := g.parent.(type) { + case *Command: + g = i.Group + case *Group: + g = i + } + } + + // concatenate long name with namespace + longName := option.LongName + g = option.group + + for g != nil { + if g.Namespace != "" { + longName = g.Namespace + namespaceDelimiter + longName + } + + switch i := g.parent.(type) { + case *Command: + g = i.Group + case *Group: + g = i + case *Parser: + g = nil + } + } + + return longName } // String converts an option to a human friendly readable string describing the @@ -78,12 +140,12 @@ func (option *Option) String() string { if len(option.LongName) != 0 { s = fmt.Sprintf("%s%s, %s%s", string(defaultShortOptDelimiter), short, - defaultLongOptDelimiter, option.LongName) + defaultLongOptDelimiter, option.LongNameWithNamespace()) } else { s = fmt.Sprintf("%s%s", string(defaultShortOptDelimiter), short) } } else if len(option.LongName) != 0 { - s = fmt.Sprintf("%s%s", defaultLongOptDelimiter, option.LongName) + s = fmt.Sprintf("%s%s", defaultLongOptDelimiter, option.LongNameWithNamespace()) } return s diff --git a/vendor/src/github.com/jessevdk/go-flags/option_private.go b/vendor/src/github.com/jessevdk/go-flags/option_private.go index ea836bd083d..d36c8411786 100644 --- a/vendor/src/github.com/jessevdk/go-flags/option_private.go +++ b/vendor/src/github.com/jessevdk/go-flags/option_private.go @@ -2,12 +2,16 @@ package flags import ( "reflect" + "strings" + "syscall" ) // Set the value of an option to the specified value. An error will be returned // if the specified value could not be converted to the corresponding option // value type. func (option *Option) set(value *string) error { + option.isSet = true + if option.isFunc() { return option.call(value) } else if value != nil { @@ -29,21 +33,78 @@ func (option *Option) canArgument() bool { return !option.isBool() } -func (option *Option) clear() { +func (option *Option) emptyValue() reflect.Value { tp := option.value.Type() - switch tp.Kind() { - case reflect.Func: - // Skip - case reflect.Map: - // Empty the map - option.value.Set(reflect.MakeMap(tp)) - default: - zeroval := reflect.Zero(tp) - option.value.Set(zeroval) + if tp.Kind() == reflect.Map { + return reflect.MakeMap(tp) + } + + return reflect.Zero(tp) +} + +func (option *Option) empty() { + if !option.isFunc() { + option.value.Set(option.emptyValue()) + } +} + +func (option *Option) clearDefault() { + usedDefault := option.Default + if envKey := option.EnvDefaultKey; envKey != "" { + // os.Getenv() makes no distinction between undefined and + // empty values, so we use syscall.Getenv() + if value, ok := syscall.Getenv(envKey); ok { + if option.EnvDefaultDelim != "" { + usedDefault = strings.Split(value, + option.EnvDefaultDelim) + } else { + usedDefault = []string{value} + } + } + } + + if len(usedDefault) > 0 { + option.empty() + + for _, d := range usedDefault { + option.set(&d) + } + } else { + tp := option.value.Type() + + switch tp.Kind() { + case reflect.Map: + if option.value.IsNil() { + option.empty() + } + case reflect.Slice: + if option.value.IsNil() { + option.empty() + } + } } } +func (option *Option) valueIsDefault() bool { + // Check if the value of the option corresponds to its + // default value + emptyval := option.emptyValue() + + checkvalptr := reflect.New(emptyval.Type()) + checkval := reflect.Indirect(checkvalptr) + + checkval.Set(emptyval) + + if len(option.Default) != 0 { + for _, v := range option.Default { + convert(v, checkval, option.tag) + } + } + + return reflect.DeepEqual(option.value.Interface(), checkval.Interface()) +} + func (option *Option) isUnmarshaler() Unmarshaler { v := option.value diff --git a/vendor/src/github.com/jessevdk/go-flags/optstyle_other.go b/vendor/src/github.com/jessevdk/go-flags/optstyle_other.go index 8ee886eef68..794de23e09a 100644 --- a/vendor/src/github.com/jessevdk/go-flags/optstyle_other.go +++ b/vendor/src/github.com/jessevdk/go-flags/optstyle_other.go @@ -30,15 +30,15 @@ func stripOptionPrefix(optname string) (prefix string, name string, islong bool) // splitOption attempts to split the passed option into a name and an argument. // When there is no argument specified, nil will be returned for it. -func splitOption(prefix string, option string, islong bool) (string, *string) { +func splitOption(prefix string, option string, islong bool) (string, string, *string) { pos := strings.Index(option, "=") if (islong && pos >= 0) || (!islong && pos == 1) { rest := option[pos+1:] - return option[:pos], &rest + return option[:pos], "=", &rest } - return option, nil + return option, "", nil } // addHelpGroup adds a new group that contains default help parameters. diff --git a/vendor/src/github.com/jessevdk/go-flags/optstyle_windows.go b/vendor/src/github.com/jessevdk/go-flags/optstyle_windows.go index edd398a5936..096bbffe683 100644 --- a/vendor/src/github.com/jessevdk/go-flags/optstyle_windows.go +++ b/vendor/src/github.com/jessevdk/go-flags/optstyle_windows.go @@ -43,9 +43,9 @@ func stripOptionPrefix(optname string) (prefix string, name string, islong bool) // splitOption attempts to split the passed option into a name and an argument. // When there is no argument specified, nil will be returned for it. -func splitOption(prefix string, option string, islong bool) (string, *string) { +func splitOption(prefix string, option string, islong bool) (string, string, *string) { if len(option) == 0 { - return option, nil + return option, "", nil } // Windows typically uses a colon for the option name and argument @@ -53,19 +53,22 @@ func splitOption(prefix string, option string, islong bool) (string, *string) { // but don't allow the two to be mixed. That is to say /foo:bar and // --foo=bar are acceptable, but /foo=bar and --foo:bar are not. var pos int + var sp string if prefix == "/" { - pos = strings.Index(option, ":") + sp = ":" + pos = strings.Index(option, sp) } else if len(prefix) > 0 { - pos = strings.Index(option, "=") + sp = "=" + pos = strings.Index(option, sp) } if (islong && pos >= 0) || (!islong && pos == 1) { rest := option[pos+1:] - return option[:pos], &rest + return option[:pos], sp, &rest } - return option, nil + return option, "", nil } // addHelpGroup adds a new group that contains default help parameters. diff --git a/vendor/src/github.com/jessevdk/go-flags/parser.go b/vendor/src/github.com/jessevdk/go-flags/parser.go index 674d0e4337f..a417ba9b238 100644 --- a/vendor/src/github.com/jessevdk/go-flags/parser.go +++ b/vendor/src/github.com/jessevdk/go-flags/parser.go @@ -7,7 +7,6 @@ package flags import ( "os" "path" - "reflect" ) // A Parser provides command line option parsing. It can contain several @@ -22,6 +21,16 @@ type Parser struct { // Option flags changing the behavior of the parser. Options Options + // NamespaceDelimiter separates group namespaces and option long names + NamespaceDelimiter string + + // UnknownOptionsHandler is a function which gets called when the parser + // encounters an unknown option. The function receives the unknown option + // name and the remaining command line arguments. + // It should return a new list of remaining arguments to continue parsing, + // or an error to indicate a parse failure. + UnknownOptionHandler func(option string, args []string) ([]string, error) + internalError error } @@ -88,23 +97,34 @@ func ParseArgs(data interface{}, args []string) ([]string, error) { // group should not be added. The options parameter specifies a set of options // for the parser. func NewParser(data interface{}, options Options) *Parser { - ret := NewNamedParser(path.Base(os.Args[0]), options) + p := NewNamedParser(path.Base(os.Args[0]), options) if data != nil { - _, ret.internalError = ret.AddGroup("Application Options", "", data) + g, err := p.AddGroup("Application Options", "", data) + + if err == nil { + g.parent = p + } + + p.internalError = err } - return ret + return p } // NewNamedParser creates a new parser. The appname is used to display the // executable name in the built-in help message. Option groups and commands can // be added to this parser by using AddGroup and AddCommand. func NewNamedParser(appname string, options Options) *Parser { - return &Parser{ - Command: newCommand(appname, "", "", nil), - Options: options, + p := &Parser{ + Command: newCommand(appname, "", "", nil), + Options: options, + NamespaceDelimiter: ".", } + + p.Command.parent = p + + return p } // Parse parses the command line arguments from os.Args using Parser.ParseArgs. @@ -128,38 +148,41 @@ func (p *Parser) ParseArgs(args []string) ([]string, error) { return nil, p.internalError } - p.eachCommand(func(c *Command) { - c.eachGroup(func(g *Group) { - g.storeDefaults() - - for _, option := range g.options { - switch option.value.Type().Kind() { - case reflect.Func, reflect.Map, reflect.Slice: - option.clear() - } - } - }) - }, true) + p.clearIsSet() // Add built-in help group to all commands if necessary if (p.Options & HelpFlag) != None { p.addHelpGroups(p.showBuiltinHelp) } + compval := os.Getenv("GO_FLAGS_COMPLETION") + + if len(compval) != 0 { + comp := &completion{parser: p} + + if compval == "verbose" { + comp.ShowDescriptions = true + } + + comp.execute(args) + + return nil, nil + } + s := &parseState{ args: args, retargs: make([]string, 0, len(args)), - command: p.Command, - lookup: p.makeLookup(), } + p.fillParseState(s) + for !s.eof() { arg := s.pop() // When PassDoubleDash is set and we encounter a --, then // simply append all the rest as arguments and break out if (p.Options&PassDoubleDash) != None && arg == "--" { - s.retargs = append(s.retargs, s.args...) + s.addArgs(s.args...) break } @@ -174,32 +197,36 @@ func (p *Parser) ParseArgs(args []string) ([]string, error) { } var err error - var options []*Option prefix, optname, islong := stripOptionPrefix(arg) - optname, argument := splitOption(prefix, optname, islong) + optname, _, argument := splitOption(prefix, optname, islong) if islong { - options, err = p.parseLong(s, optname, argument) + err = p.parseLong(s, optname, argument) } else { - options, err = p.parseShort(s, optname, argument) + err = p.parseShort(s, optname, argument) } if err != nil { ignoreUnknown := (p.Options & IgnoreUnknown) != None parseErr := wrapError(err) - if !(parseErr.Type == ErrUnknownFlag && ignoreUnknown) { + if parseErr.Type != ErrUnknownFlag || (!ignoreUnknown && p.UnknownOptionHandler == nil) { s.err = parseErr break } if ignoreUnknown { - s.retargs = append(s.retargs, arg) - } - } else { - for _, option := range options { - delete(s.lookup.required, option) + s.addArgs(arg) + } else if p.UnknownOptionHandler != nil { + modifiedArgs, err := p.UnknownOptionHandler(optname, s.args) + + if err != nil { + s.err = err + break + } + + s.args = modifiedArgs } } } @@ -208,39 +235,30 @@ func (p *Parser) ParseArgs(args []string) ([]string, error) { p.eachCommand(func(c *Command) { c.eachGroup(func(g *Group) { for _, option := range g.options { - tp := option.value.Type() - - switch tp.Kind() { - case reflect.Map: - if option.value.Len() == 0 { - for _, k := range option.defaultValue.MapKeys() { - option.value.SetMapIndex(k, option.defaultValue.MapIndex(k)) - } - } - case reflect.Slice: - if reflect.DeepEqual(option.value.Interface(), reflect.Zero(tp).Interface()) { - option.value.Set(option.defaultValue) - } + if option.isSet { + continue } + + option.clearDefault() } }) }, true) - s.checkRequired() + s.checkRequired(p) } var reterr error if s.err != nil { - reterr = p.printError(s.err) + reterr = s.err } else if len(s.command.commands) != 0 && !s.command.SubcommandsOptional { - reterr = p.printError(s.estimateCommand()) + reterr = s.estimateCommand() } else if cmd, ok := s.command.data.(Commander); ok { - reterr = p.printError(cmd.Execute(s.retargs)) + reterr = cmd.Execute(s.retargs) } if reterr != nil { - return append([]string{s.arg}, s.args...), reterr + return append([]string{s.arg}, s.args...), p.printError(reterr) } return s.retargs, nil diff --git a/vendor/src/github.com/jessevdk/go-flags/parser_private.go b/vendor/src/github.com/jessevdk/go-flags/parser_private.go index 8d29b6b6bf0..6ee506fd660 100644 --- a/vendor/src/github.com/jessevdk/go-flags/parser_private.go +++ b/vendor/src/github.com/jessevdk/go-flags/parser_private.go @@ -4,15 +4,17 @@ import ( "bytes" "fmt" "os" + "sort" "strings" "unicode/utf8" ) type parseState struct { - arg string - args []string - retargs []string - err error + arg string + args []string + retargs []string + positional []*Arg + err error command *Command lookup lookup @@ -41,19 +43,63 @@ func (p *parseState) peek() string { return p.args[0] } -func (p *parseState) checkRequired() error { - required := p.lookup.required +func (p *parseState) checkRequired(parser *Parser) error { + c := parser.Command + + var required []*Option + + for c != nil { + c.eachGroup(func(g *Group) { + for _, option := range g.options { + if !option.isSet && option.Required { + required = append(required, option) + } + } + }) + + c = c.Active + } if len(required) == 0 { + if len(p.positional) > 0 && p.command.ArgsRequired { + var reqnames []string + + for _, arg := range p.positional { + if arg.isRemaining() { + break + } + + reqnames = append(reqnames, "`"+arg.Name+"`") + } + + if len(reqnames) == 0 { + return nil + } + + var msg string + + if len(reqnames) == 1 { + msg = fmt.Sprintf("the required argument %s was not provided", reqnames[0]) + } else { + msg = fmt.Sprintf("the required arguments %s and %s were not provided", + strings.Join(reqnames[:len(reqnames)-1], ", "), reqnames[len(reqnames)-1]) + } + + p.err = newError(ErrRequired, msg) + return p.err + } + return nil } names := make([]string, 0, len(required)) - for k := range required { + for _, k := range required { names = append(names, "`"+k.String()+"'") } + sort.Strings(names) + var msg string if len(names) == 1 { @@ -113,18 +159,34 @@ func (p *parseState) estimateCommand() error { func (p *Parser) parseOption(s *parseState, name string, option *Option, canarg bool, argument *string) (err error) { if !option.canArgument() { if argument != nil { - msg := fmt.Sprintf("bool flag `%s' cannot have an argument", option) - return newError(ErrNoArgumentForBool, msg) + return newErrorf(ErrNoArgumentForBool, "bool flag `%s' cannot have an argument", option) } err = option.set(nil) - } else if argument != nil { - err = option.set(argument) - } else if canarg && !s.eof() { - arg := s.pop() - err = option.set(&arg) + } else if argument != nil || (canarg && !s.eof()) { + var arg string + + if argument != nil { + arg = *argument + } else { + arg = s.pop() + + // Accept any single character arguments including '-'. + // '-' is the special file name for the standard input or the standard output in many cases. + if len(arg) > 1 && argumentIsOption(arg) { + return newErrorf(ErrExpectedArgument, "expected argument for flag `%s', but got option `%s'", option, arg) + } + } + + if option.tag.Get("unquote") != "false" { + arg, err = unquoteIfPossible(arg) + } + + if err == nil { + err = option.set(&arg) + } } else if option.OptionalArgument { - option.clear() + option.empty() for _, v := range option.OptionalValue { err = option.set(&v) @@ -134,38 +196,31 @@ func (p *Parser) parseOption(s *parseState, name string, option *Option, canarg } } } else { - msg := fmt.Sprintf("expected argument for flag `%s'", option) - err = newError(ErrExpectedArgument, msg) + err = newErrorf(ErrExpectedArgument, "expected argument for flag `%s'", option) } if err != nil { if _, ok := err.(*Error); !ok { - msg := fmt.Sprintf("invalid argument for flag `%s' (expected %s): %s", + err = newErrorf(ErrMarshal, "invalid argument for flag `%s' (expected %s): %s", option, option.value.Type(), err.Error()) - - err = newError(ErrMarshal, msg) } } return err } -func (p *Parser) parseLong(s *parseState, name string, argument *string) (options []*Option, err error) { +func (p *Parser) parseLong(s *parseState, name string, argument *string) error { if option := s.lookup.longNames[name]; option != nil { - options = append(options, option) - // Only long options that are required can consume an argument // from the argument list canarg := !option.OptionalArgument - err := p.parseOption(s, name, option, canarg, argument) - - return options, err + return p.parseOption(s, name, option, canarg, argument) } - return nil, newError(ErrUnknownFlag, fmt.Sprintf("unknown flag `%s'", name)) + return newErrorf(ErrUnknownFlag, "unknown flag `%s'", name) } func (p *Parser) splitShortConcatArg(s *parseState, optname string) (string, *string) { @@ -185,7 +240,7 @@ func (p *Parser) splitShortConcatArg(s *parseState, optname string) (string, *st return optname, nil } -func (p *Parser) parseShort(s *parseState, optname string, argument *string) (options []*Option, err error) { +func (p *Parser) parseShort(s *parseState, optname string, argument *string) error { if argument == nil { optname, argument = p.splitShortConcatArg(s, optname) } @@ -194,17 +249,15 @@ func (p *Parser) parseShort(s *parseState, optname string, argument *string) (op shortname := string(c) if option := s.lookup.shortNames[shortname]; option != nil { - options = append(options, option) - // Only the last short argument can consume an argument from // the arguments list, and only if it's non optional canarg := (i+utf8.RuneLen(c) == len(optname)) && !option.OptionalArgument if err := p.parseOption(s, shortname, option, canarg, argument); err != nil { - return options, err + return err } } else { - return nil, newError(ErrUnknownFlag, fmt.Sprintf("unknown flag `%s'", shortname)) + return newErrorf(ErrUnknownFlag, "unknown flag `%s'", shortname) } // Only the first option can have a concatted argument, so just @@ -212,26 +265,50 @@ func (p *Parser) parseShort(s *parseState, optname string, argument *string) (op argument = nil } - return options, nil + return nil } -func (p *Parser) parseNonOption(s *parseState) error { - if cmd := s.lookup.commands[s.arg]; cmd != nil { - if err := s.checkRequired(); err != nil { +func (p *parseState) addArgs(args ...string) error { + for len(p.positional) > 0 && len(args) > 0 { + arg := p.positional[0] + + if err := convert(args[0], arg.value, arg.tag); err != nil { return err } - s.command.Active = cmd + if !arg.isRemaining() { + p.positional = p.positional[1:] + } + + args = args[1:] + } + + p.retargs = append(p.retargs, args...) + return nil +} - s.command = cmd - s.lookup = cmd.makeLookup() +func (p *Parser) parseNonOption(s *parseState) error { + if len(s.positional) > 0 { + return s.addArgs(s.arg) + } + + if cmd := s.lookup.commands[s.arg]; cmd != nil { + s.command.Active = cmd + cmd.fillParseState(s) } else if (p.Options & PassAfterNonOption) != None { // If PassAfterNonOption is set then all remaining arguments // are considered positional - s.retargs = append(append(s.retargs, s.arg), s.args...) + if err := s.addArgs(s.arg); err != nil { + return err + } + + if err := s.addArgs(s.args...); err != nil { + return err + } + s.args = []string{} } else { - s.retargs = append(s.retargs, s.arg) + return s.addArgs(s.arg) } return nil @@ -251,3 +328,13 @@ func (p *Parser) printError(err error) error { return err } + +func (p *Parser) clearIsSet() { + p.eachCommand(func(c *Command) { + c.eachGroup(func(g *Group) { + for _, option := range g.options { + option.isSet = false + } + }) + }, true) +} diff --git a/vendor/src/github.com/jessevdk/go-flags/parser_test.go b/vendor/src/github.com/jessevdk/go-flags/parser_test.go index b227244d78c..bba7576e717 100644 --- a/vendor/src/github.com/jessevdk/go-flags/parser_test.go +++ b/vendor/src/github.com/jessevdk/go-flags/parser_test.go @@ -1,7 +1,11 @@ package flags import ( + "fmt" + "os" "reflect" + "strconv" + "strings" "testing" "time" ) @@ -10,6 +14,10 @@ type defaultOptions struct { Int int `long:"i"` IntDefault int `long:"id" default:"1"` + String string `long:"str"` + StringDefault string `long:"strd" default:"abc"` + StringNotUnquoted string `long:"strnot" unquote:"false"` + Time time.Duration `long:"t"` TimeDefault time.Duration `long:"td" default:"1m"` @@ -33,6 +41,9 @@ func TestDefaults(t *testing.T) { Int: 0, IntDefault: 1, + String: "", + StringDefault: "abc", + Time: 0, TimeDefault: time.Minute, @@ -45,11 +56,14 @@ func TestDefaults(t *testing.T) { }, { msg: "non-zero value arguments, expecting overwritten arguments", - args: []string{"--i=3", "--id=3", "--t=3ms", "--td=3ms", "--m=c:3", "--md=c:3", "--s=3", "--sd=3"}, + args: []string{"--i=3", "--id=3", "--str=def", "--strd=def", "--t=3ms", "--td=3ms", "--m=c:3", "--md=c:3", "--s=3", "--sd=3"}, expected: defaultOptions{ Int: 3, IntDefault: 3, + String: "def", + StringDefault: "def", + Time: 3 * time.Millisecond, TimeDefault: 3 * time.Millisecond, @@ -62,11 +76,14 @@ func TestDefaults(t *testing.T) { }, { msg: "zero value arguments, expecting overwritten arguments", - args: []string{"--i=0", "--id=0", "--t=0ms", "--td=0s", "--m=:0", "--md=:0", "--s=0", "--sd=0"}, + args: []string{"--i=0", "--id=0", "--str", "", "--strd=\"\"", "--t=0ms", "--td=0s", "--m=:0", "--md=:0", "--s=0", "--sd=0"}, expected: defaultOptions{ Int: 0, IntDefault: 0, + String: "", + StringDefault: "", + Time: 0, TimeDefault: 0, @@ -96,3 +113,298 @@ func TestDefaults(t *testing.T) { } } } + +func TestUnquoting(t *testing.T) { + var tests = []struct { + arg string + err error + value string + }{ + { + arg: "\"abc", + err: strconv.ErrSyntax, + value: "", + }, + { + arg: "\"\"abc\"", + err: strconv.ErrSyntax, + value: "", + }, + { + arg: "\"abc\"", + err: nil, + value: "abc", + }, + { + arg: "\"\\\"abc\\\"\"", + err: nil, + value: "\"abc\"", + }, + { + arg: "\"\\\"abc\"", + err: nil, + value: "\"abc", + }, + } + + for _, test := range tests { + var opts defaultOptions + + for _, delimiter := range []bool{false, true} { + p := NewParser(&opts, None) + + var err error + if delimiter { + _, err = p.ParseArgs([]string{"--str=" + test.arg, "--strnot=" + test.arg}) + } else { + _, err = p.ParseArgs([]string{"--str", test.arg, "--strnot", test.arg}) + } + + if test.err == nil { + if err != nil { + t.Fatalf("Expected no error but got: %v", err) + } + + if test.value != opts.String { + t.Fatalf("Expected String to be %q but got %q", test.value, opts.String) + } + if q := strconv.Quote(test.value); q != opts.StringNotUnquoted { + t.Fatalf("Expected StringDefault to be %q but got %q", q, opts.StringNotUnquoted) + } + } else { + if err == nil { + t.Fatalf("Expected error") + } else if e, ok := err.(*Error); ok { + if strings.HasPrefix(e.Message, test.err.Error()) { + t.Fatalf("Expected error message to end with %q but got %v", test.err.Error(), e.Message) + } + } + } + } + } +} + +// envRestorer keeps a copy of a set of env variables and can restore the env from them +type envRestorer struct { + env map[string]string +} + +func (r *envRestorer) Restore() { + os.Clearenv() + for k, v := range r.env { + os.Setenv(k, v) + } +} + +// EnvSnapshot returns a snapshot of the currently set env variables +func EnvSnapshot() *envRestorer { + r := envRestorer{make(map[string]string)} + for _, kv := range os.Environ() { + parts := strings.SplitN(kv, "=", 2) + if len(parts) != 2 { + panic("got a weird env variable: " + kv) + } + r.env[parts[0]] = parts[1] + } + return &r +} + +type envDefaultOptions struct { + Int int `long:"i" default:"1" env:"TEST_I"` + Time time.Duration `long:"t" default:"1m" env:"TEST_T"` + Map map[string]int `long:"m" default:"a:1" env:"TEST_M" env-delim:";"` + Slice []int `long:"s" default:"1" default:"2" env:"TEST_S" env-delim:","` +} + +func TestEnvDefaults(t *testing.T) { + var tests = []struct { + msg string + args []string + expected envDefaultOptions + env map[string]string + }{ + { + msg: "no arguments, no env, expecting default values", + args: []string{}, + expected: envDefaultOptions{ + Int: 1, + Time: time.Minute, + Map: map[string]int{"a": 1}, + Slice: []int{1, 2}, + }, + }, + { + msg: "no arguments, env defaults, expecting env default values", + args: []string{}, + expected: envDefaultOptions{ + Int: 2, + Time: 2 * time.Minute, + Map: map[string]int{"a": 2, "b": 3}, + Slice: []int{4, 5, 6}, + }, + env: map[string]string{ + "TEST_I": "2", + "TEST_T": "2m", + "TEST_M": "a:2;b:3", + "TEST_S": "4,5,6", + }, + }, + { + msg: "non-zero value arguments, expecting overwritten arguments", + args: []string{"--i=3", "--t=3ms", "--m=c:3", "--s=3"}, + expected: envDefaultOptions{ + Int: 3, + Time: 3 * time.Millisecond, + Map: map[string]int{"c": 3}, + Slice: []int{3}, + }, + env: map[string]string{ + "TEST_I": "2", + "TEST_T": "2m", + "TEST_M": "a:2;b:3", + "TEST_S": "4,5,6", + }, + }, + { + msg: "zero value arguments, expecting overwritten arguments", + args: []string{"--i=0", "--t=0ms", "--m=:0", "--s=0"}, + expected: envDefaultOptions{ + Int: 0, + Time: 0, + Map: map[string]int{"": 0}, + Slice: []int{0}, + }, + env: map[string]string{ + "TEST_I": "2", + "TEST_T": "2m", + "TEST_M": "a:2;b:3", + "TEST_S": "4,5,6", + }, + }, + } + + oldEnv := EnvSnapshot() + defer oldEnv.Restore() + + for _, test := range tests { + var opts envDefaultOptions + oldEnv.Restore() + for envKey, envValue := range test.env { + os.Setenv(envKey, envValue) + } + _, err := ParseArgs(&opts, test.args) + if err != nil { + t.Fatalf("%s:\nUnexpected error: %v", test.msg, err) + } + + if opts.Slice == nil { + opts.Slice = []int{} + } + + if !reflect.DeepEqual(opts, test.expected) { + t.Errorf("%s:\nUnexpected options with arguments %+v\nexpected\n%+v\nbut got\n%+v\n", test.msg, test.args, test.expected, opts) + } + } +} + +func TestOptionAsArgument(t *testing.T) { + var tests = []struct { + args []string + expectError bool + errType ErrorType + errMsg string + }{ + { + // short option must not be accepted as argument + args: []string{"--string-slice", "foobar", "--string-slice", "-o"}, + expectError: true, + errType: ErrExpectedArgument, + errMsg: "expected argument for flag `--string-slice', but got option `-o'", + }, + { + // long option must not be accepted as argument + args: []string{"--string-slice", "foobar", "--string-slice", "--other-option"}, + expectError: true, + errType: ErrExpectedArgument, + errMsg: "expected argument for flag `--string-slice', but got option `--other-option'", + }, + { + // long option must not be accepted as argument + args: []string{"--string-slice", "--"}, + expectError: true, + errType: ErrExpectedArgument, + errMsg: "expected argument for flag `--string-slice', but got option `--'", + }, + { + // quoted and appended option should be accepted as argument (even if it looks like an option) + args: []string{"--string-slice", "foobar", "--string-slice=\"--other-option\""}, + }, + { + // Accept any single character arguments including '-' + args: []string{"--string-slice", "-"}, + }, + } + var opts struct { + StringSlice []string `long:"string-slice"` + OtherOption bool `long:"other-option" short:"o"` + } + + for _, test := range tests { + if test.expectError { + assertParseFail(t, test.errType, test.errMsg, &opts, test.args...) + } else { + assertParseSuccess(t, &opts, test.args...) + } + } +} + +func TestUnknownFlagHandler(t *testing.T) { + + var opts struct { + Flag1 string `long:"flag1"` + Flag2 string `long:"flag2"` + } + + p := NewParser(&opts, None) + + var unknownFlag1 string + var unknownFlag2 bool + + // Set up a callback to intercept unknown options during parsing + p.UnknownOptionHandler = func(option string, args []string) ([]string, error) { + if option == "unknownFlag1" { + // consume a value from remaining args list + unknownFlag1 = args[0] + return args[1:], nil + } else if option == "unknownFlag2" { + // treat this one as a bool switch, don't consume any args + unknownFlag2 = true + return args, nil + } + + return args, fmt.Errorf("Unknown flag: %v", option) + } + + // Parse args containing some unknown flags, verify that + // our callback can handle all of them + _, err := p.ParseArgs([]string{"--flag1=stuff", "--unknownFlag1", "blah", "--unknownFlag2", "--flag2=foo"}) + + if err != nil { + assertErrorf(t, "Parser returned unexpected error %v", err) + } + + assertString(t, opts.Flag1, "stuff") + assertString(t, opts.Flag2, "foo") + assertString(t, unknownFlag1, "blah") + + if !unknownFlag2 { + assertErrorf(t, "Flag should have been set by unknown handler, but had value: %v", unknownFlag2) + } + + // Parse args with unknown flags that callback doesn't handle, verify it returns error + _, err = p.ParseArgs([]string{"--flag1=stuff", "--unknownFlagX", "blah", "--flag2=foo"}) + + if err == nil { + assertErrorf(t, "Parser should have returned error, but returned nil") + } +} diff --git a/vendor/src/github.com/jessevdk/go-flags/short_test.go b/vendor/src/github.com/jessevdk/go-flags/short_test.go index d08702b9ae2..95712c16238 100644 --- a/vendor/src/github.com/jessevdk/go-flags/short_test.go +++ b/vendor/src/github.com/jessevdk/go-flags/short_test.go @@ -1,6 +1,7 @@ package flags import ( + "fmt" "testing" ) @@ -31,7 +32,7 @@ func TestShortRequired(t *testing.T) { Value bool `short:"v" required:"true"` }{} - assertParseFail(t, ErrRequired, "the required flag `-v' was not specified", &opts) + assertParseFail(t, ErrRequired, fmt.Sprintf("the required flag `%cv' was not specified", defaultShortOptDelimiter), &opts) } func TestShortMultiConcat(t *testing.T) { @@ -143,7 +144,7 @@ func TestShortMultiWithEqualArg(t *testing.T) { Value string `short:"v"` }{} - assertParseFail(t, ErrExpectedArgument, "expected argument for flag `-v'", &opts, "-ffv=value") + assertParseFail(t, ErrExpectedArgument, fmt.Sprintf("expected argument for flag `%cv'", defaultShortOptDelimiter), &opts, "-ffv=value") } func TestShortMultiArg(t *testing.T) { @@ -165,7 +166,7 @@ func TestShortMultiArgConcatFail(t *testing.T) { Value string `short:"v"` }{} - assertParseFail(t, ErrExpectedArgument, "expected argument for flag `-v'", &opts, "-ffvvalue") + assertParseFail(t, ErrExpectedArgument, fmt.Sprintf("expected argument for flag `%cv'", defaultShortOptDelimiter), &opts, "-ffvvalue") } func TestShortMultiArgConcat(t *testing.T) { diff --git a/vendor/src/github.com/spacemonkeygo/spacelog/setup.go b/vendor/src/github.com/spacemonkeygo/spacelog/setup.go index bd46b0ab27e..26ad00572c9 100644 --- a/vendor/src/github.com/spacemonkeygo/spacelog/setup.go +++ b/vendor/src/github.com/spacemonkeygo/spacelog/setup.go @@ -137,12 +137,12 @@ func Setup(procname string, config SetupConfig) error { textout = w case "stdout": if t == nil { - t = ColorTemplate + t = DefaultTemplate } textout = NewWriterOutput(os.Stdout) case "stderr", "": if t == nil { - t = ColorTemplate + t = DefaultTemplate } textout = NewWriterOutput(os.Stderr) default: diff --git a/vendor/src/github.com/spacemonkeygo/spacelog/templates_others.go b/vendor/src/github.com/spacemonkeygo/spacelog/templates_others.go new file mode 100644 index 00000000000..114e2e14312 --- /dev/null +++ b/vendor/src/github.com/spacemonkeygo/spacelog/templates_others.go @@ -0,0 +1,22 @@ +// Copyright (C) 2014 Space Monkey, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !windows + +package spacelog + +var ( + // DefaultTemplate is default template for stdout/stderr for the platform + DefaultTemplate = ColorTemplate +) diff --git a/vendor/src/github.com/spacemonkeygo/spacelog/templates_windows.go b/vendor/src/github.com/spacemonkeygo/spacelog/templates_windows.go new file mode 100644 index 00000000000..512b600481e --- /dev/null +++ b/vendor/src/github.com/spacemonkeygo/spacelog/templates_windows.go @@ -0,0 +1,20 @@ +// Copyright (C) 2014 Space Monkey, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package spacelog + +var ( + // DefaultTemplate is default template for stdout/stderr for the platform + DefaultTemplate = StandardTemplate +) |