Import tools: c3d21876c39e136a5573356ddde1bc5ec5069ffb from branch v4.2

ref: 3e3dc4003e..c3d21876c3
for: 4.2.1

TOOLS-2412    Strip unsupported legacy index options

4 files changed, 175 insertions, 4 deletions

diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/import.data b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/import.data
index 69f39f5ff34..a7bb26c771b 100644
--- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/import.data
+++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/import.data
@@ -1,6 +1,6 @@
 {
-    "vendor": "tools",
-    "github": "mongodb/mongo-tools.git",
-    "branch": "v4.2",
-    "commit": "3e3dc4003e34efa2ba3c4d09ba94817d429d3d03"
+    "commit": "c3d21876c39e136a5573356ddde1bc5ec5069ffb", 
+    "github": "mongodb/mongo-tools.git", 
+    "vendor": "tools", 
+    "branch": "v4.2"
 }
diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongorestore/mongorestore_test.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongorestore/mongorestore_test.go
index 53763deba2c..db3cb950893 100644
--- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongorestore/mongorestore_test.go
+++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongorestore/mongorestore_test.go
@@ -347,3 +347,66 @@ func TestMongorestoreMIOSOE(t *testing.T) {
 
 	_ = database.Drop(nil)
 }
+
+func TestDeprecatedIndexOptions(t *testing.T) {
+	testtype.SkipUnlessTestType(t, testtype.IntegrationTestType)
+	session, err := testutil.GetBareSession()
+	if err != nil {
+		t.Fatalf("No server available")
+	}
+
+	Convey("With a test MongoRestore", t, func() {
+		args := []string{
+			NumParallelCollectionsOption, "1",
+			NumInsertionWorkersOption, "1",
+		}
+
+		restore, err := getRestoreWithArgs(args...)
+		So(err, ShouldBeNil)
+
+		session, _ = restore.SessionProvider.GetSession()
+
+		db := session.Database("indextest")
+
+		coll := db.Collection("test_collection")
+		coll.Drop(nil)
+		defer func() {
+			coll.Drop(nil)
+		}()
+		Convey("Creating index with invalid option should throw error", func() {
+			restore.TargetDirectory = "testdata/indextestdump"
+			result := restore.Restore()
+			So(result.Err, ShouldNotBeNil)
+			So(result.Err.Error(), ShouldStartWith, `indextest.test_collection: error creating indexes for indextest.test_collection: createIndex error: (InvalidIndexSpecificationOption)`)
+
+			So(result.Successes, ShouldEqual, 100)
+			So(result.Failures, ShouldEqual, 0)
+			count, err := coll.CountDocuments(nil, bson.M{})
+			So(err, ShouldBeNil)
+			So(count, ShouldEqual, 100)
+		})
+
+		coll.Drop(nil)
+
+		args = []string{
+			NumParallelCollectionsOption, "1",
+			NumInsertionWorkersOption, "1",
+			ConvertLegacyIndexesOption, "true",
+		}
+
+		restore, err = getRestoreWithArgs(args...)
+		So(err, ShouldBeNil)
+
+		Convey("Creating index with invalid option and --ignoreInvalidIndexOptions should succeed", func() {
+			restore.TargetDirectory = "testdata/indextestdump"
+			result := restore.Restore()
+			So(result.Err, ShouldBeNil)
+
+			So(result.Successes, ShouldEqual, 100)
+			So(result.Failures, ShouldEqual, 0)
+			count, err := coll.CountDocuments(nil, bson.M{})
+			So(err, ShouldBeNil)
+			So(count, ShouldEqual, 100)
+		})
+	})
+}
diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongorestore/options.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongorestore/options.go
index cd615c0004a..ead2766c2ae 100644
--- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongorestore/options.go
+++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongorestore/options.go
@@ -69,6 +69,7 @@ const (
 	DryRunOption                   = "--dryRun"
 	WriteConcernOption             = "--writeConcern"
 	NoIndexRestoreOption           = "--noIndexRestore"
+	ConvertLegacyIndexesOption     = "--convertLegacyIndexes"
 	NoOptionsRestoreOption         = "--noOptionsRestore"
 	KeepIndexVersionOption         = "--keepIndexVersion"
 	MaintainInsertionOrderOption   = "--maintainInsertionOrder"
@@ -90,6 +91,7 @@ type OutputOptions struct {
 	// By default mongorestore uses a write concern of 'majority'.
 	WriteConcern             string `long:"writeConcern" value-name:"<write-concern>" default-mask:"-" description:"write concern options e.g. --writeConcern majority, --writeConcern '{w: 3, wtimeout: 500, fsync: true, j: true}'"`
 	NoIndexRestore           bool   `long:"noIndexRestore" description:"don't restore indexes"`
+	ConvertLegacyIndexes     bool   `long:"convertLegacyIndexes" description:"Removes invalid index options and rewrites legacy option values (e.g. true becomes 1)."`
 	NoOptionsRestore         bool   `long:"noOptionsRestore" description:"don't restore collection options"`
 	KeepIndexVersion         bool   `long:"keepIndexVersion" description:"don't update index version"`
 	MaintainInsertionOrder   bool   `long:"maintainInsertionOrder" description:"restore the documents in the order of their appearance in the input source. By default the insertions will be performed in an arbitrary order. Setting this flag also enables the behavior of --stopOnError and restricts NumInsertionWorkersPerCollection to 1."`
diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongorestore/restore.go b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongorestore/restore.go
index 1ddc7b639f0..44da4b5ddb3 100644
--- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongorestore/restore.go
+++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/mongorestore/restore.go
@@ -20,11 +20,39 @@ import (
 	"github.com/mongodb/mongo-tools-common/util"
 
 	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo"
 )
 
 const insertBufferFactor = 16
 
+// validIndexOptions are taken from https://github.com/mongodb/mongo/blob/master/src/mongo/db/index/index_descriptor.h
+var validIndexOptions = map[string]bool{
+	"2dsphereIndexVersion":    true,
+	"background":              true,
+	"bits":                    true,
+	"bucketSize":              true,
+	"coarsestIndexedLevel":    true,
+	"collation":               true,
+	"default_language":        true,
+	"expireAfterSeconds":      true,
+	"finestIndexedLevel":      true,
+	"key":                     true,
+	"language_override":       true,
+	"max":                     true,
+	"min":                     true,
+	"name":                    true,
+	"ns":                      true,
+	"partialFilterExpression": true,
+	"sparse":                  true,
+	"storageEngine":           true,
+	"textIndexVersion":        true,
+	"unique":                  true,
+	"v":                       true,
+	"weights":                 true,
+	"wildcardProjection":      true,
+}
+
 // Result encapsulates the outcome of a particular restore attempt.
 type Result struct {
 	Successes int64
@@ -289,6 +317,9 @@ func (restore *MongoRestore) RestoreIntent(intent *intents.Intent) Result {
 	// finally, add indexes
 	if len(indexes) > 0 && !restore.OutputOptions.NoIndexRestore {
 		log.Logvf(log.Always, "restoring indexes for collection %v from metadata", intent.Namespace())
+		if restore.OutputOptions.ConvertLegacyIndexes {
+			convertLegacyIndexes(indexes)
+		}
 		err = restore.CreateIndexes(intent, indexes, hasNonSimpleCollation)
 		if err != nil {
 			result.Err = fmt.Errorf("error creating indexes for %v: %v", intent.Namespace(), err)
@@ -301,6 +332,81 @@ func (restore *MongoRestore) RestoreIntent(intent *intents.Intent) Result {
 	return result
 }
 
+func convertLegacyIndexes(indexes []IndexDocument) {
+	for _, index := range indexes {
+		convertLegacyIndexKeys(index)
+		convertLegacyIndexOptions(index)
+	}
+}
+
+func convertLegacyIndexKeys(index IndexDocument) {
+	var converted bool
+	originalJSONString := createExtJSONString(index.Key)
+	for j, elem := range index.Key {
+		switch v := elem.Value.(type) {
+		case int32, int64, float64:
+			// Only convert 0 value
+			if v == 0 {
+				index.Key[j].Value = 1
+				converted = true
+			}
+		case primitive.Decimal128:
+			// Note, this doesn't catch Decimal values which are equivalent to "0" (e.g. 0.00 or -0).
+			// These values are so unlikely we just ignore them
+			zeroVal, err := primitive.ParseDecimal128("0")
+			if err == nil {
+				if v == zeroVal {
+					index.Key[j].Value = 1
+					converted = true
+				}
+			}
+		case string:
+			// Only convert an empty string
+			if v == "" {
+				index.Key[j].Value = 1
+				converted = true
+			}
+		default:
+			// Convert all types that aren't strings or numbers
+			index.Key[j].Value = 1
+			converted = true
+		}
+	}
+	if converted {
+		newJSONString := createExtJSONString(index.Key)
+		log.Logvf(log.Always, "convertLegacyIndexes: converted index values '%s' to '%s' on collection '%s'",
+			originalJSONString, newJSONString, index.Options["ns"])
+	}
+}
+
+func convertLegacyIndexOptions(index IndexDocument) {
+	var converted bool
+	originalJSONString := createExtJSONString(index.Options)
+	for key := range index.Options {
+		if _, ok := validIndexOptions[key]; !ok {
+			delete(index.Options, key)
+			converted = true
+		}
+	}
+	if converted {
+		newJSONString := createExtJSONString(index.Options)
+		log.Logvf(log.Always, "convertLegacyIndexes: converted index options '%s' to '%s'",
+			originalJSONString, newJSONString)
+	}
+}
+
+func createExtJSONString(doc interface{}) string {
+	// by default return "<unable to format document>"" since we don't
+	// want to throw an error when formatting informational messages.
+	// An error would be inconsequential.
+	JSONString := "<unable to format document>"
+	JSONBytes, err := bson.MarshalExtJSON(doc, false, false)
+	if err == nil {
+		JSONString = string(JSONBytes)
+	}
+	return JSONString
+}
+
 // RestoreCollectionToDB pipes the given BSON data into the database.
 // Returns the number of documents restored and any errors that occurred.
 func (restore *MongoRestore) RestoreCollectionToDB(dbName, colName string,