TOOLS-1016 look for compressed metadata for compressed bson files

13 files changed, 107 insertions, 26 deletions

diff --git a/common.yml b/common.yml
index 7577f378284..c4e8f2cd720 100644
--- a/common.yml
+++ b/common.yml
@@ -70,6 +70,7 @@ mongo_tools_variables:
       - name: legacy24
       - name: qa-tests
       - name: qa-dump-restore-archiving
+      - name: qa-dump-restore-gzip
       - name: unit
     osx_108_ssl_task_list: &osx_108_ssl_tasks
       - name: dist
@@ -99,6 +100,7 @@ mongo_tools_variables:
       - name: qa-tests
       - name: qa-tests-unstable
       - name: qa-dump-restore-archiving
+      - name: qa-dump-restore-gzip
       - name: qa-tests-wt
       - name: text
       - name: unit
@@ -181,6 +183,9 @@ mongo_tools_variables:
       - name: qa-dump-restore-archiving
         distros:
         - windows-64-vs2013-test
+      - name: qa-dump-restore-gzip
+        distros:
+        - windows-64-vs2013-test
       - name: unit
     windows_64_ssl_task_list: &windows_64_ssl_tasks
       - name: dist
@@ -1153,6 +1158,43 @@ tasks:
           chmod 400 jstests/libs/key*
           SMOKE_EVAL="load('jstests/configs/archive_targets.js')" PATH=$PATH:/data/multiversion python buildscripts/smoke.py ${smoke_use_ssl} --nopreallocj --with-cleanbb --mongod ./mongod --mongo ./mongo --report-file report.json --continue-on-failure --buildlogger-builder MCI_${build_variant} --buildlogger-buildnum ${builder_num|} --buildlogger-credentials ./mci.buildlogger --buildlogger-phase ${task_name}_${execution} --authMechanism SCRAM-SHA-1 dump restore
 
+- name: qa-dump-restore-gzip
+  depends_on:
+  - name: dist
+  commands:
+    - command: git.get_project
+      params:
+        directory: src
+    - command: git.apply_patch
+      params:
+        directory: src
+    - func: "get buildnumber"
+    - func: "setup credentials"
+    - func: "download mongod"
+      vars:
+        mongo_url: "${mongo30_url}"
+    - func: "fetch tool"
+      vars:
+        tool: mongodump
+    - func: "fetch tool"
+      vars:
+        tool: mongorestore
+    - command: shell.exec
+      type: test
+      params:
+        working_dir: src
+        script: |
+          set -e
+          mv ./mongodb/mongod${extension} .
+          mv ./mongodb/mongo${extension} .
+          mv ./mongodb/mongos${extension} .
+          mv test/qa-tests/* .
+          chmod +x mongo*
+          rm -rf /data/install /data/multiversion
+          python buildscripts/setup_multiversion_mongodb.py /data/install /data/multiversion ${arch} "2.6" "2.4" --latest ${smoke_use_ssl} --os="${mongo_os}" || true
+          chmod 400 jstests/libs/key*
+          SMOKE_EVAL="load('jstests/configs/gzip_targets.js')" PATH=$PATH:/data/multiversion python buildscripts/smoke.py ${smoke_use_ssl} --nopreallocj --with-cleanbb --mongod ./mongod --mongo ./mongo --report-file report.json --continue-on-failure --buildlogger-builder MCI_${build_variant} --buildlogger-buildnum ${builder_num|} --buildlogger-credentials ./mci.buildlogger --buildlogger-phase ${task_name}_${execution} --authMechanism SCRAM-SHA-1 dump restore
+
 - name: qa-tests-wt
   depends_on:
   - name: dist
diff --git a/mongodump/prepare.go b/mongodump/prepare.go
index a1e77845e39..62e7dd7aa50 100644
--- a/mongodump/prepare.go
+++ b/mongodump/prepare.go
@@ -246,10 +246,7 @@ func (dump *MongoDump) NewIntent(dbName, colName string) (*intents.Intent, error
 				return nil, fmt.Errorf(`"%v.%v" contains a path separator '%c' `+
 					`and can't be dumped to the filesystem`, dbName, colName, c)
 			}
-			path := dump.outputPath(dbName, colName) + ".bson"
-			if dump.OutputOptions.Gzip {
-				path += ".gz"
-			}
+			path := nameGz(dump.OutputOptions.Gzip, dump.outputPath(dbName, colName)+".bson")
 			intent.BSONFile = &realBSONFile{path: path, intent: intent, gzip: dump.OutputOptions.Gzip}
 		}
 		if !intent.IsSystemIndexes() {
@@ -259,10 +256,7 @@ func (dump *MongoDump) NewIntent(dbName, colName string) (*intents.Intent, error
 					Buffer: &bytes.Buffer{},
 				}
 			} else {
-				path := dump.outputPath(dbName, colName+".metadata.json")
-				if dump.OutputOptions.Gzip {
-					path += ".gz"
-				}
+				path := nameGz(dump.OutputOptions.Gzip, dump.outputPath(dbName, colName+".metadata.json"))
 				intent.MetadataFile = &realMetadataFile{path: path, intent: intent, gzip: dump.OutputOptions.Gzip}
 			}
 		}
@@ -327,9 +321,9 @@ func (dump *MongoDump) CreateUsersRolesVersionIntentsForDB(db string) error {
 		rolesIntent.BSONFile = &archive.MuxIn{Intent: rolesIntent, Mux: dump.archive.Mux}
 		versionIntent.BSONFile = &archive.MuxIn{Intent: versionIntent, Mux: dump.archive.Mux}
 	} else {
-		usersIntent.BSONFile = &realBSONFile{path: filepath.Join(outDir, "$admin.system.users.bson"), intent: usersIntent, gzip: dump.OutputOptions.Gzip}
-		rolesIntent.BSONFile = &realBSONFile{path: filepath.Join(outDir, "$admin.system.roles.bson"), intent: rolesIntent, gzip: dump.OutputOptions.Gzip}
-		versionIntent.BSONFile = &realBSONFile{path: filepath.Join(outDir, "$admin.system.version.bson"), intent: versionIntent, gzip: dump.OutputOptions.Gzip}
+		usersIntent.BSONFile = &realBSONFile{path: filepath.Join(outDir, nameGz(dump.OutputOptions.Gzip, "$admin.system.users.bson")), intent: usersIntent, gzip: dump.OutputOptions.Gzip}
+		rolesIntent.BSONFile = &realBSONFile{path: filepath.Join(outDir, nameGz(dump.OutputOptions.Gzip, "$admin.system.roles.bson")), intent: rolesIntent, gzip: dump.OutputOptions.Gzip}
+		versionIntent.BSONFile = &realBSONFile{path: filepath.Join(outDir, nameGz(dump.OutputOptions.Gzip, "$admin.system.version.bson")), intent: versionIntent, gzip: dump.OutputOptions.Gzip}
 	}
 	dump.manager.Put(usersIntent)
 	dump.manager.Put(rolesIntent)
@@ -454,3 +448,10 @@ func (dump *MongoDump) CreateAllIntents() error {
 	}
 	return nil
 }
+
+func nameGz(gz bool, name string) string {
+	if gz {
+		return name + ".gz"
+	}
+	return name
+}
diff --git a/mongorestore/filepath.go b/mongorestore/filepath.go
index 7e2372d612f..3a34a7deb31 100644
--- a/mongorestore/filepath.go
+++ b/mongorestore/filepath.go
@@ -483,6 +483,9 @@ func (restore *MongoRestore) CreateIntentForCollection(db string, collection str
 		return nil
 	}
 	metadataName := baseName + ".metadata.json"
+	if restore.InputOptions.Gzip {
+		metadataName += ".gz"
+	}
 	for _, entry := range entries {
 		if entry.Name() == metadataName {
 			metadataPath := entry.Path()
diff --git a/test/qa-tests/buildscripts/smoke.py b/test/qa-tests/buildscripts/smoke.py
index cce9a876736..bbeec4b12b4 100755
--- a/test/qa-tests/buildscripts/smoke.py
+++ b/test/qa-tests/buildscripts/smoke.py
@@ -607,6 +607,9 @@ def runTest(test, result):
         if auth and usedb:
             evalString += 'jsTest.authenticate(db.getMongo());'
 
+        if os.getenv('SMOKE_EVAL') is not None:
+            evalString += os.getenv('SMOKE_EVAL')
+
         argv = argv + [ '--eval', evalString]
 
 
diff --git a/test/qa-tests/jstests/restore/blank_collection_bson.js b/test/qa-tests/jstests/restore/blank_collection_bson.js
index 966850a973b..6d183a66f18 100644
--- a/test/qa-tests/jstests/restore/blank_collection_bson.js
+++ b/test/qa-tests/jstests/restore/blank_collection_bson.js
@@ -4,8 +4,8 @@
         load('jstests/configs/plain_28.config.js');
     }
 
-    if (dump_targets == "archive") {
-        print('skipping test incompatable with archiving');
+    if (dump_targets != "standard") {
+        print('skipping test incompatable with archiving or compression');
         return assert(true);
     }
 
diff --git a/test/qa-tests/jstests/restore/conflicting_auth_schema_version.js b/test/qa-tests/jstests/restore/conflicting_auth_schema_version.js
index 122f5ba9918..6582780b68a 100644
--- a/test/qa-tests/jstests/restore/conflicting_auth_schema_version.js
+++ b/test/qa-tests/jstests/restore/conflicting_auth_schema_version.js
@@ -9,8 +9,8 @@
     jsTest.log('Testing restoring a dump with a potentially conflicting'+
             ' authSchemaVersion in the database');
 
-    if (dump_targets == "archive") {
-        print('skipping test incompatable with archiving');
+    if (dump_targets != "standard") {
+        print('skipping test incompatable with archiving or compression');
         return assert(true);
     }
 
diff --git a/test/qa-tests/jstests/restore/extended_json_metadata.js b/test/qa-tests/jstests/restore/extended_json_metadata.js
index d0f1e58f2bf..490d8112a79 100644
--- a/test/qa-tests/jstests/restore/extended_json_metadata.js
+++ b/test/qa-tests/jstests/restore/extended_json_metadata.js
@@ -4,8 +4,8 @@
     load('jstests/configs/plain_28.config.js');
   }
 
-  if (dump_targets == "archive") {
-      print('skipping test incompatable with archiving');
+  if (dump_targets != "standard") {
+      print('skipping test incompatable with archiving or compression');
       return assert(true);
   }
 
diff --git a/test/qa-tests/jstests/restore/indexes.js b/test/qa-tests/jstests/restore/indexes.js
index bb119a4048b..92c6ab9f7f5 100644
--- a/test/qa-tests/jstests/restore/indexes.js
+++ b/test/qa-tests/jstests/restore/indexes.js
@@ -73,6 +73,36 @@
     var indexesPost = testColl.getIndexes();
     assert.eq(indexesPre.length, indexesPost.length);
 
+    if ( dump_targets != "archive" ) {
+
+        // drop the collection again
+        testColl.drop();
+        // sanity check that the drop worked
+        assert.eq(0, testColl.count());
+
+        assert.eq(0, testColl.getIndexes().length);
+        
+        // restore the data, but this time mentioning the bson file specifically
+        ret = toolTest.runTool.apply(
+                toolTest,
+                ['restore'].
+                    concat(getRestoreTarget(dumpTarget+"/test/coll.bson")).
+                    concat(commonToolArgs)
+        );
+        assert.eq(0, ret);
+
+        // make sure the data was restored correctly
+        assert.eq(15, testColl.count());
+
+        // make sure the indexes were restored correctly
+        var indexesPost = testColl.getIndexes();
+        assert.eq(indexesPre.length, indexesPost.length);
+
+    } else {
+        jsTest.log('skipping bson file restore test while running with archiving'); 
+    }
+
+
     // success
     toolTest.stop();
 
diff --git a/test/qa-tests/jstests/restore/nonempty_temp_users.js b/test/qa-tests/jstests/restore/nonempty_temp_users.js
index 6b19850ae9e..6704078e43e 100644
--- a/test/qa-tests/jstests/restore/nonempty_temp_users.js
+++ b/test/qa-tests/jstests/restore/nonempty_temp_users.js
@@ -29,7 +29,9 @@
     );
 
     // dump the data
-    var ret = toolTest.runTool('dump', '--out', dumpTarget);
+    var ret = toolTest.runTool.apply(toolTest, ['dump'].
+            concat(getDumpTarget(dumpTarget)));
+    assert.neq(1, ret);
 
     // clear out the user
     adminDB.dropAllUsers();
diff --git a/test/qa-tests/jstests/restore/oplog_replay_and_limit.js b/test/qa-tests/jstests/restore/oplog_replay_and_limit.js
index ddf98fac902..ae8240b0a16 100644
--- a/test/qa-tests/jstests/restore/oplog_replay_and_limit.js
+++ b/test/qa-tests/jstests/restore/oplog_replay_and_limit.js
@@ -4,8 +4,8 @@
         load('jstests/configs/plain_28.config.js');
     }
 
-    if (dump_targets == "archive") {
-        print('skipping test incompatable with archiving');
+    if (dump_targets != "standard") {
+        print('skipping test incompatable with archiving or compression');
         return assert(true);
     }
 
diff --git a/test/qa-tests/jstests/restore/oplog_replay_noop.js b/test/qa-tests/jstests/restore/oplog_replay_noop.js
index c4a8d99607d..2557f666c22 100644
--- a/test/qa-tests/jstests/restore/oplog_replay_noop.js
+++ b/test/qa-tests/jstests/restore/oplog_replay_noop.js
@@ -4,8 +4,8 @@
         load('jstests/configs/plain_28.config.js');
     }
 
-    if (dump_targets == "archive") {
-        print('skipping test incompatable with archiving');
+    if (dump_targets != "standard") {
+        print('skipping test incompatable with archiving or compression');
         return assert(true);
     }
   
diff --git a/test/qa-tests/jstests/restore/symlinks.js b/test/qa-tests/jstests/restore/symlinks.js
index 31ded29e884..b8e9a11735b 100644
--- a/test/qa-tests/jstests/restore/symlinks.js
+++ b/test/qa-tests/jstests/restore/symlinks.js
@@ -6,8 +6,8 @@
         load('jstests/configs/plain_28.config.js');
     }
 
-    if (dump_targets == "archive") {
-        print('skipping test incompatable with archiving');
+    if (dump_targets != "standard") {
+        print('skipping test incompatable with archiving or compression');
         return assert(true);
     }
     
diff --git a/test/qa-tests/jstests/restore/users_and_roles_full_dump.js b/test/qa-tests/jstests/restore/users_and_roles_full_dump.js
index 68f9629f291..de38fd9f6ee 100644
--- a/test/qa-tests/jstests/restore/users_and_roles_full_dump.js
+++ b/test/qa-tests/jstests/restore/users_and_roles_full_dump.js
@@ -11,8 +11,8 @@
         load('jstests/configs/standard_dump_targets.config.js');
     }
 
-    if (dump_targets == "archive") {
-        print('skipping test incompatable with archiving');
+    if (dump_targets != "standard") {
+        print('skipping test incompatable with archiving or compression');
         return assert(true);
     }