diff options
author | Benety Goh <benety@mongodb.com> | 2014-10-07 19:48:14 -0400 |
---|---|---|
committer | Benety Goh <benety@mongodb.com> | 2014-10-08 08:29:27 -0400 |
commit | 4b033ae930afded6677c19022341241935992b33 (patch) | |
tree | 0d7c619d1543478cabcce936ed4af63aa79ea606 /src/mongo/tools | |
parent | c46e86610205985ae9744685e62fe269e03835e1 (diff) | |
download | mongo-4b033ae930afded6677c19022341241935992b33.tar.gz |
SERVER-15547 removed legacy tools from source tree and build configuration. make use-new-tools option a no-op
Diffstat (limited to 'src/mongo/tools')
50 files changed, 0 insertions, 8653 deletions
diff --git a/src/mongo/tools/bsondump.cpp b/src/mongo/tools/bsondump.cpp deleted file mode 100644 index d7945034c25..00000000000 --- a/src/mongo/tools/bsondump.cpp +++ /dev/null @@ -1,155 +0,0 @@ -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see <http://www.gnu.org/licenses/>. -* -* As a special exception, the copyright holders give permission to link the -* code of portions of this program with the OpenSSL library under certain -* conditions as described in each individual source file and distribute -* linked combinations including the program with the OpenSSL library. You -* must comply with the GNU Affero General Public License in all respects -* for all of the code used other than as permitted herein. If you modify -* file(s) with this exception, you may extend this exception to your -* version of the file(s), but you are not obligated to do so. If you do not -* wish to do so, delete this exception statement from your version. If you -* delete this exception statement from all source files in the program, -* then also delete it in the license file. -*/ - -#include "mongo/platform/basic.h" - -#include <fcntl.h> -#include <iomanip> - -#include "mongo/client/dbclientcursor.h" -#include "mongo/tools/bsondump_options.h" -#include "mongo/tools/tool.h" -#include "mongo/util/options_parser/option_section.h" -#include "mongo/util/text.h" - -using namespace mongo; - -class BSONDump : public BSONTool { - - enum OutputType { JSON , DEBUG } _type; - -public: - - BSONDump() : BSONTool() { } - - virtual void printHelp(ostream& out) { - printBSONDumpHelp(&out); - } - - virtual int doRun() { - { - if (bsonDumpGlobalParams.type == "json") - _type = JSON; - else if (bsonDumpGlobalParams.type == "debug") - _type = DEBUG; - else { - cerr << "bad type: " << bsonDumpGlobalParams.type << endl; - return 1; - } - } - - boost::filesystem::path root = bsonDumpGlobalParams.file; - if ( root == "" ) { - printBSONDumpHelp(&std::cout); - return 1; - } - - processFile(root, NULL); - return 0; - } - - bool debug( const BSONObj& o , int depth=0) { - string prefix = ""; - for ( int i=0; i<depth; i++ ) { - prefix += "\t\t\t"; - } - - int read = 4; - - try { - cout << prefix << "--- new object ---\n"; - cout << prefix << "\t size : " << o.objsize() << "\n"; - - // Note: this will recursively check each level of the bson and will also be called by - // this function at each level. While inefficient, it shouldn't effect correctness. - const Status status = validateBSON(o.objdata(), o.objsize()); - if (!status.isOK()) { - cout << prefix << "\t OBJECT IS INVALID: " << status.reason() << '\n' - << prefix << "\t attempting to print as much as possible" << endl; - } - - BSONObjIterator i(o); - while ( i.more() ) { - // This call verifies it is safe to call size() and fieldName() but doesn't check - // whether the element extends past the end of the object. That is done below. - BSONElement e = i.next(/*checkEnd=*/true); - - cout << prefix << "\t\t " << e.fieldName() << "\n" - << prefix << "\t\t\t type:" << setw(3) << e.type() << " size: " << e.size() - << endl; - - if ( ( read + e.size() ) > o.objsize() ) { - cout << prefix << " SIZE DOES NOT WORK" << endl; - return false; - } - read += e.size(); - try { - if ( e.isABSONObj() ) { - if ( ! debug( e.Obj() , depth + 1 ) ) { - //return false; - cout << prefix << "\t\t\t BAD BAD BAD" << endl; - - if ( e.size() < 1000 ) { - cout << "---\n" << e.Obj().hexDump() << "\n---" << endl; - } - } - } - else if ( e.type() == String && ! isValidUTF8( e.valuestr() ) ) { - cout << prefix << "\t\t\t" << "bad utf8 String!" << endl; - } - else if ( logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1)) ) { - cout << prefix << "\t\t\t" << e << endl; - } - } - catch ( std::exception& e ) { - cout << prefix << "\t\t\t bad value: " << e.what() << endl; - } - } - } - catch ( std::exception& e ) { - cout << prefix << "\tbad\t" << e.what() << endl; - cout << "----\n" << o.hexDump() << "\n---" << endl; - } - return true; - } - - virtual void gotObject(const BSONObj& o, std::ostream* out) { - switch ( _type ) { - case JSON: - cout << o.jsonString( TenGen ) << endl; - break; - case DEBUG: - debug(o); - break; - default: - cerr << "bad type? : " << _type << endl; - } - } -}; - -REGISTER_MONGO_TOOL(BSONDump); diff --git a/src/mongo/tools/bsondump_options.cpp b/src/mongo/tools/bsondump_options.cpp deleted file mode 100644 index 6c2e65e51a6..00000000000 --- a/src/mongo/tools/bsondump_options.cpp +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/tools/bsondump_options.h" - -#include "mongo/base/status.h" -#include "mongo/util/options_parser/startup_options.h" - -namespace mongo { - - BSONDumpGlobalParams bsonDumpGlobalParams; - - Status addBSONDumpOptions(moe::OptionSection* options) { - Status ret = addGeneralToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addBSONToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - options->addOptionChaining("type", "type", moe::String, "type of output: json,debug") - .setDefault(moe::Value(std::string("json"))); - - options->addOptionChaining("file", "file", moe::String, "path to BSON file to dump") - .hidden() - .setSources(moe::SourceCommandLine) - .positional(1, 1); - - - return Status::OK(); - } - - void printBSONDumpHelp(std::ostream* out) { - *out << "Display BSON documents in a data file.\n" << std::endl; - *out << "usage: bsondump [options] <bson filename>" << std::endl; - *out << moe::startupOptions.helpString(); - *out << std::flush; - } - - bool handlePreValidationBSONDumpOptions(const moe::Environment& params) { - if (!handlePreValidationGeneralToolOptions(params)) { - return false; - } - if (params.count("help")) { - printBSONDumpHelp(&std::cout); - return false; - } - return true; - } - - Status storeBSONDumpOptions(const moe::Environment& params, - const std::vector<std::string>& args) { - Status ret = storeGeneralToolOptions(params, args); - if (!ret.isOK()) { - return ret; - } - - ret = storeBSONToolOptions(params, args); - if (!ret.isOK()) { - return ret; - } - - // BSONDump never has a db connection - toolGlobalParams.noconnection = true; - - bsonDumpGlobalParams.type = getParam("type"); - bsonDumpGlobalParams.file = getParam("file"); - - // Make the default db "" if it was not explicitly set - if (!params.count("db")) { - toolGlobalParams.db = ""; - } - - // bsondump always outputs data to stdout, so we can't send messages there - toolGlobalParams.canUseStdout = false; - - return Status::OK(); - } - -} diff --git a/src/mongo/tools/bsondump_options.h b/src/mongo/tools/bsondump_options.h deleted file mode 100644 index b2e71a1a095..00000000000 --- a/src/mongo/tools/bsondump_options.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#pragma once - -#include <iosfwd> -#include <string> -#include <vector> - -#include "mongo/base/status.h" -#include "mongo/tools/tool_options.h" - -namespace mongo { - - struct BSONDumpGlobalParams { - std::string type; - std::string file; - }; - - extern BSONDumpGlobalParams bsonDumpGlobalParams; - - Status addBSONDumpOptions(moe::OptionSection* options); - - void printBSONDumpHelp(std::ostream* out); - - /** - * Handle options that should come before validation, such as "help". - * - * Returns false if an option was found that implies we should prematurely exit with success. - */ - bool handlePreValidationBSONDumpOptions(const moe::Environment& params); - - Status storeBSONDumpOptions(const moe::Environment& params, - const std::vector<std::string>& args); -} diff --git a/src/mongo/tools/bsondump_options_init.cpp b/src/mongo/tools/bsondump_options_init.cpp deleted file mode 100644 index cd517bea953..00000000000 --- a/src/mongo/tools/bsondump_options_init.cpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/tools/bsondump_options.h" - -#include "mongo/util/options_parser/startup_option_init.h" -#include "mongo/util/options_parser/startup_options.h" -#include "mongo/util/quick_exit.h" - -namespace mongo { - MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(BSONDumpOptions)(InitializerContext* context) { - return addBSONDumpOptions(&moe::startupOptions); - } - - MONGO_STARTUP_OPTIONS_VALIDATE(BSONDumpOptions)(InitializerContext* context) { - if (!handlePreValidationBSONDumpOptions(moe::startupOptionsParsed)) { - quickExit(EXIT_SUCCESS); - } - Status ret = moe::startupOptionsParsed.validate(); - if (!ret.isOK()) { - return ret; - } - return Status::OK(); - } - - MONGO_STARTUP_OPTIONS_STORE(BSONDumpOptions)(InitializerContext* context) { - Status ret = storeBSONDumpOptions(moe::startupOptionsParsed, context->args()); - if (!ret.isOK()) { - std::cerr << ret.toString() << std::endl; - std::cerr << "try '" << context->args()[0] << " --help' for more information" - << std::endl; - quickExit(EXIT_BADOPTIONS); - } - return Status::OK(); - } -} diff --git a/src/mongo/tools/dump.cpp b/src/mongo/tools/dump.cpp deleted file mode 100644 index 97772f54655..00000000000 --- a/src/mongo/tools/dump.cpp +++ /dev/null @@ -1,557 +0,0 @@ -/** -* Copyright (C) 2008-2014 MongoDB Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see <http://www.gnu.org/licenses/>. -* -* As a special exception, the copyright holders give permission to link the -* code of portions of this program with the OpenSSL library under certain -* conditions as described in each individual source file and distribute -* linked combinations including the program with the OpenSSL library. You -* must comply with the GNU Affero General Public License in all respects -* for all of the code used other than as permitted herein. If you modify -* file(s) with this exception, you may extend this exception to your -* version of the file(s), but you are not obligated to do so. If you do not -* wish to do so, delete this exception statement from your version. If you -* delete this exception statement from all source files in the program, -* then also delete it in the license file. -*/ - -#include "mongo/platform/basic.h" - -#include <boost/filesystem/operations.hpp> -#include <boost/filesystem/convenience.hpp> -#include <fcntl.h> -#include <fstream> -#include <map> - -#include "mongo/base/status.h" -#include "mongo/client/auth_helpers.h" -#include "mongo/client/dbclientcursor.h" -#include "mongo/db/auth/authorization_manager.h" -#include "mongo/db/catalog/database_catalog_entry.h" -#include "mongo/db/catalog/database_holder.h" -#include "mongo/db/namespace_string.h" -#include "mongo/db/operation_context_impl.h" -#include "mongo/db/catalog/collection.h" -#include "mongo/db/catalog/database.h" -#include "mongo/tools/mongodump_options.h" -#include "mongo/tools/tool.h" -#include "mongo/util/options_parser/option_section.h" -#include "mongo/util/log.h" -#include "mongo/util/mongoutils/str.h" - -using namespace mongo; - -class Dump : public Tool { - class FilePtr : boost::noncopyable { - public: - /*implicit*/ FilePtr(FILE* f) : _f(f) {} - ~FilePtr() { fclose(_f); } - operator FILE*() { return _f; } - private: - FILE* _f; - }; -public: - Dump() : Tool() { } - - virtual void printHelp(ostream& out) { - printMongoDumpHelp(&out); - } - - // This is a functor that writes a BSONObj to a file - struct Writer { - Writer(FILE* out, ProgressMeter* m) :_out(out), _m(m) {} - - void operator () (const BSONObj& obj) { - size_t toWrite = obj.objsize(); - size_t written = 0; - - while (toWrite) { - size_t ret = fwrite( obj.objdata()+written, 1, toWrite, _out ); - uassert(14035, errnoWithPrefix("couldn't write to file"), ret); - toWrite -= ret; - written += ret; - } - - // if there's a progress bar, hit it - if (_m) { - _m->hit(); - } - } - - FILE* _out; - ProgressMeter* _m; - }; - - void doCollection( const string coll , Query q, FILE* out , ProgressMeter *m, - bool usingMongos ) { - int queryOptions = QueryOption_SlaveOk | QueryOption_NoCursorTimeout; - if (startsWith(coll.c_str(), "local.oplog.") && q.obj.hasField("ts")) - queryOptions |= QueryOption_OplogReplay; - else if (mongoDumpGlobalParams.snapShotQuery) { - q.snapshot(); - } - - DBClientBase& connBase = conn(true); - Writer writer(out, m); - - // use low-latency "exhaust" mode if going over the network - if (!usingMongos && typeid(connBase) == typeid(DBClientConnection&)) { - DBClientConnection& conn = static_cast<DBClientConnection&>(connBase); - stdx::function<void(const BSONObj&)> castedWriter(writer); // needed for overload resolution - conn.query( castedWriter, coll.c_str() , q , NULL, queryOptions | QueryOption_Exhaust); - } - else { - //This branch should only be taken with DBDirectClient or mongos which doesn't support exhaust mode - scoped_ptr<DBClientCursor> cursor(connBase.query( coll.c_str() , q , 0 , 0 , 0 , queryOptions )); - while ( cursor->more() ) { - writer(cursor->next()); - } - } - } - - void writeCollectionFile( const string coll , Query q, boost::filesystem::path outputFile, - bool usingMongos ) { - toolInfoLog() << "\t" << coll << " to " << outputFile.string() << std::endl; - - FilePtr f (fopen(outputFile.string().c_str(), "wb")); - uassert(10262, errnoWithPrefix("couldn't open file"), f); - - ProgressMeter m(conn(true).count(coll.c_str(), BSONObj(), QueryOption_SlaveOk)); - m.setName("Collection File Writing Progress"); - m.setUnits("documents"); - - doCollection(coll, q, f, &m, usingMongos); - - toolInfoLog() << "\t\t " << m.done() - << ((m.done() == 1) ? " document" : " documents") - << std::endl; - } - - void writeMetadataFile( const string coll, boost::filesystem::path outputFile, - map<string, BSONObj> options, multimap<string, BSONObj> indexes ) { - toolInfoLog() << "\tMetadata for " << coll << " to " << outputFile.string() << std::endl; - - bool hasOptions = options.count(coll) > 0; - bool hasIndexes = indexes.count(coll) > 0; - - BSONObjBuilder metadata; - - if (hasOptions) { - metadata << "options" << options.find(coll)->second; - } - - if (hasIndexes) { - BSONArrayBuilder indexesOutput (metadata.subarrayStart("indexes")); - - // I'd kill for C++11 auto here... - const pair<multimap<string, BSONObj>::iterator, multimap<string, BSONObj>::iterator> - range = indexes.equal_range(coll); - - for (multimap<string, BSONObj>::iterator it=range.first; it!=range.second; ++it) { - indexesOutput << it->second; - } - - indexesOutput.done(); - } - - ofstream file (outputFile.string().c_str()); - uassert(15933, "Couldn't open file: " + outputFile.string(), file.is_open()); - file << metadata.done().jsonString(); - } - - - - void writeCollectionStdout( const string coll, const BSONObj& dumpQuery, bool usingMongos ) { - doCollection(coll, dumpQuery, stdout, NULL, usingMongos); - } - - void go(const string& db, - const string& coll, - const Query& query, - const boost::filesystem::path& outdir, - const string& outFilename, - bool usingMongos) { - // Can only provide outFilename if db and coll are provided - fassert(17368, outFilename.empty() || (!coll.empty() && !db.empty())); - boost::filesystem::create_directories( outdir ); - - map <string, BSONObj> collectionOptions; - multimap <string, BSONObj> indexes; - vector <string> collections; - - // Save indexes for database - string ins = db + ".system.indexes"; - auto_ptr<DBClientCursor> cursor = conn( true ).query( ins.c_str() , Query() , 0 , 0 , 0 , QueryOption_SlaveOk | QueryOption_NoCursorTimeout ); - while ( cursor->more() ) { - BSONObj obj = cursor->nextSafe(); - const string name = obj.getField( "ns" ).valuestr(); - indexes.insert( pair<string, BSONObj> (name, obj.getOwned()) ); - } - - string sns = db + ".system.namespaces"; - cursor = conn( true ).query( sns.c_str() , Query() , 0 , 0 , 0 , QueryOption_SlaveOk | QueryOption_NoCursorTimeout ); - while ( cursor->more() ) { - BSONObj obj = cursor->nextSafe(); - const string name = obj.getField( "name" ).valuestr(); - if (obj.hasField("options")) { - collectionOptions[name] = obj.getField("options").embeddedObject().getOwned(); - } - - // skip namespaces with $ in them only if we don't specify a collection to dump - if (coll == "" && name.find(".$") != string::npos) { - if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { - toolInfoLog() << "\tskipping collection: " << name << std::endl; - } - continue; - } - - // We use the collection name as the file name - const string filename = name.substr( db.size() + 1 ); - - //if a particular collections is specified, and it's not this one, skip it - if (coll != "" && db + "." + coll != name && coll != name) { - continue; - } - - // If the user has specified that we should exclude certain collections, skip them - std::vector<std::string>::const_iterator it = - std::find(mongoDumpGlobalParams.excludedCollections.begin(), - mongoDumpGlobalParams.excludedCollections.end(), filename); - - if (it != mongoDumpGlobalParams.excludedCollections.end()) { - if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { - toolInfoLog() << "\tskipping explicitly excluded collection: " << name - << std::endl; - } - continue; - } - - std::string matchedPrefix; - for (it = mongoDumpGlobalParams.excludeCollectionPrefixes.begin(); - it != mongoDumpGlobalParams.excludeCollectionPrefixes.end(); it++) { - if (startsWith(filename.c_str(), *it)) { - matchedPrefix = *it; - break; - } - } - - if (!matchedPrefix.empty()) { - if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { - toolInfoLog() << "\tskipping collection: " << name - << " that matches exclude prefix: " << matchedPrefix - << std::endl; - } - continue; - } - - // raise error before writing collection with non-permitted filename chars in the name - size_t hasBadChars = name.find_first_of("/\0"); - if (hasBadChars != string::npos){ - toolError() << "Cannot dump " << name - << ". Collection has '/' or null in the collection name." << std::endl; - continue; - } - - if (nsToCollectionSubstring(name) == "system.indexes") { - // Create system.indexes.bson for compatibility with pre 2.2 mongorestore - const string filename = name.substr( db.size() + 1 ); - writeCollectionFile( name.c_str(), query, outdir / ( filename + ".bson" ), - usingMongos ); - // Don't dump indexes as *.metadata.json - continue; - } - - if (nsToCollectionSubstring(name) == "system.users" && - !mongoDumpGlobalParams.dumpUsersAndRoles) { - continue; - } - - collections.push_back(name); - } - - for (vector<string>::iterator it = collections.begin(); it != collections.end(); ++it) { - string name = *it; - const string filename = outFilename != "" ? outFilename : name.substr( db.size() + 1 ); - writeCollectionFile( name , query, outdir / ( filename + ".bson" ), usingMongos ); - writeMetadataFile( name, outdir / (filename + ".metadata.json"), collectionOptions, indexes); - } - - } - - int repair() { - toolInfoLog() << "going to try and recover data from: " << toolGlobalParams.db << std::endl; - return _repairByName(toolGlobalParams.db); - } - - void _repairExtents(OperationContext* opCtx, Collection* coll, Writer& writer) { - scoped_ptr<RecordIterator> iter(coll->getRecordStore()->getIteratorForRepair(opCtx)); - - for (DiskLoc currLoc = iter->getNext(); !currLoc.isNull(); currLoc = iter->getNext()) { - if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { - toolInfoLog() << currLoc << std::endl; - } - - BSONObj obj; - try { - obj = coll->docFor(opCtx, currLoc); - - // If this is a corrupted object, just skip it, but do not abort the scan - // - if (!obj.valid()) { - continue; - } - - if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { - toolInfoLog() << obj << std::endl; - } - - writer(obj); - } - catch ( std::exception& e ) { - toolError() << "found invalid document @ " << currLoc << " " << e.what() << std::endl; - if ( ! obj.isEmpty() ) { - try { - BSONElement e = obj.firstElement(); - stringstream ss; - ss << "first element: " << e; - toolError() << ss.str() << std::endl; - } - catch ( std::exception& ) { - toolError() << "unable to log invalid document @ " << currLoc << std::endl; - } - } - } - } - } - - /* - * NOTE: The "outfile" parameter passed in should actually represent a directory, but it is - * called "outfile" because we append the filename and use it as our output file. - */ - void _repair(OperationContext* opCtx, - Database* db, - string ns, - boost::filesystem::path outfile) { - Collection* collection = db->getCollection(opCtx, ns); - toolInfoLog() << "nrecords: " << collection->numRecords(opCtx) - << " datasize: " << collection->dataSize(opCtx) - << std::endl; - - outfile /= ( ns.substr( ns.find( "." ) + 1 ) + ".bson" ); - toolInfoLog() << "writing to: " << outfile.string() << std::endl; - - FilePtr f (fopen(outfile.string().c_str(), "wb")); - - // init with double the docs count because we make two passes - ProgressMeter m( collection->numRecords(opCtx) * 2 ); - m.setName("Repair Progress"); - m.setUnits("documents"); - - Writer w( f , &m ); - - try { - WriteUnitOfWork wunit(opCtx); - _repairExtents(opCtx, collection, w); - wunit.commit(); - } - catch ( DBException& e ){ - toolError() << "Repair scan failed: " << e.toString() << std::endl; - } - - toolInfoLog() << "\t\t " << m.done() - << ((m.done() == 1) ? " document" : " documents") - << std::endl; - } - - int _repairByName(string dbname) { - OperationContextImpl txn; - Database* db = dbHolder().openDb(&txn, dbname); - - list<string> namespaces; - db->getDatabaseCatalogEntry()->getCollectionNamespaces( &namespaces ); - - boost::filesystem::path root = mongoDumpGlobalParams.outputDirectory; - root /= dbname; - boost::filesystem::create_directories( root ); - - for ( list<string>::iterator i=namespaces.begin(); i!=namespaces.end(); ++i ){ - LogIndentLevel lil; - string ns = *i; - - if ( str::endsWith( ns , ".system.namespaces" ) ) - continue; - - if ( str::contains( ns , ".tmp.mr." ) ) - continue; - - if (toolGlobalParams.coll != "" && - !str::endsWith(ns, toolGlobalParams.coll)) { - continue; - } - - toolInfoLog() << "trying to recover: " << ns << std::endl; - - LogIndentLevel lil1; - try { - _repair( &txn, db , ns , root ); - } - catch ( DBException& e ){ - toolError() << "ERROR recovering: " << ns << " " << e.toString() << std::endl; - } - } - - return 0; - } - - int run() { - bool usingMongos = isMongos(); - int serverAuthzVersion = 0; - BSONObj dumpQuery; - - if (mongoDumpGlobalParams.repair){ - return repair(); - } - - { - if (mongoDumpGlobalParams.query.size()) { - dumpQuery = fromjson(mongoDumpGlobalParams.query); - } - } - - if (mongoDumpGlobalParams.dumpUsersAndRoles) { - uassertStatusOK(auth::getRemoteStoredAuthorizationVersion(&conn(true), - &serverAuthzVersion)); - uassert(17369, - mongoutils::str::stream() << "Backing up users and roles is only supported for " - "clusters with auth schema versions 1, 3 or 4; found " << - serverAuthzVersion, - serverAuthzVersion == AuthorizationManager::schemaVersion24 || - serverAuthzVersion == AuthorizationManager::schemaVersion26Final || - serverAuthzVersion == AuthorizationManager::schemaVersion28SCRAM); - } - - string opLogName = ""; - unsigned long long opLogStart = 0; - if (mongoDumpGlobalParams.useOplog) { - - BSONObj isMaster; - conn("true").simpleCommand("admin", &isMaster, "isMaster"); - - if (isMaster.hasField("hosts")) { // if connected to replica set member - opLogName = "local.oplog.rs"; - } - else { - opLogName = "local.oplog.$main"; - if ( ! isMaster["ismaster"].trueValue() ) { - toolError() << "oplog mode is only supported on master or replica set member" - << std::endl; - return -1; - } - } - - BSONObj op = conn(true).findOne(opLogName, Query().sort("$natural", -1), 0, QueryOption_SlaveOk); - if (op.isEmpty()) { - toolError() << "No operations in oplog. Please ensure you are connecting to a " - << "master." << std::endl; - return -1; - } - - verify(op["ts"].type() == Timestamp); - opLogStart = op["ts"]._numberLong(); - } - - // check if we're outputting to stdout - if (mongoDumpGlobalParams.outputDirectory == "-") { - if (toolGlobalParams.db != "" && toolGlobalParams.coll != "") { - writeCollectionStdout(toolGlobalParams.db + "." + toolGlobalParams.coll, dumpQuery, - usingMongos); - return 0; - } - else { - toolError() << "You must specify database and collection to print to stdout" - << std::endl; - return -1; - } - } - - boost::filesystem::path root(mongoDumpGlobalParams.outputDirectory); - - if (toolGlobalParams.db == "") { - if (toolGlobalParams.coll != "") { - toolError() << "--db must be specified with --collection" << std::endl; - return -1; - } - - toolInfoLog() << "all dbs" << std::endl; - - BSONObj res = conn( true ).findOne( "admin.$cmd" , BSON( "listDatabases" << 1 ) ); - if ( ! res["databases"].isABSONObj() ) { - toolError() << "output of listDatabases isn't what we expected, no 'databases' " - << "field:\n" << res << std::endl; - return -2; - } - BSONObj dbs = res["databases"].embeddedObjectUserCheck(); - set<string> keys; - dbs.getFieldNames( keys ); - for ( set<string>::iterator i = keys.begin() ; i != keys.end() ; i++ ) { - string key = *i; - - if ( ! dbs[key].isABSONObj() ) { - toolError() << "database field not an document key: " << key << " value: " - << dbs[key] << std::endl; - return -3; - } - - BSONObj dbobj = dbs[key].embeddedObjectUserCheck(); - - const char * dbName = dbobj.getField( "name" ).valuestr(); - if ( (string)dbName == "local" ) - continue; - - boost::filesystem::path outdir = root / dbName; - toolInfoLog() << "DATABASE: " << dbName << "\t to \t" << outdir.string() - << std::endl; - go ( dbName , "", dumpQuery, outdir, "", usingMongos ); - } - } - else { - boost::filesystem::path outdir = root / toolGlobalParams.db; - toolInfoLog() << "DATABASE: " << toolGlobalParams.db << "\t to \t" << outdir.string() - << std::endl; - go(toolGlobalParams.db, toolGlobalParams.coll, dumpQuery, outdir, "", usingMongos); - if (mongoDumpGlobalParams.dumpUsersAndRoles && - serverAuthzVersion >= AuthorizationManager::schemaVersion26Final && - toolGlobalParams.db != "admin") { - toolInfoLog() << "Backing up user and role data for the " << toolGlobalParams.db << - " database"; - Query query = Query(BSON("db" << toolGlobalParams.db)); - go("admin", "system.users", query, outdir, "$admin.system.users", usingMongos); - go("admin", "system.roles", query, outdir, "$admin.system.roles", usingMongos); - } - } - - if (!opLogName.empty()) { - BSONObjBuilder b; - b.appendTimestamp("$gt", opLogStart); - - dumpQuery = BSON("ts" << b.obj()); - - writeCollectionFile( opLogName , dumpQuery, root / "oplog.bson", usingMongos ); - } - - return 0; - } -}; - -REGISTER_MONGO_TOOL(Dump); diff --git a/src/mongo/tools/export.cpp b/src/mongo/tools/export.cpp deleted file mode 100644 index ce34b5039d5..00000000000 --- a/src/mongo/tools/export.cpp +++ /dev/null @@ -1,244 +0,0 @@ -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see <http://www.gnu.org/licenses/>. -* -* As a special exception, the copyright holders give permission to link the -* code of portions of this program with the OpenSSL library under certain -* conditions as described in each individual source file and distribute -* linked combinations including the program with the OpenSSL library. You -* must comply with the GNU Affero General Public License in all respects -* for all of the code used other than as permitted herein. If you modify -* file(s) with this exception, you may extend this exception to your -* version of the file(s), but you are not obligated to do so. If you do not -* wish to do so, delete this exception statement from your version. If you -* delete this exception statement from all source files in the program, -* then also delete it in the license file. -*/ - -#include "mongo/pch.h" - -#include <boost/filesystem/convenience.hpp> -#include <boost/filesystem/operations.hpp> -#include <fstream> -#include <iostream> - -#include "mongo/client/dbclientcursor.h" -#include "mongo/db/json.h" -#include "mongo/tools/mongoexport_options.h" -#include "mongo/tools/tool.h" -#include "mongo/tools/tool_logger.h" -#include "mongo/util/hex.h" -#include "mongo/util/options_parser/option_section.h" - -using namespace mongo; - -class Export : public Tool { -public: - Export() : Tool() { } - - virtual void printHelp( ostream & out ) { - printMongoExportHelp(&out); - } - - // Turn every double quote character into two double quote characters - // If hasSurroundingQuotes is true, doesn't escape the first and last - // characters of the string, if it's false, add a double quote character - // around the whole string. - string csvEscape(string str, bool hasSurroundingQuotes = false) { - size_t index = hasSurroundingQuotes ? 1 : 0; - while (((index = str.find('"', index)) != string::npos) - && (index < (hasSurroundingQuotes ? str.size() - 1 : str.size()))) { - str.replace(index, 1, "\"\""); - index += 2; - } - return hasSurroundingQuotes ? str : "\"" + str + "\""; - } - - // Gets the string representation of a BSON object that can be correctly written to a CSV file - string csvString (const BSONElement& object) { - const char* binData; // Only used with BinData type - - switch (object.type()) { - case MinKey: - return "$MinKey"; - case MaxKey: - return "$MaxKey"; - case NumberInt: - case NumberDouble: - case NumberLong: - case Bool: - return object.toString(false); - case String: - case Symbol: - return csvEscape(object.toString(false, true), true); - case Object: - return csvEscape(object.jsonString(Strict, false)); - case Array: - return csvEscape(object.jsonString(Strict, false)); - case BinData: - int len; - binData = object.binDataClean(len); - return toHex(binData, len); - case jstOID: - return "ObjectId(" + object.OID().toString() + ")"; // OIDs are always 24 bytes - case Date: - // We need to check if we can actually format this date. See SERVER-13760. - return object.Date().isFormatable() ? - dateToISOStringUTC(object.Date()) : - csvEscape(object.jsonString(Strict, false)); - case Timestamp: - return csvEscape(object.jsonString(Strict, false)); - case RegEx: - return csvEscape("/" + string(object.regex()) + "/" + string(object.regexFlags())); - case Code: - return csvEscape(object.toString(false)); - case CodeWScope: - if (string(object.codeWScopeScopeDataUnsafe()) == "") { - return csvEscape(object.toString(false)); - } else { - return csvEscape(object.jsonString(Strict, false)); - } - case EOO: - case Undefined: - case DBRef: - case jstNULL: - cerr << "Invalid BSON object type for CSV output: " << object.type() << endl; - return ""; - } - // Can never get here - verify(false); - return ""; - } - - int run() { - string ns; - ostream *outPtr = &cout; - auto_ptr<ofstream> fileStream; - if (mongoExportGlobalParams.outputFileSpecified && mongoExportGlobalParams.outputFile != "-") { - size_t idx = mongoExportGlobalParams.outputFile.rfind("/"); - if ( idx != string::npos ) { - string dir = mongoExportGlobalParams.outputFile.substr( 0 , idx + 1 ); - boost::filesystem::create_directories( dir ); - } - ofstream * s = new ofstream(mongoExportGlobalParams.outputFile.c_str(), ios_base::out); - fileStream.reset( s ); - outPtr = s; - if ( ! s->good() ) { - cerr << "couldn't open [" << mongoExportGlobalParams.outputFile << "]" << endl; - return -1; - } - } - ostream &out = *outPtr; - - BSONObj * fieldsToReturn = 0; - BSONObj realFieldsToReturn; - - try { - ns = getNS(); - } - catch (...) { - printHelp(cerr); - return 1; - } - - if (toolGlobalParams.fieldsSpecified || mongoExportGlobalParams.csv) { - - // we can't use just toolGlobalParams.fields since we support everything getFieldDotted - // does - - set<string> seen; - BSONObjBuilder b; - - for (std::vector<std::string>::iterator i = toolGlobalParams.fields.begin(); - i != toolGlobalParams.fields.end(); i++) { - std::string f = str::before(*i, '.'); - if ( seen.insert( f ).second ) - b.append( f , 1 ); - } - - realFieldsToReturn = b.obj(); - fieldsToReturn = &realFieldsToReturn; - } - - - if (mongoExportGlobalParams.csv && !toolGlobalParams.fieldsSpecified) { - cerr << "csv mode requires a field list" << endl; - return -1; - } - - Query q(mongoExportGlobalParams.query); - if (mongoExportGlobalParams.sort != "") { - BSONObj sortSpec = mongo::fromjson(mongoExportGlobalParams.sort); - q.sort(sortSpec); - } - - if (mongoExportGlobalParams.snapShotQuery) { - q.snapshot(); - } - - auto_ptr<DBClientCursor> cursor = conn().query(ns.c_str(), q, - mongoExportGlobalParams.limit, mongoExportGlobalParams.skip, fieldsToReturn, - (mongoExportGlobalParams.slaveOk ? QueryOption_SlaveOk : 0) | - QueryOption_NoCursorTimeout); - - if (mongoExportGlobalParams.csv) { - for (std::vector<std::string>::iterator i = toolGlobalParams.fields.begin(); - i != toolGlobalParams.fields.end(); i++) { - if (i != toolGlobalParams.fields.begin()) - out << ","; - out << *i; - } - out << endl; - } - - if (mongoExportGlobalParams.jsonArray) - out << '['; - - long long num = 0; - while ( cursor->more() ) { - num++; - BSONObj obj = cursor->next(); - if (mongoExportGlobalParams.csv) { - for (std::vector<std::string>::iterator i = toolGlobalParams.fields.begin(); - i != toolGlobalParams.fields.end(); i++) { - if (i != toolGlobalParams.fields.begin()) - out << ","; - const BSONElement & e = obj.getFieldDotted(i->c_str()); - if ( ! e.eoo() ) { - out << csvString(e); - } - } - out << endl; - } - else { - if (mongoExportGlobalParams.jsonArray && num != 1) - out << ','; - - out << obj.jsonString(); - - if (!mongoExportGlobalParams.jsonArray) - out << endl; - } - } - - if (mongoExportGlobalParams.jsonArray) - out << ']' << endl; - - toolInfoOutput() << "exported " << num << " records" << endl; - - return 0; - } -}; - -REGISTER_MONGO_TOOL(Export); diff --git a/src/mongo/tools/files.cpp b/src/mongo/tools/files.cpp deleted file mode 100644 index 6ae94fea8df..00000000000 --- a/src/mongo/tools/files.cpp +++ /dev/null @@ -1,150 +0,0 @@ -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see <http://www.gnu.org/licenses/>. -* -* As a special exception, the copyright holders give permission to link the -* code of portions of this program with the OpenSSL library under certain -* conditions as described in each individual source file and distribute -* linked combinations including the program with the OpenSSL library. You -* must comply with the GNU Affero General Public License in all respects -* for all of the code used other than as permitted herein. If you modify -* file(s) with this exception, you may extend this exception to your -* version of the file(s), but you are not obligated to do so. If you do not -* wish to do so, delete this exception statement from your version. If you -* delete this exception statement from all source files in the program, -* then also delete it in the license file. -*/ - -#include "mongo/pch.h" - -#include <fstream> -#include <iostream> -#include <pcrecpp.h> - -#include "mongo/client/dbclientcursor.h" -#include "mongo/client/gridfs.h" -#include "mongo/tools/mongofiles_options.h" -#include "mongo/tools/tool.h" -#include "mongo/util/options_parser/option_section.h" - -using namespace mongo; - -class Files : public Tool { -public: - Files() : Tool() { } - - virtual void printHelp( ostream & out ) { - printMongoFilesHelp(&out); - } - - void display( GridFS * grid , BSONObj obj ) { - auto_ptr<DBClientCursor> c = grid->list( obj ); - while ( c->more() ) { - BSONObj obj = c->next(); - cout - << obj["filename"].str() << "\t" - << (long)obj["length"].number() - << endl; - } - } - - int run() { - if (mongoFilesGlobalParams.command.size() == 0) { - cerr << "ERROR: need command" << endl << endl; - printHelp(cout); - return -1; - } - - GridFS g(conn(), toolGlobalParams.db); - - if (mongoFilesGlobalParams.command == "list") { - BSONObjBuilder b; - if (mongoFilesGlobalParams.gridFSFilename.size()) { - b.appendRegex( "filename" , (string)"^" + - pcrecpp::RE::QuoteMeta(mongoFilesGlobalParams.gridFSFilename) ); - } - - display( &g , b.obj() ); - return 0; - } - - if (mongoFilesGlobalParams.gridFSFilename.size() == 0) { - cerr << "ERROR: need a filename" << endl << endl; - printHelp(cout); - return -1; - } - - if (mongoFilesGlobalParams.command == "search") { - BSONObjBuilder b; - b.appendRegex("filename", mongoFilesGlobalParams.gridFSFilename); - display( &g , b.obj() ); - return 0; - } - - if (mongoFilesGlobalParams.command == "get") { - GridFile f = g.findFile(mongoFilesGlobalParams.gridFSFilename); - if ( ! f.exists() ) { - cerr << "ERROR: file not found" << endl; - return -2; - } - - f.write(mongoFilesGlobalParams.localFile); - - if (mongoFilesGlobalParams.localFile != "-") { - toolInfoOutput() << "done write to: " << mongoFilesGlobalParams.localFile - << std::endl; - } - - return 0; - } - - if (mongoFilesGlobalParams.command == "put") { - BSONObj file = g.storeFile(mongoFilesGlobalParams.localFile, - mongoFilesGlobalParams.gridFSFilename, - mongoFilesGlobalParams.contentType); - toolInfoOutput() << "added file: " << file << std::endl; - - if (mongoFilesGlobalParams.replace) { - auto_ptr<DBClientCursor> cursor = - conn().query(toolGlobalParams.db + ".fs.files", - BSON("filename" << mongoFilesGlobalParams.gridFSFilename - << "_id" << NE << file["_id"] )); - while (cursor->more()) { - BSONObj o = cursor->nextSafe(); - conn().remove(toolGlobalParams.db + ".fs.files", BSON("_id" << o["_id"])); - conn().remove(toolGlobalParams.db + ".fs.chunks", BSON("_id" << o["_id"])); - toolInfoOutput() << "removed file: " << o << std::endl; - } - - } - - conn().getLastError(); - toolInfoOutput() << "done!" << std::endl; - return 0; - } - - if (mongoFilesGlobalParams.command == "delete") { - g.removeFile(mongoFilesGlobalParams.gridFSFilename); - conn().getLastError(); - toolInfoOutput() << "done!" << std::endl; - return 0; - } - - cerr << "ERROR: unknown command '" << mongoFilesGlobalParams.command << "'" << endl << endl; - printHelp(cout); - return -1; - } -}; - -REGISTER_MONGO_TOOL(Files); diff --git a/src/mongo/tools/import.cpp b/src/mongo/tools/import.cpp deleted file mode 100644 index 9d2f45ae472..00000000000 --- a/src/mongo/tools/import.cpp +++ /dev/null @@ -1,589 +0,0 @@ -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see <http://www.gnu.org/licenses/>. -* -* As a special exception, the copyright holders give permission to link the -* code of portions of this program with the OpenSSL library under certain -* conditions as described in each individual source file and distribute -* linked combinations including the program with the OpenSSL library. You -* must comply with the GNU Affero General Public License in all respects -* for all of the code used other than as permitted herein. If you modify -* file(s) with this exception, you may extend this exception to your -* version of the file(s), but you are not obligated to do so. If you do not -* wish to do so, delete this exception statement from your version. If you -* delete this exception statement from all source files in the program, -* then also delete it in the license file. -*/ - -#include "mongo/pch.h" - -#include <boost/algorithm/string.hpp> -#include <boost/filesystem/operations.hpp> -#include <fstream> -#include <iostream> - -#include "mongo/base/initializer.h" -#include "mongo/db/json.h" -#include "mongo/tools/mongoimport_options.h" -#include "mongo/tools/tool.h" -#include "mongo/util/log.h" -#include "mongo/util/options_parser/option_section.h" -#include "mongo/util/text.h" - -using namespace mongo; -using std::string; -using std::stringstream; - -class Import : public Tool { - - enum Type { JSON , CSV , TSV }; - Type _type; - - const char * _sep; - static const int BUF_SIZE; - - void csvTokenizeRow(const string& row, vector<string>& tokens) { - bool inQuotes = false; - bool prevWasQuote = false; - bool tokenQuoted = false; - string curtoken = ""; - for (string::const_iterator it = row.begin(); it != row.end(); ++it) { - char element = *it; - if (element == '"') { - if (!inQuotes) { - inQuotes = true; - tokenQuoted = true; - curtoken = ""; - } else { - if (prevWasQuote) { - curtoken += "\""; - prevWasQuote = false; - } else { - prevWasQuote = true; - } - } - } else { - if (inQuotes && prevWasQuote) { - inQuotes = false; - prevWasQuote = false; - tokens.push_back(curtoken); - } - - if (element == ',' && !inQuotes) { - if (!tokenQuoted) { // If token was quoted, it's already been added - boost::trim(curtoken); - tokens.push_back(curtoken); - } - curtoken = ""; - tokenQuoted = false; - } else { - curtoken += element; - } - } - } - if (!tokenQuoted || (inQuotes && prevWasQuote)) { - boost::trim(curtoken); - tokens.push_back(curtoken); - } - } - - void _append( BSONObjBuilder& b , const string& fieldName , const string& data ) { - if (mongoImportGlobalParams.ignoreBlanks && data.size() == 0) - return; - - if ( b.appendAsNumber( fieldName , data ) ) - return; - - // TODO: other types? - b.append ( fieldName , data ); - } - - /* - * Reads one line from in into buf. - * Returns the number of bytes that should be skipped - the caller should - * increment buf by this amount. - */ - int getLine(istream* in, char* buf) { - in->getline( buf , BUF_SIZE ); - if ((in->rdstate() & ios_base::eofbit) && (in->rdstate() & ios_base::failbit)) { - // this is the last line, and it's empty (not even a newline) - buf[0] = '\0'; - return 0; - } - - uassert(16329, str::stream() << "read error, or input line too long (max length: " - << BUF_SIZE << ")", !(in->rdstate() & ios_base::failbit)); - if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { - toolInfoLog() << "got line:" << buf << std::endl; - } - - uassert( 10263 , "unknown error reading file" , - (!(in->rdstate() & ios_base::badbit)) && - (!(in->rdstate() & ios_base::failbit) || (in->rdstate() & ios_base::eofbit)) ); - - int numBytesSkipped = 0; - if (strncmp("\xEF\xBB\xBF", buf, 3) == 0) { // UTF-8 BOM (notepad is stupid) - buf += 3; - numBytesSkipped += 3; - } - - uassert(13289, "Invalid UTF8 character detected", isValidUTF8(buf)); - return numBytesSkipped; - } - - /* - * description: - * given a pointer into a JSON array, returns the next valid JSON object - * args: - * buf - pointer into the JSON array - * o - BSONObj to fill - * numBytesRead - return parameter for how far we read - * return: - * true if an object was successfully parsed - * false if there is no object left to parse - * throws: - * exception on parsing error - */ - bool parseJSONArray(char* buf, BSONObj* o, int* numBytesRead) { - - // Skip extra characters since fromjson must be passed a character buffer that starts with a - // valid JSON object, and does not accept JSON arrays. - // (NOTE: this doesn't catch all invalid JSON arrays, but does fail on invalid characters) - *numBytesRead = 0; - while (buf[0] == '[' || - buf[0] == ']' || - buf[0] == ',' || - isspace(buf[0])) { - (*numBytesRead)++; - buf++; - } - - if (buf[0] == '\0') - return false; - - try { - int len = 0; - *o = fromjson(buf, &len); - (*numBytesRead) += len; - } catch ( MsgAssertionException& e ) { - uasserted(13293, string("Invalid JSON passed to mongoimport: ") + e.what()); - } - - return true; - } - - /* - * Parses one object from the input file. This usually corresponds to one line in the input - * file, unless the file is a CSV and contains a newline within a quoted string entry. - * Returns a true if a BSONObj was successfully created and false if not. - */ - bool parseRow(istream* in, BSONObj& o, int& numBytesRead) { - boost::scoped_array<char> buffer(new char[BUF_SIZE+2]); - char* line = buffer.get(); - - numBytesRead = getLine(in, line); - line += numBytesRead; - - if (line[0] == '\0') { - return false; - } - numBytesRead += strlen( line ); - - if (_type == JSON) { - // Strip out trailing whitespace - char * end = ( line + strlen( line ) ) - 1; - while ( end >= line && isspace(*end) ) { - *end = 0; - end--; - } - try { - o = fromjson( line ); - } catch ( MsgAssertionException& e ) { - uasserted(13504, string("BSON representation of supplied JSON is too large: ") + e.what()); - } - return true; - } - - vector<string> tokens; - if (_type == CSV) { - string row; - bool inside_quotes = false; - size_t last_quote = 0; - while (true) { - string lineStr(line); - // Deal with line breaks in quoted strings - last_quote = lineStr.find_first_of('"'); - while (last_quote != string::npos) { - inside_quotes = !inside_quotes; - last_quote = lineStr.find_first_of('"', last_quote+1); - } - - row.append(lineStr); - - if (inside_quotes) { - row.append("\n"); - int num = getLine(in, line); - line += num; - numBytesRead += num; - - uassert(15854, "CSV file ends while inside quoted field", line[0] != '\0'); - numBytesRead += strlen( line ); - } else { - break; - } - } - // now 'row' is string corresponding to one row of the CSV file - // (which may span multiple lines) and represents one BSONObj - csvTokenizeRow(row, tokens); - } - else { // _type == TSV - while (line[0] != '\t' && isspace(line[0])) { // Strip leading whitespace, but not tabs - line++; - } - - boost::split(tokens, line, boost::is_any_of(_sep)); - } - - // Now that the row is tokenized, create a BSONObj out of it. - BSONObjBuilder b; - unsigned int pos=0; - for (vector<string>::iterator it = tokens.begin(); it != tokens.end(); ++it) { - string token = *it; - if (mongoImportGlobalParams.headerLine) { - toolGlobalParams.fields.push_back(token); - } - else { - string name; - if (pos < toolGlobalParams.fields.size()) { - name = toolGlobalParams.fields[pos]; - } - else { - stringstream ss; - ss << "field" << pos; - name = ss.str(); - } - pos++; - - _append( b , name , token ); - } - } - o = b.obj(); - return true; - } - -public: - Import() : Tool() { - _type = JSON; - } - - virtual void printHelp( ostream & out ) { - printMongoImportHelp(&out); - } - - unsigned long long lastErrorFailures; - - /** @return true if ok */ - bool checkLastError() { - string s = conn().getLastError(); - if( !s.empty() ) { - if( str::contains(s,"uplicate") ) { - // we don't want to return an error from the mongoimport process for - // dup key errors - toolInfoLog() << s << endl; - } - else { - lastErrorFailures++; - toolInfoLog() << "error: " << s << endl; - return false; - } - } - return true; - } - - void importDocument (const std::string &ns, const BSONObj& o) { - bool doUpsert = mongoImportGlobalParams.upsert; - BSONObjBuilder b; - if (mongoImportGlobalParams.upsert) { - for (vector<string>::const_iterator it = mongoImportGlobalParams.upsertFields.begin(), - end = mongoImportGlobalParams.upsertFields.end(); it != end; ++it) { - BSONElement e = o.getFieldDotted(it->c_str()); - if (e.eoo()) { - doUpsert = false; - break; - } - b.appendAs(e, *it); - } - } - - if (doUpsert) { - conn().update(ns, Query(b.obj()), o, true); - } - else { - conn().insert(ns.c_str(), o); - } - } - - int run() { - long long fileSize = 0; - int headerRows = 0; - - istream * in = &cin; - - ifstream file(mongoImportGlobalParams.filename.c_str(), ios_base::in); - - if (mongoImportGlobalParams.filename.size() > 0 && - mongoImportGlobalParams.filename != "-") { - if ( ! boost::filesystem::exists(mongoImportGlobalParams.filename) ) { - toolError() << "file doesn't exist: " << mongoImportGlobalParams.filename - << std::endl; - return -1; - } - in = &file; - fileSize = boost::filesystem::file_size(mongoImportGlobalParams.filename); - } - - // check if we're actually talking to a machine that can write - if (!isMaster()) { - return -1; - } - - string ns; - - try { - ns = getNS(); - } - catch (...) { - // The only time getNS throws is when the collection was not specified. In that case, - // check if the user specified a file name and use that as the collection name. - if (!mongoImportGlobalParams.filename.empty()) { - string oldCollName = - boost::filesystem::path(mongoImportGlobalParams.filename).leaf().string(); - oldCollName = oldCollName.substr( 0 , oldCollName.find_last_of( "." ) ); - cerr << "using filename '" << oldCollName << "' as collection." << endl; - ns = toolGlobalParams.db + "." + oldCollName; - } - else { - printHelp(cerr); - return -1; - } - } - - if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { - toolInfoLog() << "ns: " << ns << endl; - } - - if (mongoImportGlobalParams.drop) { - toolInfoLog() << "dropping: " << ns << endl; - conn().dropCollection(ns.c_str()); - } - - if (mongoImportGlobalParams.type == "json") - _type = JSON; - else if (mongoImportGlobalParams.type == "csv") { - _type = CSV; - _sep = ","; - } - else if (mongoImportGlobalParams.type == "tsv") { - _type = TSV; - _sep = "\t"; - } - else { - toolError() << "don't know what type [" << mongoImportGlobalParams.type << "] is" - << std::endl; - return -1; - } - - if (_type == CSV || _type == TSV) { - if (mongoImportGlobalParams.headerLine) { - headerRows = 1; - } - else { - if (!toolGlobalParams.fieldsSpecified) { - throw UserException(9998, "You need to specify fields or have a headerline to " - "import this file type"); - } - } - } - - - time_t start = time(0); - if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { - toolInfoLog() << "filesize: " << fileSize << endl; - } - ProgressMeter pm( fileSize ); - int num = 0; - int lastNumChecked = num; - int errors = 0; - lastErrorFailures = 0; - int len = 0; - - // We have to handle jsonArrays differently since we can't read line by line - if (_type == JSON && mongoImportGlobalParams.jsonArray) { - - // We cycle through these buffers in order to continuously read from the stream - boost::scoped_array<char> buffer1(new char[BUF_SIZE]); - boost::scoped_array<char> buffer2(new char[BUF_SIZE]); - char* current_buffer = buffer1.get(); - char* next_buffer = buffer2.get(); - char* temp_buffer; - - // buffer_base_offset is the offset into the stream where our buffer starts, while - // input_stream_offset is the total number of bytes read from the stream - uint64_t buffer_base_offset = 0; - uint64_t input_stream_offset = 0; - - // Fill our buffer - // NOTE: istream::get automatically appends '\0' at the end of what it reads - in->get(current_buffer, BUF_SIZE, '\0'); - uassert(16808, str::stream() << "read error: " << strerror(errno), !in->fail()); - - // Record how far we read into the stream. - input_stream_offset += in->gcount(); - - while (true) { - try { - - BSONObj o; - - // Try to parse (parseJSONArray) - if (!parseJSONArray(current_buffer, &o, &len)) { - break; - } - - // Import documents - if (mongoImportGlobalParams.doimport) { - importDocument(ns, o); - - if (num < 10) { - // we absolutely want to check the first and last op of the batch. we do - // a few more as that won't be too time expensive. - checkLastError(); - lastNumChecked = num; - } - } - - // Copy over the part of buffer that was not parsed - strcpy(next_buffer, current_buffer + len); - - // Advance our buffer base past what we've already parsed - buffer_base_offset += len; - - // Fill up the end of our next buffer only if there is something in the stream - if (!in->eof()) { - // NOTE: istream::get automatically appends '\0' at the end of what it reads - in->get(next_buffer + (input_stream_offset - buffer_base_offset), - BUF_SIZE - (input_stream_offset - buffer_base_offset), '\0'); - uassert(16809, str::stream() << "read error: " - << strerror(errno), !in->fail()); - - // Record how far we read into the stream. - input_stream_offset += in->gcount(); - } - - // Swap buffer pointers - temp_buffer = current_buffer; - current_buffer = next_buffer; - next_buffer = temp_buffer; - - num++; - } - catch ( const std::exception& e ) { - toolError() << "exception: " << e.what() - << ", current buffer: " << current_buffer << std::endl; - errors++; - - // Since we only support JSON arrays all on one line, we might as well stop now - // because we can't read any more documents - break; - } - - if (!toolGlobalParams.quiet) { - if (pm.hit(len + 1)) { - log() << "\t\t\t" << num << "\t" << (num / (time(0) - start)) << "/second" - << std::endl; - } - } - } - } - else { - while (in->rdstate() == 0) { - try { - BSONObj o; - - if (!parseRow(in, o, len)) { - continue; - } - - if (mongoImportGlobalParams.headerLine) { - mongoImportGlobalParams.headerLine = false; - } - else if (mongoImportGlobalParams.doimport) { - importDocument(ns, o); - - if (num < 10) { - // we absolutely want to check the first and last op of the batch. we do - // a few more as that won't be too time expensive. - checkLastError(); - lastNumChecked = num; - } - } - - num++; - } - catch ( const std::exception& e ) { - toolError() << "exception:" << e.what() << std::endl; - errors++; - - if (mongoImportGlobalParams.stopOnError) - break; - } - - if (!toolGlobalParams.quiet) { - if (pm.hit(len + 1)) { - log() << "\t\t\t" << num << "\t" << (num / (time(0) - start)) << "/second" - << std::endl; - } - } - } - } - - // this is for two reasons: to wait for all operations to reach the server and be processed, and this will wait until all data reaches the server, - // and secondly to check if there were an error (on the last op) - if( lastNumChecked+1 != num ) { // avoid redundant log message if already reported above - toolInfoLog() << "check " << lastNumChecked << " " << num << endl; - checkLastError(); - } - - bool hadErrors = lastErrorFailures || errors; - - // the message is vague on lastErrorFailures as we don't call it on every single operation. - // so if we have a lastErrorFailure there might be more than just what has been counted. - toolInfoLog() << (lastErrorFailures ? "tried to import " : "imported ") - << (num - headerRows) - << (((num - headerRows) == 1) ? " document" : " documents") - << std::endl; - - if ( !hadErrors ) - return 0; - - toolError() << "encountered " << (lastErrorFailures?"at least ":"") - << lastErrorFailures+errors << " error(s)" - << (lastErrorFailures+errors == 1 ? "" : "s") << std::endl; - return -1; - } -}; - -const int Import::BUF_SIZE(1024 * 1024 * 16); - -REGISTER_MONGO_TOOL(Import); diff --git a/src/mongo/tools/mongodump_options.cpp b/src/mongo/tools/mongodump_options.cpp deleted file mode 100644 index cfa6f822d99..00000000000 --- a/src/mongo/tools/mongodump_options.cpp +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/tools/mongodump_options.h" - -#include "mongo/base/status.h" -#include "mongo/util/log.h" -#include "mongo/util/options_parser/startup_options.h" - -namespace mongo { - - MongoDumpGlobalParams mongoDumpGlobalParams; - - Status addMongoDumpOptions(moe::OptionSection* options) { - Status ret = addGeneralToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addRemoteServerToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addLocalServerToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addSpecifyDBCollectionToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - options->addOptionChaining("out", "out,o", moe::String, - "output directory or \"-\" for stdout") - .setDefault(moe::Value(std::string("dump"))); - - options->addOptionChaining("query", "query,q", moe::String, "json query"); - - options->addOptionChaining("oplog", "oplog", moe::Switch, - "Use oplog for point-in-time snapshotting"); - - options->addOptionChaining("repair", "repair", moe::Switch, - "try to recover a crashed database"); - - options->addOptionChaining("forceTableScan", "forceTableScan", moe::Switch, - "force a table scan (do not use $snapshot)"); - - options->addOptionChaining("dumpDbUsersAndRoles", "dumpDbUsersAndRoles", moe::Switch, - "Dump user and role definitions for the given database") - .requires("db").incompatibleWith("collection"); - - options->addOptionChaining("excludeCollection", "excludeCollection", moe::StringVector, - "Collection to exclude from the dump") - .requires("db").incompatibleWith("collection"); - - options->addOptionChaining("excludeCollectionsWithPrefix", "excludeCollectionsWithPrefix", - moe::StringVector, - "Exclude all collections from the dump that have the given prefix") - .requires("db").incompatibleWith("collection"); - - return Status::OK(); - } - - void printMongoDumpHelp(std::ostream* out) { - *out << "Export MongoDB data to BSON files.\n" << std::endl; - *out << moe::startupOptions.helpString(); - *out << std::flush; - } - - bool handlePreValidationMongoDumpOptions(const moe::Environment& params) { - if (!handlePreValidationGeneralToolOptions(params)) { - return false; - } - if (params.count("help")) { - printMongoDumpHelp(&std::cout); - return false;; - } - return true; - } - - Status storeMongoDumpOptions(const moe::Environment& params, - const std::vector<std::string>& args) { - Status ret = storeGeneralToolOptions(params, args); - if (!ret.isOK()) { - return ret; - } - - mongoDumpGlobalParams.repair = hasParam("repair"); - if (mongoDumpGlobalParams.repair){ - if (!hasParam("dbpath")) { - return Status(ErrorCodes::BadValue, "repair mode only works with --dbpath"); - } - - if (!hasParam("db")) { - return Status(ErrorCodes::BadValue, - "repair mode only works on 1 db at a time right now"); - } - } - mongoDumpGlobalParams.query = getParam("query"); - mongoDumpGlobalParams.useOplog = hasParam("oplog"); - if (mongoDumpGlobalParams.useOplog) { - if (hasParam("query") || hasParam("db") || hasParam("collection") || - hasParam("excludeCollection") || hasParam("excludeCollectionsWithPrefix")) { - return Status(ErrorCodes::BadValue, "oplog mode is only supported on full dumps"); - } - } - mongoDumpGlobalParams.outputDirectory = getParam("out"); - mongoDumpGlobalParams.snapShotQuery = false; - if (!hasParam("query") && !hasParam("dbpath") && !hasParam("forceTableScan")) { - mongoDumpGlobalParams.snapShotQuery = true; - } - - // Make the default db "" if it was not explicitly set - if (!params.count("db")) { - toolGlobalParams.db = ""; - } - - if (hasParam("dumpDbUsersAndRoles") && toolGlobalParams.db == "admin") { - return Status(ErrorCodes::BadValue, - "Cannot provide --dumpDbUsersAndRoles when dumping the admin db as " - "user and role definitions for the whole server are dumped by default " - "when dumping the admin db"); - } - - // Always dump users and roles if doing a full dump. If doing a db dump, only dump users - // and roles if --dumpDbUsersAndRoles provided or you're dumping the admin db. - mongoDumpGlobalParams.dumpUsersAndRoles = hasParam("dumpDbUsersAndRoles") || - (toolGlobalParams.db.empty() && toolGlobalParams.coll.empty()) || - toolGlobalParams.db == "admin"; - - if (mongoDumpGlobalParams.outputDirectory == "-") { - // write output to standard error to avoid mangling output - // must happen early to avoid sending junk to stdout - toolGlobalParams.canUseStdout = false; - } - - if (params.count("excludeCollection")) { - mongoDumpGlobalParams.excludedCollections = - params["excludeCollection"].as< std::vector<std::string> >(); - } - if (params.count("excludeCollectionsWithPrefix")) { - mongoDumpGlobalParams.excludeCollectionPrefixes = - params["excludeCollectionsWithPrefix"].as< std::vector<std::string> >(); - } - - return Status::OK(); - } - -} diff --git a/src/mongo/tools/mongodump_options.h b/src/mongo/tools/mongodump_options.h deleted file mode 100644 index ef1ead7505d..00000000000 --- a/src/mongo/tools/mongodump_options.h +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#pragma once - -#include <iosfwd> -#include <string> -#include <vector> - -#include "mongo/base/status.h" -#include "mongo/tools/tool_options.h" - -namespace mongo { - - struct MongoDumpGlobalParams { - std::string outputDirectory; - std::string query; - bool useOplog; - bool repair; - bool snapShotQuery; - bool dumpUsersAndRoles; - std::vector<std::string> excludedCollections; - std::vector<std::string> excludeCollectionPrefixes; - }; - - extern MongoDumpGlobalParams mongoDumpGlobalParams; - - Status addMongoDumpOptions(moe::OptionSection* options); - - void printMongoDumpHelp(std::ostream* out); - - /** - * Handle options that should come before validation, such as "help". - * - * Returns false if an option was found that implies we should prematurely exit with success. - */ - bool handlePreValidationMongoDumpOptions(const moe::Environment& params); - - Status storeMongoDumpOptions(const moe::Environment& params, - const std::vector<std::string>& args); -} diff --git a/src/mongo/tools/mongodump_options_init.cpp b/src/mongo/tools/mongodump_options_init.cpp deleted file mode 100644 index 3771e511dab..00000000000 --- a/src/mongo/tools/mongodump_options_init.cpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/tools/mongodump_options.h" - -#include "mongo/util/options_parser/startup_option_init.h" -#include "mongo/util/options_parser/startup_options.h" -#include "mongo/util/quick_exit.h" - -namespace mongo { - MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(MongoDumpOptions)(InitializerContext* context) { - return addMongoDumpOptions(&moe::startupOptions); - } - - MONGO_STARTUP_OPTIONS_VALIDATE(MongoDumpOptions)(InitializerContext* context) { - if (!handlePreValidationMongoDumpOptions(moe::startupOptionsParsed)) { - quickExit(EXIT_SUCCESS); - } - Status ret = moe::startupOptionsParsed.validate(); - if (!ret.isOK()) { - return ret; - } - return Status::OK(); - } - - MONGO_STARTUP_OPTIONS_STORE(MongoDumpOptions)(InitializerContext* context) { - Status ret = storeMongoDumpOptions(moe::startupOptionsParsed, context->args()); - if (!ret.isOK()) { - std::cerr << ret.toString() << std::endl; - std::cerr << "try '" << context->args()[0] << " --help' for more information" - << std::endl; - quickExit(EXIT_BADOPTIONS); - } - return Status::OK(); - } -} diff --git a/src/mongo/tools/mongoexport_options.cpp b/src/mongo/tools/mongoexport_options.cpp deleted file mode 100644 index 71e952827f3..00000000000 --- a/src/mongo/tools/mongoexport_options.cpp +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/tools/mongoexport_options.h" - -#include "mongo/base/status.h" -#include "mongo/util/options_parser/startup_options.h" - -namespace mongo { - - MongoExportGlobalParams mongoExportGlobalParams; - - Status addMongoExportOptions(moe::OptionSection* options) { - Status ret = addGeneralToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addRemoteServerToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addLocalServerToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addSpecifyDBCollectionToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addFieldOptions(options); - if (!ret.isOK()) { - return ret; - } - - options->addOptionChaining("query", "query,q", moe::String, - "query filter, as a JSON string, e.g., '{x:{$gt:1}}'"); - - options->addOptionChaining("csv", "csv", moe::Switch, "export to csv instead of json"); - - options->addOptionChaining("out", "out,o", moe::String, - "output file; if not specified, stdout is used"); - - options->addOptionChaining("jsonArray", "jsonArray", moe::Switch, - "output to a json array rather than one object per line"); - - options->addOptionChaining("slaveOk", "slaveOk,k", moe::Bool, - "use secondaries for export if available, default true") - .setDefault(moe::Value(true)); - - options->addOptionChaining("forceTableScan", "forceTableScan", moe::Switch, - "force a table scan (do not use $snapshot)"); - - options->addOptionChaining("skip", "skip", moe::Int, "documents to skip, default 0") - .setDefault(moe::Value(0)); - - options->addOptionChaining("limit", "limit", moe::Int, - "limit the numbers of documents returned, default all") - .setDefault(moe::Value(0)); - - options->addOptionChaining("sort", "sort", moe::String, - "sort order, as a JSON string, e.g., '{x:1}'"); - - - return Status::OK(); - } - - void printMongoExportHelp(std::ostream* out) { - *out << "Export MongoDB data to CSV, TSV or JSON files.\n" << std::endl; - *out << moe::startupOptions.helpString(); - *out << std::flush; - } - - bool handlePreValidationMongoExportOptions(const moe::Environment& params) { - if (!handlePreValidationGeneralToolOptions(params)) { - return false; - } - if (params.count("help")) { - printMongoExportHelp(&std::cout); - return false; - } - return true; - } - - Status storeMongoExportOptions(const moe::Environment& params, - const std::vector<std::string>& args) { - Status ret = storeGeneralToolOptions(params, args); - if (!ret.isOK()) { - return ret; - } - - ret = storeFieldOptions(params, args); - if (!ret.isOK()) { - return ret; - } - - mongoExportGlobalParams.outputFile = getParam("out"); - mongoExportGlobalParams.outputFileSpecified = hasParam("out"); - mongoExportGlobalParams.csv = hasParam("csv"); - mongoExportGlobalParams.jsonArray = hasParam("jsonArray"); - mongoExportGlobalParams.query = getParam("query", ""); - mongoExportGlobalParams.snapShotQuery = false; - - // Only allow snapshot query (requires _id idx scan) if following conditions are false - if (!hasParam("query") && - !hasParam("sort") && - !hasParam("dbpath") && - !hasParam("forceTableScan")) { - mongoExportGlobalParams.snapShotQuery = true; - } - - mongoExportGlobalParams.slaveOk = params["slaveOk"].as<bool>(); - mongoExportGlobalParams.limit = getParam("limit", 0); - mongoExportGlobalParams.skip = getParam("skip", 0); - mongoExportGlobalParams.sort = getParam("sort", ""); - - // we write output to standard error by default to avoid mangling output, but we don't need - // to do this if an output file was specified - toolGlobalParams.canUseStdout = false; - if (mongoExportGlobalParams.outputFileSpecified) { - if (mongoExportGlobalParams.outputFile != "-") { - toolGlobalParams.canUseStdout = true; - } - } - - return Status::OK(); - } - -} diff --git a/src/mongo/tools/mongoexport_options.h b/src/mongo/tools/mongoexport_options.h deleted file mode 100644 index 1db7206b462..00000000000 --- a/src/mongo/tools/mongoexport_options.h +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#pragma once - -#include <iosfwd> -#include <string> -#include <vector> - -#include "mongo/base/status.h" -#include "mongo/tools/tool_options.h" - -namespace mongo { - - struct MongoExportGlobalParams { - std::string query; - bool csv; - std::string outputFile; - bool outputFileSpecified; - bool jsonArray; - bool slaveOk; - bool snapShotQuery; - unsigned int skip; - unsigned int limit; - std::string sort; - }; - - extern MongoExportGlobalParams mongoExportGlobalParams; - - Status addMongoExportOptions(moe::OptionSection* options); - - void printMongoExportHelp(std::ostream* out); - - /** - * Handle options that should come before validation, such as "help". - * - * Returns false if an option was found that implies we should prematurely exit with success. - */ - bool handlePreValidationMongoExportOptions(const moe::Environment& params); - - Status storeMongoExportOptions(const moe::Environment& params, - const std::vector<std::string>& args); -} diff --git a/src/mongo/tools/mongoexport_options_init.cpp b/src/mongo/tools/mongoexport_options_init.cpp deleted file mode 100644 index ba9fef84513..00000000000 --- a/src/mongo/tools/mongoexport_options_init.cpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/tools/mongoexport_options.h" - -#include "mongo/util/options_parser/startup_option_init.h" -#include "mongo/util/options_parser/startup_options.h" -#include "mongo/util/quick_exit.h" - -namespace mongo { - MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(MongoExportOptions)(InitializerContext* context) { - return addMongoExportOptions(&moe::startupOptions); - } - - MONGO_STARTUP_OPTIONS_VALIDATE(MongoExportOptions)(InitializerContext* context) { - if (!handlePreValidationMongoExportOptions(moe::startupOptionsParsed)) { - quickExit(EXIT_SUCCESS); - } - Status ret = moe::startupOptionsParsed.validate(); - if (!ret.isOK()) { - return ret; - } - return Status::OK(); - } - - MONGO_STARTUP_OPTIONS_STORE(MongoExportOptions)(InitializerContext* context) { - Status ret = storeMongoExportOptions(moe::startupOptionsParsed, context->args()); - if (!ret.isOK()) { - std::cerr << ret.toString() << std::endl; - std::cerr << "try '" << context->args()[0] << " --help' for more information" - << std::endl; - quickExit(EXIT_BADOPTIONS); - } - return Status::OK(); - } -} diff --git a/src/mongo/tools/mongofiles_options.cpp b/src/mongo/tools/mongofiles_options.cpp deleted file mode 100644 index 1e423449fe6..00000000000 --- a/src/mongo/tools/mongofiles_options.cpp +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/tools/mongofiles_options.h" - -#include "mongo/base/status.h" -#include "mongo/util/options_parser/startup_options.h" - -namespace mongo { - - MongoFilesGlobalParams mongoFilesGlobalParams; - - Status addMongoFilesOptions(moe::OptionSection* options) { - Status ret = addGeneralToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addRemoteServerToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addLocalServerToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addSpecifyDBCollectionToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - options->addOptionChaining("local", "local,l", moe::String, - "local filename for put|get (default is to use the same name as " - "'gridfs filename')"); - - options->addOptionChaining("type", "type,t", moe::String, - "MIME type for put (default is to omit)"); - - options->addOptionChaining("replace", "replace,r", moe::Switch, - "Remove other files with same name after PUT"); - - options->addOptionChaining("command", "command", moe::String, - "gridfs command to run") - .hidden() - .setSources(moe::SourceCommandLine) - .positional(1, 1); - - options->addOptionChaining("file", "file", moe::String, - "'gridfs filename' with a special meaning for various commands") - .hidden() - .setSources(moe::SourceCommandLine) - .positional(2, 2); - - - return Status::OK(); - } - - void printMongoFilesHelp(std::ostream* out) { - *out << "Browse and modify a GridFS filesystem.\n" << std::endl; - *out << "usage: mongofiles [options] command [gridfs filename]" << std::endl; - *out << "command:" << std::endl; - *out << " one of (list|search|put|get)" << std::endl; - *out << " list - list all files. 'gridfs filename' is an optional prefix " << std::endl; - *out << " which listed filenames must begin with." << std::endl; - *out << " search - search all files. 'gridfs filename' is a substring " << std::endl; - *out << " which listed filenames must contain." << std::endl; - *out << " put - add a file with filename 'gridfs filename'" << std::endl; - *out << " get - get a file with filename 'gridfs filename'" << std::endl; - *out << " delete - delete all files with filename 'gridfs filename'" << std::endl; - *out << moe::startupOptions.helpString(); - *out << std::flush; - } - - bool handlePreValidationMongoFilesOptions(const moe::Environment& params) { - if (!handlePreValidationGeneralToolOptions(params)) { - return false; - } - if (params.count("help")) { - printMongoFilesHelp(&std::cout); - return false; - } - return true; - } - - Status storeMongoFilesOptions(const moe::Environment& params, - const std::vector<std::string>& args) { - Status ret = storeGeneralToolOptions(params, args); - if (!ret.isOK()) { - return ret; - } - - mongoFilesGlobalParams.command = getParam("command"); - mongoFilesGlobalParams.gridFSFilename = getParam("file"); - mongoFilesGlobalParams.localFile = getParam("local", mongoFilesGlobalParams.gridFSFilename); - mongoFilesGlobalParams.contentType = getParam("type", ""); - mongoFilesGlobalParams.replace = hasParam("replace"); - - return Status::OK(); - } - -} diff --git a/src/mongo/tools/mongofiles_options.h b/src/mongo/tools/mongofiles_options.h deleted file mode 100644 index 52a1ac9f187..00000000000 --- a/src/mongo/tools/mongofiles_options.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#pragma once - -#include <iosfwd> -#include <string> -#include <vector> - -#include "mongo/base/status.h" -#include "mongo/tools/tool_options.h" - -namespace mongo { - - struct MongoFilesGlobalParams { - std::string localFile; - std::string contentType; - bool replace; - std::string command; - std::string gridFSFilename; - }; - - extern MongoFilesGlobalParams mongoFilesGlobalParams; - - Status addMongoFilesOptions(moe::OptionSection* options); - - void printMongoFilesHelp(std::ostream* out); - - /** - * Handle options that should come before validation, such as "help". - * - * Returns false if an option was found that implies we should prematurely exit with success. - */ - bool handlePreValidationMongoFilesOptions(const moe::Environment& params); - - Status storeMongoFilesOptions(const moe::Environment& params, - const std::vector<std::string>& args); -} diff --git a/src/mongo/tools/mongofiles_options_init.cpp b/src/mongo/tools/mongofiles_options_init.cpp deleted file mode 100644 index 515acf1ab33..00000000000 --- a/src/mongo/tools/mongofiles_options_init.cpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/tools/mongofiles_options.h" - -#include "mongo/util/options_parser/startup_option_init.h" -#include "mongo/util/options_parser/startup_options.h" -#include "mongo/util/quick_exit.h" - -namespace mongo { - MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(MongoFilesOptions)(InitializerContext* context) { - return addMongoFilesOptions(&moe::startupOptions); - } - - MONGO_STARTUP_OPTIONS_VALIDATE(MongoFilesOptions)(InitializerContext* context) { - if (!handlePreValidationMongoFilesOptions(moe::startupOptionsParsed)) { - quickExit(EXIT_SUCCESS); - } - Status ret = moe::startupOptionsParsed.validate(); - if (!ret.isOK()) { - return ret; - } - return Status::OK(); - } - - MONGO_STARTUP_OPTIONS_STORE(MongoFilesOptions)(InitializerContext* context) { - Status ret = storeMongoFilesOptions(moe::startupOptionsParsed, context->args()); - if (!ret.isOK()) { - std::cerr << ret.toString() << std::endl; - std::cerr << "try '" << context->args()[0] << " --help' for more information" - << std::endl; - quickExit(EXIT_BADOPTIONS); - } - return Status::OK(); - } -} diff --git a/src/mongo/tools/mongoimport_options.cpp b/src/mongo/tools/mongoimport_options.cpp deleted file mode 100644 index 652a46336fe..00000000000 --- a/src/mongo/tools/mongoimport_options.cpp +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/tools/mongoimport_options.h" - -#include "mongo/base/status.h" -#include "mongo/util/options_parser/startup_options.h" -#include "mongo/util/text.h" - -namespace mongo { - - MongoImportGlobalParams mongoImportGlobalParams; - - Status addMongoImportOptions(moe::OptionSection* options) { - Status ret = addGeneralToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addRemoteServerToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addLocalServerToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addSpecifyDBCollectionToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addFieldOptions(options); - if (!ret.isOK()) { - return ret; - } - - options->addOptionChaining("ignoreBlanks", "ignoreBlanks", moe::Switch, - "if given, empty fields in csv and tsv will be ignored"); - - options->addOptionChaining("type", "type", moe::String, - "type of file to import. default: json (json,csv,tsv)"); - - options->addOptionChaining("file", "file", moe::String, - "file to import from; if not specified stdin is used") - .positional(1, 1); - - options->addOptionChaining("drop", "drop", moe::Switch, "drop collection first "); - - options->addOptionChaining("headerline", "headerline", moe::Switch, - "first line in input file is a header (CSV and TSV only)"); - - options->addOptionChaining("upsert", "upsert", moe::Switch, - "insert or update documents that already exist"); - - options->addOptionChaining("upsertFields", "upsertFields", moe::String, - "comma-separated fields for the query part of the upsert. " - "You should make sure this is indexed"); - - options->addOptionChaining("stopOnError", "stopOnError", moe::Switch, - "stop importing at first error rather than continuing"); - - options->addOptionChaining("jsonArray", "jsonArray", moe::Switch, - "load a json array, not one item per line. Currently limited to 16MB."); - - - options->addOptionChaining("noimport", "noimport", moe::Switch, - "don't actually import. useful for benchmarking parser") - .hidden(); - - - return Status::OK(); - } - - void printMongoImportHelp(std::ostream* out) { - *out << "Import CSV, TSV or JSON data into MongoDB.\n" << std::endl; - *out << "When importing JSON documents, each document must be a separate line of the input file.\n"; - *out << "\nExample:\n"; - *out << " mongoimport --host myhost --db my_cms --collection docs < mydocfile.json\n" << std::endl; - *out << moe::startupOptions.helpString(); - *out << std::flush; - } - - bool handlePreValidationMongoImportOptions(const moe::Environment& params) { - if (!handlePreValidationGeneralToolOptions(params)) { - return false; - } - if (params.count("help")) { - printMongoImportHelp(&std::cout); - return false; - } - return true; - } - - Status storeMongoImportOptions(const moe::Environment& params, - const std::vector<std::string>& args) { - Status ret = storeGeneralToolOptions(params, args); - if (!ret.isOK()) { - return ret; - } - - ret = storeFieldOptions(params, args); - if (!ret.isOK()) { - return ret; - } - - mongoImportGlobalParams.filename = getParam("file"); - mongoImportGlobalParams.drop = hasParam("drop"); - mongoImportGlobalParams.ignoreBlanks = hasParam("ignoreBlanks"); - - if (hasParam("upsert") || hasParam("upsertFields")) { - mongoImportGlobalParams.upsert = true; - - string uf = getParam("upsertFields"); - if (uf.empty()) { - mongoImportGlobalParams.upsertFields.push_back("_id"); - } - else { - StringSplitter(uf.c_str(), ",").split(mongoImportGlobalParams.upsertFields); - } - } - else { - mongoImportGlobalParams.upsert = false; - } - - mongoImportGlobalParams.doimport = !hasParam("noimport"); - mongoImportGlobalParams.type = getParam("type", "json"); - mongoImportGlobalParams.jsonArray = hasParam("jsonArray"); - mongoImportGlobalParams.headerLine = hasParam("headerline"); - mongoImportGlobalParams.stopOnError = hasParam("stopOnError"); - - return Status::OK(); - } - -} diff --git a/src/mongo/tools/mongoimport_options.h b/src/mongo/tools/mongoimport_options.h deleted file mode 100644 index 13bbd5f05a2..00000000000 --- a/src/mongo/tools/mongoimport_options.h +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#pragma once - -#include <iosfwd> -#include <string> -#include <vector> - -#include "mongo/base/status.h" -#include "mongo/tools/tool_options.h" - -namespace mongo { - - struct MongoImportGlobalParams { - bool ignoreBlanks; - std::string type; - std::string filename; - bool drop; - bool headerLine; - bool upsert; - std::vector<std::string> upsertFields; - bool stopOnError; - bool jsonArray; - bool doimport; - }; - - extern MongoImportGlobalParams mongoImportGlobalParams; - - Status addMongoImportOptions(moe::OptionSection* options); - - void printMongoImportHelp(std::ostream* out); - - /** - * Handle options that should come before validation, such as "help". - * - * Returns false if an option was found that implies we should prematurely exit with success. - */ - bool handlePreValidationMongoImportOptions(const moe::Environment& params); - - Status storeMongoImportOptions(const moe::Environment& params, - const std::vector<std::string>& args); -} diff --git a/src/mongo/tools/mongoimport_options_init.cpp b/src/mongo/tools/mongoimport_options_init.cpp deleted file mode 100644 index 2bd94cd25e9..00000000000 --- a/src/mongo/tools/mongoimport_options_init.cpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/tools/mongoimport_options.h" - -#include "mongo/util/options_parser/startup_option_init.h" -#include "mongo/util/options_parser/startup_options.h" -#include "mongo/util/quick_exit.h" - -namespace mongo { - MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(MongoImportOptions)(InitializerContext* context) { - return addMongoImportOptions(&moe::startupOptions); - } - - MONGO_STARTUP_OPTIONS_VALIDATE(MongoImportOptions)(InitializerContext* context) { - if (!handlePreValidationMongoImportOptions(moe::startupOptionsParsed)) { - quickExit(EXIT_SUCCESS); - } - Status ret = moe::startupOptionsParsed.validate(); - if (!ret.isOK()) { - return ret; - } - return Status::OK(); - } - - MONGO_STARTUP_OPTIONS_STORE(MongoImportOptions)(InitializerContext* context) { - Status ret = storeMongoImportOptions(moe::startupOptionsParsed, context->args()); - if (!ret.isOK()) { - std::cerr << ret.toString() << std::endl; - std::cerr << "try '" << context->args()[0] << " --help' for more information" - << std::endl; - quickExit(EXIT_BADOPTIONS); - } - return Status::OK(); - } -} diff --git a/src/mongo/tools/mongooplog_options.cpp b/src/mongo/tools/mongooplog_options.cpp deleted file mode 100644 index e67a33b9710..00000000000 --- a/src/mongo/tools/mongooplog_options.cpp +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/tools/mongooplog_options.h" - -#include "mongo/base/status.h" -#include "mongo/util/log.h" -#include "mongo/util/options_parser/startup_options.h" - -namespace mongo { - - MongoOplogGlobalParams mongoOplogGlobalParams; - - Status addMongoOplogOptions(moe::OptionSection* options) { - Status ret = addGeneralToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addRemoteServerToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addLocalServerToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addSpecifyDBCollectionToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - options->addOptionChaining("seconds", "seconds,s", moe::Int, - "seconds to go back default:86400"); - - options->addOptionChaining("from", "from", moe::String, "host to pull from"); - - options->addOptionChaining("oplogns", "oplogns", moe::String, "ns to pull from") - .setDefault(moe::Value(std::string("local.oplog.rs"))); - - - return Status::OK(); - } - - void printMongoOplogHelp(std::ostream* out) { - *out << "Pull and replay a remote MongoDB oplog.\n" << std::endl; - *out << moe::startupOptions.helpString(); - *out << std::flush; - } - - bool handlePreValidationMongoOplogOptions(const moe::Environment& params) { - if (!handlePreValidationGeneralToolOptions(params)) { - return false; - } - if (params.count("help")) { - printMongoOplogHelp(&std::cout); - return false; - } - return true; - } - - Status storeMongoOplogOptions(const moe::Environment& params, - const std::vector<std::string>& args) { - Status ret = storeGeneralToolOptions(params, args); - if (!ret.isOK()) { - return ret; - } - - if (!hasParam("from")) { - return Status(ErrorCodes::BadValue, "need to specify --from"); - } - else { - mongoOplogGlobalParams.from = getParam("from"); - } - - mongoOplogGlobalParams.seconds = getParam("seconds", 86400); - mongoOplogGlobalParams.ns = getParam("oplogns"); - - return Status::OK(); - } - -} diff --git a/src/mongo/tools/mongooplog_options.h b/src/mongo/tools/mongooplog_options.h deleted file mode 100644 index 84d53f75a15..00000000000 --- a/src/mongo/tools/mongooplog_options.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#pragma once - -#include <iosfwd> -#include <string> -#include <vector> - -#include "mongo/base/status.h" -#include "mongo/tools/tool_options.h" - -namespace mongo { - - struct MongoOplogGlobalParams { - int seconds; - std::string from; - std::string ns; - }; - - extern MongoOplogGlobalParams mongoOplogGlobalParams; - - Status addMongoOplogOptions(moe::OptionSection* options); - - void printMongoOplogHelp(std::ostream* out); - - /** - * Handle options that should come before validation, such as "help". - * - * Returns false if an option was found that implies we should prematurely exit with success. - */ - bool handlePreValidationMongoOplogOptions(const moe::Environment& params); - - Status storeMongoOplogOptions(const moe::Environment& params, - const std::vector<std::string>& args); -} diff --git a/src/mongo/tools/mongooplog_options_init.cpp b/src/mongo/tools/mongooplog_options_init.cpp deleted file mode 100644 index f8b7f1da71a..00000000000 --- a/src/mongo/tools/mongooplog_options_init.cpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/tools/mongooplog_options.h" - -#include "mongo/util/options_parser/startup_option_init.h" -#include "mongo/util/options_parser/startup_options.h" -#include "mongo/util/quick_exit.h" - -namespace mongo { - MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(MongoOplogOptions)(InitializerContext* context) { - return addMongoOplogOptions(&moe::startupOptions); - } - - MONGO_STARTUP_OPTIONS_VALIDATE(MongoOplogOptions)(InitializerContext* context) { - if (!handlePreValidationMongoOplogOptions(moe::startupOptionsParsed)) { - quickExit(EXIT_SUCCESS); - } - Status ret = moe::startupOptionsParsed.validate(); - if (!ret.isOK()) { - return ret; - } - return Status::OK(); - } - - MONGO_STARTUP_OPTIONS_STORE(MongoOplogOptions)(InitializerContext* context) { - Status ret = storeMongoOplogOptions(moe::startupOptionsParsed, context->args()); - if (!ret.isOK()) { - std::cerr << ret.toString() << std::endl; - std::cerr << "try '" << context->args()[0] << " --help' for more information" - << std::endl; - quickExit(EXIT_BADOPTIONS); - } - return Status::OK(); - } -} diff --git a/src/mongo/tools/mongorestore_options.cpp b/src/mongo/tools/mongorestore_options.cpp deleted file mode 100644 index 2ee83dbcfff..00000000000 --- a/src/mongo/tools/mongorestore_options.cpp +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/tools/mongorestore_options.h" - -#include "mongo/base/status.h" -#include "mongo/db/auth/authorization_manager.h" -#include "mongo/util/options_parser/startup_options.h" - -namespace mongo { - - MongoRestoreGlobalParams mongoRestoreGlobalParams; - - Status addMongoRestoreOptions(moe::OptionSection* options) { - Status ret = addGeneralToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addRemoteServerToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addLocalServerToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addSpecifyDBCollectionToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addBSONToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - options->addOptionChaining("drop", "drop", moe::Switch, - "drop each collection before import"); - - options->addOptionChaining("oplogReplay", "oplogReplay", moe::Switch, - "replay oplog for point-in-time restore"); - - options->addOptionChaining("oplogLimit", "oplogLimit", moe::String, - "include oplog entries before the provided Timestamp " - "(seconds[:ordinal]) during the oplog replay; " - "the ordinal value is optional"); - - options->addOptionChaining("keepIndexVersion", "keepIndexVersion", moe::Switch, - "don't upgrade indexes to newest version"); - - options->addOptionChaining("noOptionsRestore", "noOptionsRestore", moe::Switch, - "don't restore collection options"); - - options->addOptionChaining("noIndexRestore", "noIndexRestore", moe::Switch, - "don't restore indexes"); - - options->addOptionChaining("restoreDbUsersAndRoles", "restoreDbUsersAndRoles", moe::Switch, - "Restore user and role definitions for the given database") - .requires("db").incompatibleWith("collection"); - - options->addOptionChaining( - "tempUsersCollection", "tempUsersCollection", moe::String, - "Collection in which to temporarily store user data during the restore") - .hidden() - .setDefault(moe::Value( - AuthorizationManager::defaultTempUsersCollectionNamespace.toString())); - - options->addOptionChaining( - "tempRolesCollection", "tempRolesCollection", moe::String, - "Collection in which to temporarily store role data during the restore") - .hidden() - .setDefault(moe::Value( - AuthorizationManager::defaultTempRolesCollectionNamespace.toString())); - - options->addOptionChaining("w", "w", moe::Int, "minimum number of replicas per write") - .setDefault(moe::Value(0)); - - options->addOptionChaining("dir", "dir", moe::String, "directory to restore from") - .hidden() - .setDefault(moe::Value(std::string("dump"))) - .positional(1, 1); - - - // left in for backwards compatibility - options->addOptionChaining("indexesLast", "indexesLast", moe::Switch, - "wait to add indexes (now default)") - .hidden(); - - - return Status::OK(); - } - - void printMongoRestoreHelp(std::ostream* out) { - *out << "Import BSON files into MongoDB.\n" << std::endl; - *out << "usage: mongorestore [options] [directory or filename to restore from]" - << std::endl; - *out << moe::startupOptions.helpString(); - *out << std::flush; - } - - bool handlePreValidationMongoRestoreOptions(const moe::Environment& params) { - if (!handlePreValidationGeneralToolOptions(params)) { - return false; - } - if (params.count("help")) { - printMongoRestoreHelp(&std::cout); - return false; - } - return true; - } - - Status storeMongoRestoreOptions(const moe::Environment& params, - const std::vector<std::string>& args) { - Status ret = storeGeneralToolOptions(params, args); - if (!ret.isOK()) { - return ret; - } - - ret = storeBSONToolOptions(params, args); - if (!ret.isOK()) { - return ret; - } - - mongoRestoreGlobalParams.restoreDirectory = getParam("dir"); - mongoRestoreGlobalParams.drop = hasParam("drop"); - mongoRestoreGlobalParams.keepIndexVersion = hasParam("keepIndexVersion"); - mongoRestoreGlobalParams.restoreOptions = !hasParam("noOptionsRestore"); - mongoRestoreGlobalParams.restoreIndexes = !hasParam("noIndexRestore"); - mongoRestoreGlobalParams.w = getParam( "w" , 0 ); - mongoRestoreGlobalParams.oplogReplay = hasParam("oplogReplay"); - mongoRestoreGlobalParams.oplogLimit = getParam("oplogLimit", ""); - mongoRestoreGlobalParams.tempUsersColl = getParam("tempUsersCollection"); - mongoRestoreGlobalParams.tempRolesColl = getParam("tempRolesCollection"); - - // Make the default db "" if it was not explicitly set - if (!params.count("db")) { - toolGlobalParams.db = ""; - } - - if (hasParam("restoreDbUsersAndRoles") && toolGlobalParams.db == "admin") { - return Status(ErrorCodes::BadValue, - "Cannot provide --restoreDbUsersAndRoles when restoring the admin db as " - "user and role definitions for the whole server are restored by " - "default (if present) when restoring the admin db"); - } - - // Always restore users and roles if doing a full restore. If doing a db restore, only - // restore users and roles if --restoreDbUsersAndRoles provided or you're restoring the - // admin db - mongoRestoreGlobalParams.restoreUsersAndRoles = hasParam("restoreDbUsersAndRoles") || - (toolGlobalParams.db.empty() && toolGlobalParams.coll.empty()) || - (toolGlobalParams.db == "admin" && toolGlobalParams.coll.empty()); - - return Status::OK(); - } - -} diff --git a/src/mongo/tools/mongorestore_options.h b/src/mongo/tools/mongorestore_options.h deleted file mode 100644 index f0208952110..00000000000 --- a/src/mongo/tools/mongorestore_options.h +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#pragma once - -#include <iosfwd> -#include <string> -#include <vector> - -#include "mongo/base/status.h" -#include "mongo/tools/tool_options.h" - -namespace mongo { - - struct MongoRestoreGlobalParams { - bool drop; - bool oplogReplay; - std::string oplogLimit; - bool keepIndexVersion; - bool restoreOptions; - bool restoreIndexes; - bool restoreUsersAndRoles; - int w; - std::string restoreDirectory; - std::string tempUsersColl; - std::string tempRolesColl; - }; - - extern MongoRestoreGlobalParams mongoRestoreGlobalParams; - - Status addMongoRestoreOptions(moe::OptionSection* options); - - void printMongoRestoreHelp(std::ostream* out); - - /** - * Handle options that should come before validation, such as "help". - * - * Returns false if an option was found that implies we should prematurely exit with success. - */ - bool handlePreValidationMongoRestoreOptions(const moe::Environment& params); - - Status storeMongoRestoreOptions(const moe::Environment& params, - const std::vector<std::string>& args); -} diff --git a/src/mongo/tools/mongorestore_options_init.cpp b/src/mongo/tools/mongorestore_options_init.cpp deleted file mode 100644 index 2e37c7c955e..00000000000 --- a/src/mongo/tools/mongorestore_options_init.cpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/tools/mongorestore_options.h" - -#include "mongo/util/options_parser/startup_option_init.h" -#include "mongo/util/options_parser/startup_options.h" -#include "mongo/util/quick_exit.h" - -namespace mongo { - MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(MongoRestoreOptions)(InitializerContext* context) { - return addMongoRestoreOptions(&moe::startupOptions); - } - - MONGO_STARTUP_OPTIONS_VALIDATE(MongoRestoreOptions)(InitializerContext* context) { - if (!handlePreValidationMongoRestoreOptions(moe::startupOptionsParsed)) { - quickExit(EXIT_SUCCESS); - } - Status ret = moe::startupOptionsParsed.validate(); - if (!ret.isOK()) { - return ret; - } - return Status::OK(); - } - - MONGO_STARTUP_OPTIONS_STORE(MongoRestoreOptions)(InitializerContext* context) { - Status ret = storeMongoRestoreOptions(moe::startupOptionsParsed, context->args()); - if (!ret.isOK()) { - std::cerr << ret.toString() << std::endl; - std::cerr << "try '" << context->args()[0] << " --help' for more information" - << std::endl; - quickExit(EXIT_BADOPTIONS); - } - return Status::OK(); - } -} diff --git a/src/mongo/tools/mongoshim_options.cpp b/src/mongo/tools/mongoshim_options.cpp deleted file mode 100644 index 2ea23c01f30..00000000000 --- a/src/mongo/tools/mongoshim_options.cpp +++ /dev/null @@ -1,258 +0,0 @@ -// mongoshim_options.cpp - -/* - * Copyright (C) 2014 MongoDB Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/platform/basic.h" - -#include "mongo/tools/mongoshim_options.h" - -#include <vector> - -#include "mongo/base/status.h" -#include "mongo/db/json.h" -#include "mongo/util/options_parser/startup_options.h" -#include "mongo/util/stringutils.h" -#include "mongo/util/text.h" - -using std::string; -using std::vector; - -namespace mongo { - - MongoShimGlobalParams mongoShimGlobalParams; - - Status addMongoShimOptions(moe::OptionSection* options) { - Status ret = addGeneralToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addLocalServerToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addSpecifyDBCollectionToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addFieldOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addBSONToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - // Required option --mode. - // Refer to shim_mode.h for list of supported modes. - vector<string> modeStrings; - for (int i = 0; i <= int(ShimMode::kNumShimModes); ++i) { - ShimMode mode = static_cast<ShimMode::Value>(i); - modeStrings.push_back(mode.toString()); - for (int i = 0; i <= int(ShimMode::kNumShimModes); ++i) { - ShimMode mode = static_cast<ShimMode::Value>(i); - modeStrings.push_back(mode.toString()); - } - } - string modeDisplayFormat; - joinStringDelim(modeStrings, &modeDisplayFormat, '|'); - options->addOptionChaining("mode", "mode", moe::String, - "Runs shim in one of several modes: " + modeDisplayFormat) - .format(modeDisplayFormat, - modeDisplayFormat); - - // Compatible with --mode=insert only. - // Used to drop collection before inserting documents read from input. - options->addOptionChaining("drop", "drop", moe::Switch, - "drop collection before inserting documents"); - - // Compatible with --mode=upsert only. - // Used to specify fields for matching existing documents when - // updating/inserting. - options->addOptionChaining("upsertFields", "upsertFields", moe::String, - "comma-separated fields for the query part of the upsert. " - "Ensure these fields are indexed."); - - // Used to filter document results before writing to to output. - options->addOptionChaining("query", "query,q", moe::String, - "query filter, as a JSON string, e.g., '{x:{$gt:1}}'"); - - options->addOptionChaining("skip", "skip", moe::Int, "documents to skip, default 0") - .setDefault(moe::Value(0)); - - options->addOptionChaining("limit", "limit", moe::Int, - "limit the numbers of documents returned, default all") - .setDefault(moe::Value(0)); - - options->addOptionChaining("sort", "sort", moe::String, - "sort order, as a JSON string, e.g., '{x:1}'"); - - // Used for testing. - options->addOptionChaining("in", "in", moe::String, - "input file; if not specified, stdin is used") - .hidden(); - - // Used for testing. - options->addOptionChaining("inputDocuments", "inputDocuments", moe::String, - "input documents. If specified, stdin will be ignored. " - "Format: --inputDocuments=\"{in: [doc1, doc2, doc3, ...]}\". ") - .incompatibleWith("in") - .hidden(); - // Used for testing. - options->addOptionChaining("out", "out", moe::String, - "output file; if not specified, stdout is used") - .hidden(); - - return Status::OK(); - } - - void printMongoShimHelp(std::ostream* out) { - *out << "Read/write directly to stored format.\n" << std::endl; - *out << moe::startupOptions.helpString(); - *out << std::flush; - } - - bool handlePreValidationMongoShimOptions(const moe::Environment& params) { - if (!handlePreValidationGeneralToolOptions(params)) { - return false; - } - if (params.count("help")) { - printMongoShimHelp(&std::cout); - return false; - } - return true; - } - - Status storeMongoShimOptions(const moe::Environment& params, - const std::vector<std::string>& args) { - - Status ret = storeGeneralToolOptions(params, args); - if (!ret.isOK()) { - return ret; - } - - if (!toolGlobalParams.useDirectClient) { - return Status(ErrorCodes::BadValue, - "MongoShim requires a --dbpath value to proceed"); - } - - ret = storeFieldOptions(params, args); - if (!ret.isOK()) { - return ret; - } - - ret = storeBSONToolOptions(params, args); - if (!ret.isOK()) { - return ret; - } - - // Process shim mode. - if (params.count("mode") == 0) { - return Status(ErrorCodes::BadValue, - "MongoShim requires a --mode value to proceed"); - } - - const string modeParam = getParam("mode"); - for (int i = 0; i <= int(ShimMode::kNumShimModes); ++i) { - ShimMode mode = static_cast<ShimMode::Value>(i); - if (modeParam == mode.toString()) { - mongoShimGlobalParams.mode = mode; - break; - } - } - // --mode value is enforced by format string in option description. - invariant(mongoShimGlobalParams.mode != ShimMode::kNumShimModes); - - mongoShimGlobalParams.drop = params.count("drop") > 0; - - // Ensure that collection is specified when mode is not "command". - // Tool::getNs() validates --collection but error - // is not propagated to calling process because Tool::main() - // always exits cleanly when --dbpath is enabled. - if (toolGlobalParams.coll.size() == 0 && - !(mongoShimGlobalParams.mode == ShimMode::kCommand)) { - return Status(ErrorCodes::BadValue, - "MongoShim requires a --collection value to proceed when not running " - "in \"command\" mode"); - } - // --drop and --collection are not allowed in "command" mode. - if (mongoShimGlobalParams.mode == ShimMode::kCommand) { - if (!toolGlobalParams.coll.empty()) { - return Status(ErrorCodes::BadValue, - "--collection is not allowed in \"command\" mode"); - } - if (mongoShimGlobalParams.drop) { - return Status(ErrorCodes::BadValue, - "--drop is not allowed in \"command\" mode"); - } - } - - // Parse upsert fields. - // --upsertFields is illegal when not in "upsert" mode. - if (params.count("upsertFields") > 0) { - if (mongoShimGlobalParams.mode != ShimMode::kUpsert) { - return Status(ErrorCodes::BadValue, - "--upsertFields is not allowed when not in \"upsert\" mode"); - } - string uf = getParam("upsertFields"); - if (uf.empty()) { - mongoShimGlobalParams.upsertFields.push_back("_id"); - } - else { - StringSplitter(uf.c_str(), ",").split(mongoShimGlobalParams.upsertFields); - } - } - - // Used for testing. In normal operation, results will be written to stdout. - mongoShimGlobalParams.outputFile = getParam("out"); - mongoShimGlobalParams.outputFileSpecified = params.count("out") > 0; - - // Used for testing. In normal operation, documents will be read from stdin. - mongoShimGlobalParams.inputFile = getParam("in"); - mongoShimGlobalParams.inputFileSpecified = params.count("in") > 0; - - // Used for testing. In normal operation, documents will be read from stdin. - mongoShimGlobalParams.inputDocuments = mongo::fromjson(getParam("inputDocuments", "{}")); - - mongoShimGlobalParams.query = getParam("query", ""); - - mongoShimGlobalParams.limit = getParam("limit", 0); - mongoShimGlobalParams.skip = getParam("skip", 0); - mongoShimGlobalParams.sort = getParam("sort", ""); - - toolGlobalParams.canUseStdout = false; - - return Status::OK(); - } - -} diff --git a/src/mongo/tools/mongoshim_options.h b/src/mongo/tools/mongoshim_options.h deleted file mode 100644 index 5c1c9a12c62..00000000000 --- a/src/mongo/tools/mongoshim_options.h +++ /dev/null @@ -1,92 +0,0 @@ -// mongoshim_options.h - -/* - * Copyright (C) 2014 MongoDB Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#pragma once - -#include <iosfwd> -#include <string> -#include <vector> - -#include "mongo/base/status.h" -#include "mongo/bson/bsonobj.h" -#include "mongo/tools/shim_mode.h" -#include "mongo/tools/tool_options.h" - -namespace mongo { - - struct MongoShimGlobalParams { - - MongoShimGlobalParams() : mode(ShimMode::kNumShimModes) { } - - // Mongoshim can run in one mode per invocation. - // See shim_mode.h for list of supported modes. - ShimMode mode; - bool drop; - - std::vector<std::string> upsertFields; - - std::string query; - - // If --in is specified, reads documents from 'outputFile' instead of stdin. - // Used primarily for testing. - std::string inputFile; - bool inputFileSpecified; - - // If --inputDocuments is specified, documents will be parsed from option value - // instead of being read from stdin. - // Used primarily for testing. - BSONObj inputDocuments; - - // If --out is specified, writes output to 'outputFile' instead of stdout. - // Used primarily for testing. - std::string outputFile; - bool outputFileSpecified; - - unsigned int skip; - unsigned int limit; - std::string sort; - }; - - extern MongoShimGlobalParams mongoShimGlobalParams; - - Status addMongoShimOptions(moe::OptionSection* options); - - void printMongoShimHelp(std::ostream* out); - - /** - * Handle options that should come before validation, such as "help". - * - * Returns false if an option was found that implies we should prematurely exit with success. - */ - bool handlePreValidationMongoShimOptions(const moe::Environment& params); - - Status storeMongoShimOptions(const moe::Environment& params, - const std::vector<std::string>& args); -} diff --git a/src/mongo/tools/mongoshim_options_init.cpp b/src/mongo/tools/mongoshim_options_init.cpp deleted file mode 100644 index ee8fffb1780..00000000000 --- a/src/mongo/tools/mongoshim_options_init.cpp +++ /dev/null @@ -1,63 +0,0 @@ -// mongoshim_options_init.cpp - -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/tools/mongoshim_options.h" - -#include "mongo/util/options_parser/startup_option_init.h" -#include "mongo/util/options_parser/startup_options.h" -#include "mongo/util/quick_exit.h" - -namespace mongo { - MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(MongoShimOptions)(InitializerContext* context) { - return addMongoShimOptions(&moe::startupOptions); - } - - MONGO_STARTUP_OPTIONS_VALIDATE(MongoShimOptions)(InitializerContext* context) { - if (!handlePreValidationMongoShimOptions(moe::startupOptionsParsed)) { - quickExit(EXIT_SUCCESS); - } - Status ret = moe::startupOptionsParsed.validate(); - if (!ret.isOK()) { - return ret; - } - return Status::OK(); - } - - MONGO_STARTUP_OPTIONS_STORE(MongoShimOptions)(InitializerContext* context) { - Status ret = storeMongoShimOptions(moe::startupOptionsParsed, context->args()); - if (!ret.isOK()) { - std::cerr << ret.toString() << std::endl; - std::cerr << "try '" << context->args()[0] << " --help' for more information" - << std::endl; - quickExit(EXIT_BADOPTIONS); - } - return Status::OK(); - } -} diff --git a/src/mongo/tools/mongostat_options.cpp b/src/mongo/tools/mongostat_options.cpp deleted file mode 100644 index 9376d5fe961..00000000000 --- a/src/mongo/tools/mongostat_options.cpp +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/tools/mongostat_options.h" - -#include "mongo/base/status.h" -#include "mongo/util/options_parser/startup_options.h" - -namespace mongo { - - MongoStatGlobalParams mongoStatGlobalParams; - - Status addMongoStatOptions(moe::OptionSection* options) { - Status ret = addGeneralToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addRemoteServerToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - options->addOptionChaining("noheaders", "noheaders", moe::Switch, - "don't output column names"); - - options->addOptionChaining("rowcount", "rowcount,n", moe::Int, - "number of stats lines to print (0 for indefinite)") - .setDefault(moe::Value(0)); - - options->addOptionChaining("http", "http", moe::Switch, - "use http instead of raw db connection"); - - options->addOptionChaining("discover", "discover", moe::Switch, - "discover nodes and display stats for all"); - - options->addOptionChaining("all", "all", moe::Switch, "all optional fields"); - - options->addOptionChaining("sleep", "sleep", moe::Int, "seconds to sleep between samples") - .hidden() - .setSources(moe::SourceCommandLine) - .positional(1, 1); - - - return Status::OK(); - } - - void printMongoStatHelp(std::ostream* out) { - *out << "View live MongoDB performance statistics.\n" << std::endl; - *out << "usage: mongostat [options] [sleep time]" << std::endl; - *out << "sleep time: time to wait (in seconds) between calls" << std::endl; - *out << moe::startupOptions.helpString(); - *out << "\n"; - *out << " Fields\n"; - *out << " inserts \t- # of inserts per second (* means replicated op)\n"; - *out << " query \t- # of queries per second\n"; - *out << " update \t- # of updates per second\n"; - *out << " delete \t- # of deletes per second\n"; - *out << " getmore \t- # of get mores (cursor batch) per second\n"; - *out << " command \t- # of commands per second, on a slave its local|replicated\n"; - *out << " flushes \t- # of fsync flushes per second\n"; - *out << " mapped \t- amount of data mmapped (total data size) in megabytes\n"; - *out << " vsize \t- virtual size of process in megabytes\n"; - *out << " res \t- resident size of process in megabytes\n"; - *out << " non-mapped \t- amount virtual memory less mapped memory in megabytes (only with --all)\n"; - *out << " faults \t- # of pages faults per sec\n"; - *out << " locked \t- name of and percent time for most locked database\n"; - *out << " idx miss \t- percent of btree page misses (sampled)\n"; - *out << " qr|qw \t- queue lengths for clients waiting (read|write)\n"; - *out << " ar|aw \t- active clients (read|write)\n"; - *out << " netIn \t- network traffic in - bytes\n"; - *out << " netOut \t- network traffic out - bytes\n"; - *out << " conn \t- number of open connections\n"; - *out << " set \t- replica set name\n"; - *out << " repl \t- replication type \n"; - *out << " \t PRI - primary (master)\n"; - *out << " \t SEC - secondary\n"; - *out << " \t REC - recovering\n"; - *out << " \t UNK - unknown\n"; - *out << " \t SLV - slave\n"; - *out << " \t RTR - mongos process (\"router\")\n"; - *out << std::flush; - } - - bool handlePreValidationMongoStatOptions(const moe::Environment& params) { - if (!handlePreValidationGeneralToolOptions(params)) { - return false; - } - if (params.count("help")) { - printMongoStatHelp(&std::cout); - return false; - } - return true; - } - - Status storeMongoStatOptions(const moe::Environment& params, - const std::vector<std::string>& args) { - Status ret = storeGeneralToolOptions(params, args); - if (!ret.isOK()) { - return ret; - } - - if (hasParam("http")) { - mongoStatGlobalParams.http = true; - toolGlobalParams.noconnection = true; - } - - if (hasParam("host") && getParam("host").find(',') != string::npos) { - toolGlobalParams.noconnection = true; - mongoStatGlobalParams.many = true; - } - - if (hasParam("discover")) { - mongoStatGlobalParams.discover = true; - mongoStatGlobalParams.many = true; - } - - mongoStatGlobalParams.showHeaders = !hasParam("noheaders"); - mongoStatGlobalParams.rowCount = getParam("rowcount", 0); - mongoStatGlobalParams.sleep = getParam("sleep", 1); - mongoStatGlobalParams.allFields = hasParam("all"); - - // Make the default db "admin" if it was not explicitly set - if (!params.count("db")) { - toolGlobalParams.db = "admin"; - } - - // end of storage / start of validation - - if (mongoStatGlobalParams.sleep <= 0) { - return Status(ErrorCodes::BadValue, - "Error parsing command line: --sleep must be greater than 0"); - } - - if (mongoStatGlobalParams.rowCount < 0) { - return Status(ErrorCodes::BadValue, - "Error parsing command line: --rowcount (-n) can't be negative"); - } - - return Status::OK(); - } - -} diff --git a/src/mongo/tools/mongostat_options.h b/src/mongo/tools/mongostat_options.h deleted file mode 100644 index 0787ed585d9..00000000000 --- a/src/mongo/tools/mongostat_options.h +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#pragma once - -#include <iosfwd> -#include <string> -#include <vector> - -#include "mongo/base/status.h" -#include "mongo/tools/tool_options.h" - -namespace mongo { - - struct MongoStatGlobalParams { - bool showHeaders; - int rowCount; - bool http; - bool discover; - bool many; - bool allFields; - int sleep; - std::string url; - }; - - extern MongoStatGlobalParams mongoStatGlobalParams; - - Status addMongoStatOptions(moe::OptionSection* options); - - void printMongoStatHelp(std::ostream* out); - - /** - * Handle options that should come before validation, such as "help". - * - * Returns false if an option was found that implies we should prematurely exit with success. - */ - bool handlePreValidationMongoStatOptions(const moe::Environment& params); - - Status storeMongoStatOptions(const moe::Environment& params, - const std::vector<std::string>& args); -} diff --git a/src/mongo/tools/mongostat_options_init.cpp b/src/mongo/tools/mongostat_options_init.cpp deleted file mode 100644 index 6209be01fff..00000000000 --- a/src/mongo/tools/mongostat_options_init.cpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/tools/mongostat_options.h" - -#include "mongo/util/options_parser/startup_option_init.h" -#include "mongo/util/options_parser/startup_options.h" -#include "mongo/util/quick_exit.h" - -namespace mongo { - MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(MongoStatOptions)(InitializerContext* context) { - return addMongoStatOptions(&moe::startupOptions); - } - - MONGO_STARTUP_OPTIONS_VALIDATE(MongoStatOptions)(InitializerContext* context) { - if (!handlePreValidationMongoStatOptions(moe::startupOptionsParsed)) { - quickExit(EXIT_SUCCESS); - } - Status ret = moe::startupOptionsParsed.validate(); - if (!ret.isOK()) { - return ret; - } - return Status::OK(); - } - - MONGO_STARTUP_OPTIONS_STORE(MongoStatOptions)(InitializerContext* context) { - Status ret = storeMongoStatOptions(moe::startupOptionsParsed, context->args()); - if (!ret.isOK()) { - std::cerr << ret.toString() << std::endl; - std::cerr << "try '" << context->args()[0] << " --help' for more information" - << std::endl; - quickExit(EXIT_BADOPTIONS); - } - return Status::OK(); - } -} diff --git a/src/mongo/tools/mongotop_options.cpp b/src/mongo/tools/mongotop_options.cpp deleted file mode 100644 index 4c128364d2f..00000000000 --- a/src/mongo/tools/mongotop_options.cpp +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/tools/mongotop_options.h" - -#include "mongo/base/status.h" -#include "mongo/util/options_parser/startup_options.h" - -namespace mongo { - - MongoTopGlobalParams mongoTopGlobalParams; - - Status addMongoTopOptions(moe::OptionSection* options) { - Status ret = addGeneralToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - ret = addRemoteServerToolOptions(options); - if (!ret.isOK()) { - return ret; - } - - options->addOptionChaining("locks", "locks", moe::Switch, - "use db lock info instead of top"); - - options->addOptionChaining("sleep", "sleep", moe::Int, "seconds to sleep between samples") - .hidden() - .setSources(moe::SourceCommandLine) - .positional(1, 1); - - - return Status::OK(); - } - - void printMongoTopHelp(std::ostream* out) { - *out << "View live MongoDB collection statistics.\n" << std::endl; - *out << moe::startupOptions.helpString(); - *out << std::flush; - } - - bool handlePreValidationMongoTopOptions(const moe::Environment& params) { - if (!handlePreValidationGeneralToolOptions(params)) { - return false; - } - if (params.count("help")) { - printMongoTopHelp(&std::cout); - return false; - } - return true; - } - - Status storeMongoTopOptions(const moe::Environment& params, - const std::vector<std::string>& args) { - Status ret = storeGeneralToolOptions(params, args); - if (!ret.isOK()) { - return ret; - } - - mongoTopGlobalParams.sleep = getParam("sleep", 1); - mongoTopGlobalParams.useLocks = hasParam("locks"); - - // Make the default db "admin" if it was not explicitly set - if (!params.count("db")) { - toolGlobalParams.db = "admin"; - } - - return Status::OK(); - } - -} diff --git a/src/mongo/tools/mongotop_options.h b/src/mongo/tools/mongotop_options.h deleted file mode 100644 index 5f9e6b1fb4f..00000000000 --- a/src/mongo/tools/mongotop_options.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#pragma once - -#include <iosfwd> -#include <string> -#include <vector> - -#include "mongo/base/status.h" -#include "mongo/tools/tool_options.h" - -namespace mongo { - - struct MongoTopGlobalParams { - bool useLocks; - int sleep; - }; - - extern MongoTopGlobalParams mongoTopGlobalParams; - - Status addMongoTopOptions(moe::OptionSection* options); - - void printMongoTopHelp(std::ostream* out); - - /** - * Handle options that should come before validation, such as "help". - * - * Returns false if an option was found that implies we should prematurely exit with success. - */ - bool handlePreValidationMongoTopOptions(const moe::Environment& params); - - Status storeMongoTopOptions(const moe::Environment& params, - const std::vector<std::string>& args); -} diff --git a/src/mongo/tools/mongotop_options_init.cpp b/src/mongo/tools/mongotop_options_init.cpp deleted file mode 100644 index c52a531b094..00000000000 --- a/src/mongo/tools/mongotop_options_init.cpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/tools/mongotop_options.h" - -#include "mongo/util/options_parser/startup_option_init.h" -#include "mongo/util/options_parser/startup_options.h" -#include "mongo/util/quick_exit.h" - -namespace mongo { - MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(MongoTopOptions)(InitializerContext* context) { - return addMongoTopOptions(&moe::startupOptions); - } - - MONGO_STARTUP_OPTIONS_VALIDATE(MongoTopOptions)(InitializerContext* context) { - if (!handlePreValidationMongoTopOptions(moe::startupOptionsParsed)) { - quickExit(EXIT_SUCCESS); - } - Status ret = moe::startupOptionsParsed.validate(); - if (!ret.isOK()) { - return ret; - } - return Status::OK(); - } - - MONGO_STARTUP_OPTIONS_STORE(MongoTopOptions)(InitializerContext* context) { - Status ret = storeMongoTopOptions(moe::startupOptionsParsed, context->args()); - if (!ret.isOK()) { - std::cerr << ret.toString() << std::endl; - std::cerr << "try '" << context->args()[0] << " --help' for more information" - << std::endl; - quickExit(EXIT_BADOPTIONS); - } - return Status::OK(); - } -} diff --git a/src/mongo/tools/oplog.cpp b/src/mongo/tools/oplog.cpp deleted file mode 100644 index fe8e4726fa1..00000000000 --- a/src/mongo/tools/oplog.cpp +++ /dev/null @@ -1,116 +0,0 @@ -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see <http://www.gnu.org/licenses/>. -* -* As a special exception, the copyright holders give permission to link the -* code of portions of this program with the OpenSSL library under certain -* conditions as described in each individual source file and distribute -* linked combinations including the program with the OpenSSL library. You -* must comply with the GNU Affero General Public License in all respects -* for all of the code used other than as permitted herein. If you modify -* file(s) with this exception, you may extend this exception to your -* version of the file(s), but you are not obligated to do so. If you do not -* wish to do so, delete this exception statement from your version. If you -* delete this exception statement from all source files in the program, -* then also delete it in the license file. -*/ - -#include "mongo/pch.h" - -#include <fstream> -#include <iostream> - -#include "mongo/db/json.h" -#include "mongo/db/repl/oplogreader.h" -#include "mongo/tools/mongooplog_options.h" -#include "mongo/tools/tool.h" -#include "mongo/util/options_parser/option_section.h" - -using namespace mongo; - -class OplogTool : public Tool { -public: - OplogTool() : Tool() { } - - virtual void printHelp( ostream & out ) { - printMongoOplogHelp(&out); - } - - int run() { - - Client::initThread( "oplogreplay" ); - - toolInfoLog() << "going to connect" << std::endl; - - repl::OplogReader r; - r.setTailingQueryOptions( QueryOption_SlaveOk | QueryOption_AwaitData ); - - bool connected = r.connect(HostAndPort(mongoOplogGlobalParams.from)); - - if (!connected) - { - toolInfoLog() << "unable to connect to " << mongoOplogGlobalParams.from << std::endl; - return -1; - } - - toolInfoLog() << "connected" << std::endl; - - OpTime start(time(0) - mongoOplogGlobalParams.seconds, 0); - toolInfoLog() << "starting from " << start.toStringPretty() << std::endl; - - r.tailingQueryGTE(mongoOplogGlobalParams.ns.c_str(), start); - - int num = 0; - while ( r.more() ) { - BSONObj o = r.next(); - if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(2))) { - toolInfoLog() << o << std::endl; - } - - if ( o["$err"].type() ) { - toolError() << "error getting oplog" << std::endl; - toolError() << o << std::endl; - return -1; - } - - - bool print = ++num % 100000 == 0; - if (print) { - toolInfoLog() << num << "\t" << o << std::endl; - } - - if ( o["op"].String() == "n" ) - continue; - - BSONObjBuilder b( o.objsize() + 32 ); - BSONArrayBuilder updates( b.subarrayStart( "applyOps" ) ); - updates.append( o ); - updates.done(); - - BSONObj c = b.obj(); - - BSONObj res; - bool ok = conn().runCommand( "admin" , c , res ); - if (!ok) { - toolError() << res << std::endl; - } else if (print) { - toolInfoLog() << res << std::endl; - } - } - - return 0; - } -}; - -REGISTER_MONGO_TOOL(OplogTool); diff --git a/src/mongo/tools/restore.cpp b/src/mongo/tools/restore.cpp deleted file mode 100644 index ad4998dea74..00000000000 --- a/src/mongo/tools/restore.cpp +++ /dev/null @@ -1,920 +0,0 @@ -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see <http://www.gnu.org/licenses/>. -* -* As a special exception, the copyright holders give permission to link the -* code of portions of this program with the OpenSSL library under certain -* conditions as described in each individual source file and distribute -* linked combinations including the program with the OpenSSL library. You -* must comply with the GNU Affero General Public License in all respects -* for all of the code used other than as permitted herein. If you modify -* file(s) with this exception, you may extend this exception to your -* version of the file(s), but you are not obligated to do so. If you do not -* wish to do so, delete this exception statement from your version. If you -* delete this exception statement from all source files in the program, -* then also delete it in the license file. -*/ - -#include "mongo/platform/basic.h" - -#include <boost/filesystem/convenience.hpp> -#include <boost/filesystem/operations.hpp> -#include <boost/lexical_cast.hpp> -#include <boost/scoped_ptr.hpp> -#include <fcntl.h> -#include <fstream> -#include <set> - -#include "mongo/base/init.h" -#include "mongo/bson/util/bson_extract.h" -#include "mongo/client/auth_helpers.h" -#include "mongo/client/dbclientcursor.h" -#include "mongo/db/auth/authorization_manager.h" -#include "mongo/db/auth/authorization_manager_global.h" -#include "mongo/db/auth/authz_manager_external_state_d.h" -#include "mongo/db/auth/user_name.h" -#include "mongo/db/auth/role_name.h" -#include "mongo/db/json.h" -#include "mongo/db/namespace_string.h" -#include "mongo/tools/mongorestore_options.h" -#include "mongo/tools/tool.h" -#include "mongo/util/options_parser/option_section.h" -#include "mongo/util/stringutils.h" - -using namespace mongo; - -namespace { - const char* OPLOG_SENTINEL = "$oplog"; // compare by ptr not strcmp -} - -MONGO_INITIALIZER_WITH_PREREQUISITES(RestoreAuthExternalState, ("ToolMocks"))( - InitializerContext* context) { - // Give restore the mongod implementation of AuthorizationManager so that it can run - // the _mergeAuthzCollections command directly against the data files - clearGlobalAuthorizationManager(); - setGlobalAuthorizationManager(new AuthorizationManager( - new AuthzManagerExternalStateMongod())); - - return Status::OK(); -} - -class Restore : public BSONTool { -public: - - string _curns; - string _curdb; - string _curcoll; - string _serverBinVersion; // Version identifier of the server we're restoring to - set<UserName> _users; // Holds users that are already in the cluster when restoring with --drop - set<RoleName> _roles; // Holds roles that are already in the cluster when restoring with --drop - scoped_ptr<Matcher> _opmatcher; // For oplog replay - scoped_ptr<OpTime> _oplogLimitTS; // for oplog replay (limit) - int _oplogEntrySkips; // oplog entries skipped - int _oplogEntryApplies; // oplog entries applied - int _serverAuthzVersion; // authSchemaVersion of the cluster being restored into. - int _dumpFileAuthzVersion; // version extracted from admin.system.version file in dump. - bool _serverAuthzVersionDocExists; // Whether the remote cluster has an admin.system.version doc - Restore() : BSONTool(), _oplogEntrySkips(0), _oplogEntryApplies(0), _serverAuthzVersion(0), - _dumpFileAuthzVersion(0), _serverAuthzVersionDocExists(false) { } - - virtual void printHelp(ostream& out) { - printMongoRestoreHelp(&out); - } - - void storeRemoteAuthzVersion() { - Status status = auth::getRemoteStoredAuthorizationVersion(&conn(), - &_serverAuthzVersion); - uassertStatusOK(status); - uassert(17370, - mongoutils::str::stream() << "Restoring users and roles is only supported for " - "clusters with auth schema versions " << - AuthorizationManager::schemaVersion24 << ", " << - AuthorizationManager::schemaVersion26Final << " or " << - AuthorizationManager::schemaVersion28SCRAM << ", found: " << - _serverAuthzVersion, - _serverAuthzVersion == AuthorizationManager::schemaVersion24 || - _serverAuthzVersion == AuthorizationManager::schemaVersion26Final || - _serverAuthzVersion == AuthorizationManager::schemaVersion28SCRAM); - - _serverAuthzVersionDocExists = !conn().findOne( - AuthorizationManager::versionCollectionNamespace, - AuthorizationManager::versionDocumentQuery).isEmpty(); - } - - virtual int doRun() { - - boost::filesystem::path root = mongoRestoreGlobalParams.restoreDirectory; - - // check if we're actually talking to a machine that can write - if (!isMaster()) { - return -1; - } - - if (isMongos() && toolGlobalParams.db == "" && exists(root / "config")) { - toolError() << "Cannot do a full restore on a sharded system" << std::endl; - return -1; - } - - { - // Store server's version - BSONObj out; - if (! conn().simpleCommand("admin", &out, "buildinfo")) { - toolError() << "buildinfo command failed: " - << out["errmsg"].String() << std::endl; - return -1; - } - - _serverBinVersion = out["version"].String(); - } - - if (mongoRestoreGlobalParams.restoreUsersAndRoles) { - storeRemoteAuthzVersion(); // populate _serverAuthzVersion - - if (_serverAuthzVersion == AuthorizationManager::schemaVersion26Final) { - uassert(17410, - mongoutils::str::stream() << mongoRestoreGlobalParams.tempUsersColl << - " collection already exists, but is needed to restore user data. " - "Drop this collection or specify a different collection (via " - "--tempUsersColl) to use to temporarily hold user data during the " - "restore process", - !conn().exists(mongoRestoreGlobalParams.tempUsersColl)); - uassert(17411, - mongoutils::str::stream() << mongoRestoreGlobalParams.tempRolesColl << - " collection already exists, but is needed to restore role data. " - "Drop this collection or specify a different collection (via " - "--tempRolesColl) to use to temporarily hold role data during the " - "restore process", - !conn().exists(mongoRestoreGlobalParams.tempRolesColl)); - } - - if (toolGlobalParams.db.empty() && toolGlobalParams.coll.empty() && - exists(root / "admin" / "system.version.bson")) { - // Will populate _dumpFileAuthzVersion - processFileAndMetadata(root / "admin" / "system.version.bson", - "admin.system.version"); - } else if (!toolGlobalParams.db.empty()) { - // DB-specific restore - if (exists(root / "$admin.system.users.bson")) { - uassert(17372, - mongoutils::str::stream() << "$admin.system.users.bson file found, " - "which implies that the dump was taken from a system with " - "schema version " << AuthorizationManager::schemaVersion26Final - << " users, but server has authorization schema version " - << _serverAuthzVersion, - _serverAuthzVersion == AuthorizationManager::schemaVersion26Final); - toolInfoLog() << "Restoring users for the " << toolGlobalParams.db << - " database to admin.system.users" << endl; - processFileAndMetadata(root / "$admin.system.users.bson", "admin.system.users"); - } - if (exists(root / "$admin.system.roles.bson")) { - uassert(17373, - mongoutils::str::stream() << "$admin.system.roles.bson file found, " - "which implies that the dump was taken from a system with " - "schema version " << AuthorizationManager::schemaVersion26Final - << " authorization data, but server has authorization schema " - "version " << _serverAuthzVersion, - _serverAuthzVersion == AuthorizationManager::schemaVersion26Final); - toolInfoLog() << "Restoring roles for the " << toolGlobalParams.db << - " database to admin.system.roles" << endl; - processFileAndMetadata(root / "$admin.system.roles.bson", "admin.system.roles"); - } - } - } - - if (mongoRestoreGlobalParams.oplogReplay) { - // fail early if errors - - if (toolGlobalParams.db != "") { - toolError() << "Can only replay oplog on full restore" << std::endl; - return -1; - } - - if ( ! exists(root / "oplog.bson") ) { - toolError() << "No oplog file to replay. Make sure you run mongodump with --oplog." - << std::endl; - return -1; - } - - if (versionCmp(_serverBinVersion, "1.7.4-pre-") < 0) { - toolError() << "Can only replay oplog to server version >= 1.7.4" << std::endl; - return -1; - } - - string oplogInc = "0"; - - if(!mongoRestoreGlobalParams.oplogLimit.empty()) { - size_t i = mongoRestoreGlobalParams.oplogLimit.find_first_of(':'); - if ( i != string::npos ) { - if (i + 1 < mongoRestoreGlobalParams.oplogLimit.length()) { - oplogInc = mongoRestoreGlobalParams.oplogLimit.substr(i + 1); - } - - mongoRestoreGlobalParams.oplogLimit = - mongoRestoreGlobalParams.oplogLimit.substr(0, i); - } - - try { - _oplogLimitTS.reset(new OpTime( - boost::lexical_cast<unsigned long>( - mongoRestoreGlobalParams.oplogLimit.c_str()), - boost::lexical_cast<unsigned long>(oplogInc.c_str()))); - } catch( const boost::bad_lexical_cast& ) { - toolError() << "Could not parse oplogLimit into Timestamp from values ( " - << mongoRestoreGlobalParams.oplogLimit << " , " << oplogInc << " )" - << std::endl; - return -1; - } - - if (!mongoRestoreGlobalParams.oplogLimit.empty()) { - // Only for a replica set as master will have no-op entries so we would need to - // skip them all to find the real op - scoped_ptr<DBClientCursor> cursor( - conn().query("local.oplog.rs", Query().sort(BSON("$natural" << -1)), - 1 /*return first*/)); - OpTime tsOptime; - // get newest oplog entry and make sure it is older than the limit to apply. - if (cursor->more()) { - tsOptime = cursor->next().getField("ts")._opTime(); - if (tsOptime > *_oplogLimitTS.get()) { - toolError() << "The oplogLimit is not newer than" - << " the last oplog entry on the server." - << std::endl; - return -1; - } - } - - BSONObjBuilder tsRestrictBldr; - if (!tsOptime.isNull()) - tsRestrictBldr << "$gt" << tsOptime; - tsRestrictBldr << "$lt" << *_oplogLimitTS.get(); - - BSONObj query = BSON("ts" << tsRestrictBldr.obj()); - - if (!tsOptime.isNull()) { - toolInfoLog() << "Latest oplog entry on the server is " << tsOptime.getSecs() - << ":" << tsOptime.getInc() << std::endl; - toolInfoLog() << "Only applying oplog entries matching this criteria: " - << query.jsonString() << std::endl; - } - _opmatcher.reset(new Matcher(query, - MatchExpressionParser::WhereCallback())); - } - } - } - - /* If toolGlobalParams.db is not "" then the user specified a db name to restore as. - * - * In that case we better be given either a root directory that - * contains only .bson files or a single .bson file (a db). - * - * In the case where a collection name is specified we better be - * given either a root directory that contains only a single - * .bson file, or a single .bson file itself (a collection). - */ - drillDown(root, toolGlobalParams.db != "", toolGlobalParams.coll != "", - !(_oplogLimitTS.get() == NULL), true); - - // should this happen for oplog replay as well? - string err = conn().getLastError(toolGlobalParams.db == "" ? "admin" : toolGlobalParams.db); - if (!err.empty()) { - toolError() << err << std::endl; - } - - if (mongoRestoreGlobalParams.oplogReplay) { - toolInfoLog() << "\t Replaying oplog" << std::endl; - _curns = OPLOG_SENTINEL; - processFile(root / "oplog.bson", NULL ); - toolInfoLog() << "Applied " << _oplogEntryApplies << " oplog entries out of " - << _oplogEntryApplies + _oplogEntrySkips << " (" << _oplogEntrySkips - << " skipped)." << std::endl; - } - - return EXIT_CLEAN; - } - - void drillDown( boost::filesystem::path root, - bool use_db, - bool use_coll, - bool oplogReplayLimit, - bool top_level=false) { - bool json_metadata = false; - if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(2))) { - toolInfoLog() << "drillDown: " << root.string() << std::endl; - } - - // skip hidden files and directories - if (root.leaf().string()[0] == '.' && root.leaf().string() != ".") - return; - - if ( is_directory( root ) ) { - boost::filesystem::directory_iterator end; - boost::filesystem::directory_iterator i(root); - boost::filesystem::path indexes; - while ( i != end ) { - boost::filesystem::path p = *i; - i++; - - if (use_db) { - if (boost::filesystem::is_directory(p)) { - toolError() << "ERROR: root directory must be a dump of a single database" - << std::endl; - toolError() << " when specifying a db name with --db" << std::endl; - toolError() << " use the --help option for more information" - << std::endl; - return; - } - } - - if (use_coll) { - if (boost::filesystem::is_directory(p) || i != end) { - toolError() << "ERROR: root directory must be a dump of a single collection" - << std::endl; - toolError() << " when specifying a collection name with --collection" - << std::endl; - toolError() << " use the --help option for more information" - << std::endl; - return; - } - } - - // Ignore system.indexes.bson if we have *.metadata.json files - if ( endsWith( p.string().c_str() , ".metadata.json" ) ) { - json_metadata = true; - } - - // don't insert oplog - if (top_level && !use_db && p.leaf() == "oplog.bson") - continue; - - if ( p.leaf() == "system.indexes.bson" ) { - indexes = p; - } else { - drillDown(p, use_db, use_coll, oplogReplayLimit); - } - } - - if (!indexes.empty() && !json_metadata) { - drillDown(indexes, use_db, use_coll, oplogReplayLimit); - } - - return; - } - - if (oplogReplayLimit) { - toolError() << "The oplogLimit option cannot be used if " - << "normal databases/collections exist in the dump directory." - << std::endl; - exit(EXIT_FAILURE); - } - - string ns; - if (use_db) { - ns += toolGlobalParams.db; - } - else { - ns = root.parent_path().filename().string(); - if (ns.empty()) - ns = "test"; - } - - verify( ns.size() ); - - string oldCollName = root.leaf().string(); // Name of the collection that was dumped from - oldCollName = oldCollName.substr( 0 , oldCollName.find_last_of( "." ) ); - if (use_coll) { - ns += "." + toolGlobalParams.coll; - } - else { - ns += "." + oldCollName; - } - - if ( endsWith( root.string().c_str() , ".metadata.json" ) ) { - // Metadata files are handled when the corresponding .bson file is handled - return; - } - - if ((root.leaf() == "system.version.bson" && toolGlobalParams.db.empty()) || - root.leaf() == "$admin.system.users.bson" || - root.leaf() == "$admin.system.roles.bson") { - // These files were already explicitly handled at the beginning of the restore. - return; - } - - boost::filesystem::file_status fileStatus = boost::filesystem::status(root); - if ( ! ( endsWith( root.string().c_str() , ".bson" ) || - endsWith( root.string().c_str() , ".bin" ) ) && - ! ( root.string() == "-" ) && - ! ( fileStatus.type() == boost::filesystem::fifo_file ) ) { - toolError() << "don't know what to do with file [" << root.string() << "]" << std::endl; - return; - } - - // Both --db and --collection have to be provided when using stdin or fifo. - if ((root.string() == "-" || fileStatus.type() == boost::filesystem::fifo_file) && - !(use_db && use_coll)) { - toolError() << "Both --db and --collection have to be provided when using stding/fifo"; - exit(EXIT_FAILURE); - } - - toolInfoLog() << root.string() << std::endl; - - if ( root.leaf() == "system.profile.bson" ) { - toolInfoLog() << "\t skipping system.profile.bson" << std::endl; - return; - } - - processFileAndMetadata(root, ns); - } - - static std::string getMessageAboutBrokenDBUserRestore(const StringData& serverBinVersion) { - return str::stream() << "Running mongorestore with --drop, --db, and " - "--restoreDbUsersAndRoles flags has erroneous behavior when the target server " - "version is between 2.6.0 and 2.6.3 (see SERVER-14212). Detected server version " << - serverBinVersion << ". Aborting."; - } - - /** - * 1) Drop collection if --drop was specified. For system.users or system.roles collections, - * however, you don't want to remove all the users/roles up front as some of them may be needed - * by the restore. Instead, keep a set of all the users/roles originally in the server, then - * after restoring the users/roles from the dump, remove any users roles that were present in - * the system originally but aren't in the dump. - * - * 2) Parse metadata file (if present) and if the collection doesn't exist (or was just dropped - * b/c we're using --drop), create the collection with the options from the metadata file - * - * 3) Restore the data from the dump file for this collection - * - * 4) If the user asked to drop this collection, then at this point the _users and _roles sets - * will contain users and roles that were in the collection but not in the dump we are - * restoring. Iterate these sets and delete any users and roles that are there. - * - * 5) Restore indexes based on index definitions from the metadata file. - */ - void processFileAndMetadata(const boost::filesystem::path& root, const std::string& ns) { - - _curns = ns; - _curdb = nsToDatabase(_curns); - _curcoll = nsToCollectionSubstring(_curns).toString(); - - toolInfoLog() << "\tgoing into namespace [" << _curns << "]" << std::endl; - - // 1) Drop collection if needed. Save user and role data if this is a system.users or - // system.roles collection - if (mongoRestoreGlobalParams.drop) { - if (_curcoll == "system.users") { - if (_serverAuthzVersion == AuthorizationManager::schemaVersion24 || - _curdb != "admin") { - // Restoring 2.4-style user docs so can't use the _mergeAuthzCollections command - // Create map of the users currently in the DB so the ones that don't show up in - // the dump file can be removed later. - BSONObj fields = BSON("user" << 1 << "userSource" << 1); - scoped_ptr<DBClientCursor> cursor(conn().query(_curns, Query(), 0, 0, &fields)); - while (cursor->more()) { - BSONObj user = cursor->next(); - string userDB; - uassertStatusOK(bsonExtractStringFieldWithDefault(user, - "userSource", - _curdb, - &userDB)); - _users.insert(UserName(user["user"].String(), userDB)); - } - } - } - else if (!startsWith(_curcoll, "system.")) { // Can't drop system collections - toolInfoLog() << "\t dropping" << std::endl; - conn().dropCollection( ns ); - } - } else { - // If drop is not used, warn if the collection exists. - scoped_ptr<DBClientCursor> cursor(conn().query(_curdb + ".system.namespaces", - Query(BSON("name" << ns)))); - if (cursor->more()) { - // collection already exists show warning - toolError() << "Restoring to " << ns << " without dropping. Restored data " - << "will be inserted without raising errors; check your server log" - << std::endl; - } - } - - // 2) Create collection with options from metadata file if present - BSONObj metadataObject; - boost::filesystem::file_status fileStatus = boost::filesystem::status(root); - if (fileStatus.type() != boost::filesystem::fifo_file && - root.string() != "-" && - (mongoRestoreGlobalParams.restoreOptions || mongoRestoreGlobalParams.restoreIndexes)) { - string oldCollName = root.leaf().string(); // Name of collection that was dumped from - oldCollName = oldCollName.substr( 0 , oldCollName.find_last_of( "." ) ); - boost::filesystem::path metadataFile = (root.branch_path() / (oldCollName + ".metadata.json")); - if (!boost::filesystem::exists(metadataFile.string())) { - // This is fine because dumps from before 2.1 won't have a metadata file, just print a warning. - // System collections shouldn't have metadata so don't warn if that file is missing. - if (!startsWith(metadataFile.leaf().string(), "system.")) { - toolInfoLog() << metadataFile.string() << " not found. Skipping." << std::endl; - } - } else { - metadataObject = parseMetadataFile(metadataFile.string()); - } - } - - if (mongoRestoreGlobalParams.restoreOptions && metadataObject.hasField("options")) { - // Try to create collection with given options - createCollectionWithOptions(metadataObject["options"].Obj()); - } - - // 3) Actually restore the BSONObjs inside the dump file - processFile(root, NULL); - - // 4) If running with --drop, remove any users/roles that were in the system at the - // beginning of the restore but weren't found in the dump file - if (_curcoll == "system.users") { - if ((_serverAuthzVersion == AuthorizationManager::schemaVersion24 || - _curdb != "admin")) { - // Restoring 2.4 style user docs so don't use the _mergeAuthzCollections command - if (mongoRestoreGlobalParams.drop) { - // Delete any users that used to exist but weren't in the dump file - for (set<UserName>::iterator it = _users.begin(); it != _users.end(); ++it) { - const UserName& name = *it; - BSONObjBuilder queryBuilder; - queryBuilder << "user" << name.getUser(); - if (name.getDB() == _curdb) { - // userSource field won't be present for v1 users docs in the same db as - // the user is defined on. - queryBuilder << "userSource" << BSONNULL; - } else { - queryBuilder << "userSource" << name.getDB(); - } - conn().remove(_curns, Query(queryBuilder.done())); - } - _users.clear(); - } - } else { - // Use _mergeAuthzCollections command to move into admin.system.users the user - // docs that were restored into the temp user collection - BSONObjBuilder cmdBuilder; - cmdBuilder.append("_mergeAuthzCollections", 1); - cmdBuilder.append("tempUsersCollection", mongoRestoreGlobalParams.tempUsersColl); - cmdBuilder.append("drop", mongoRestoreGlobalParams.drop); - cmdBuilder.append("writeConcern", BSON("w" << mongoRestoreGlobalParams.w)); - if (versionCmp(_serverBinVersion, "2.6.4") < 0) { - uassert(18528, - getMessageAboutBrokenDBUserRestore(_serverBinVersion), - !mongoRestoreGlobalParams.drop || - toolGlobalParams.db.empty() || - toolGlobalParams.db == "admin"); - } else { - // If we're doing a db-specific restore of the "admin" db, we want user data for - // *all* databases restored, not just the admin db, so we pass "" as the "db" - // param to _mergeAuthzCollections - cmdBuilder.append("db", - toolGlobalParams.db == "admin" ? "" : toolGlobalParams.db); - } - - BSONObj res; - conn().runCommand("admin", cmdBuilder.done(), res); - uassert(17412, - mongoutils::str::stream() << "Cannot restore users because the " - "_mergeAuthzCollections command failed: " << res.toString(), - res["ok"].trueValue()); - - conn().dropCollection(mongoRestoreGlobalParams.tempUsersColl); - } - } - if (_curns == "admin.system.roles") { - // Use _mergeAuthzCollections command to move into admin.system.roles the role - // docs that were restored into the temp roles collection - BSONObjBuilder cmdBuilder; - cmdBuilder.append("_mergeAuthzCollections", 1); - cmdBuilder.append("tempRolesCollection", mongoRestoreGlobalParams.tempRolesColl); - cmdBuilder.append("drop", mongoRestoreGlobalParams.drop); - cmdBuilder.append("writeConcern", BSON("w" << mongoRestoreGlobalParams.w)); - if (versionCmp(_serverBinVersion, "2.6.4") < 0) { - uassert(18529, - getMessageAboutBrokenDBUserRestore(_serverBinVersion), - !mongoRestoreGlobalParams.drop || - toolGlobalParams.db.empty() || - toolGlobalParams.db == "admin"); - } else { - // If we're doing a db-specific restore of the "admin" db, we want role data for - // *all* databases restored, not just the admin db, so we pass "" as the "db" - // param to _mergeAuthzCollections - cmdBuilder.append("db", toolGlobalParams.db == "admin" ? "" : toolGlobalParams.db); - } - - BSONObj res; - conn().runCommand("admin", cmdBuilder.done(), res); - uassert(17413, - mongoutils::str::stream() << "Cannot restore roles because the " - "_mergeAuthzCollections command failed: " << res.toString(), - res["ok"].trueValue()); - - conn().dropCollection(mongoRestoreGlobalParams.tempRolesColl); - } - - // 5) Restore indexes - if (mongoRestoreGlobalParams.restoreIndexes && metadataObject.hasField("indexes")) { - vector<BSONElement> indexes = metadataObject["indexes"].Array(); - for (vector<BSONElement>::iterator it = indexes.begin(); it != indexes.end(); ++it) { - createIndex((*it).Obj(), false); - } - } - } - - virtual void gotObject(const BSONObj& obj, std::ostream* out) { - if (_curns == OPLOG_SENTINEL) { // intentional ptr compare - if (obj["op"].valuestr()[0] == 'n') // skip no-ops - return; - - // exclude operations that don't meet (timestamp) criteria - if ( _opmatcher.get() && ! _opmatcher->matches ( obj ) ) { - _oplogEntrySkips++; - return; - } - - string db = obj["ns"].valuestr(); - db = db.substr(0, db.find('.')); - - BSONObj cmd = BSON( "applyOps" << BSON_ARRAY( obj ) ); - BSONObj out; - conn().runCommand(db, cmd, out); - _oplogEntryApplies++; - - // wait for ops to propagate to "w" nodes (doesn't warn if w used without replset) - if (mongoRestoreGlobalParams.w > 0) { - string err = conn().getLastError(db, false, false, mongoRestoreGlobalParams.w); - if (!err.empty()) { - toolError() << "Error while replaying oplog: " << err << std::endl; - } - } - return; - } - - if (nsToCollectionSubstring(_curns) == "system.indexes") { - createIndex(obj, true); - } - else if (_curns == "admin.system.roles") { - // To prevent modifying roles when other role modifications may be going on, restore - // the roles to a temporary collection and merge them into admin.system.roles later - // using the _mergeAuthzCollections command. - conn().insert(mongoRestoreGlobalParams.tempRolesColl, obj); - } - else if (_curcoll == "system.users") { - uassert(17416, - mongoutils::str::stream() << "Cannot modify user data on a server with version " - "greater than or equal to 2.5.4 that has not yet updated the " - "authorization data to at least schema version " << - AuthorizationManager::schemaVersion26Final << - ". Found server version " << _serverBinVersion << " with " - "authorization schema version " << _serverAuthzVersion, - _curdb != "admin" || - versionCmp(_serverBinVersion, "2.5.4") < 0 || - _serverAuthzVersion >= AuthorizationManager::schemaVersion26Final); - - if (obj.hasField("credentials")) { - if (_serverAuthzVersion == AuthorizationManager::schemaVersion24) { - // v3 user, v1 system - uasserted(17407, - mongoutils::str::stream() - << "Server has authorization schema version " - << AuthorizationManager::schemaVersion24 - << ", but found a schema version " - << AuthorizationManager::schemaVersion26Final << " user: " - << obj.toString()); - } else { - // v3 user, v3 system - uassert(17414, - mongoutils::str::stream() << "Found a schema version " << - AuthorizationManager::schemaVersion26Final << - " user when restoring to a non-admin db system.users " - "collection: " << obj.toString(), - _curdb == "admin"); - // To prevent modifying users when other user modifications may be going on, - // restore the users to a temporary collection and merge them into - // admin.system.users later using the _mergeAuthzCollections command. - conn().insert(mongoRestoreGlobalParams.tempUsersColl, obj); - } - } else { - if (_curdb == "admin" && - _serverAuthzVersion >= AuthorizationManager::schemaVersion26Final && - !_serverAuthzVersionDocExists) { - // server with schemaVersion26Final implies it is running 2.5.4 or greater. - uasserted(17415, - mongoutils::str::stream() << "Cannot restore users with schema " << - "version " << AuthorizationManager::schemaVersion24 << - " to a system with server version 2.5.4 or greater"); - } - - if (_serverAuthzVersion == AuthorizationManager::schemaVersion24 || - _curdb != "admin") { // Restoring 2.4 schema users to non-admin dbs is OK) - // v1 user, v1 system - string userDB; - uassertStatusOK(bsonExtractStringFieldWithDefault(obj, - "userSource", - _curdb, - &userDB)); - - if (mongoRestoreGlobalParams.drop && _users.count(UserName(obj["user"].String(), - userDB))) { - // Since system collections can't be dropped, we have to manually - // replace the contents of the system.users collection - BSONObj userMatch = BSON("user" << obj["user"].String() << - "userSource" << userDB); - conn().update(_curns, Query(userMatch), obj); - _users.erase(UserName(obj["user"].String(), userDB)); - } else { - conn().insert(_curns, obj); - } - } else { - // v1 user, v3 system - // TODO(spencer): SERVER-12491 Rather than failing here, we should convert the - // v1 user to an equivalent v3 schema user - uasserted(17408, - mongoutils::str::stream() - << "Server has authorization schema version " - << _serverAuthzVersion - << ", but found a schema version " - << AuthorizationManager::schemaVersion24 << " user: " - << obj.toString()); - } - } - } - else { - if (_curns == "admin.system.version") { - long long authVersion; - uassertStatusOK(bsonExtractIntegerField(obj, - AuthorizationManager::schemaVersionFieldName, - &authVersion)); - _dumpFileAuthzVersion = static_cast<int>(authVersion); - uassert(17371, - mongoutils::str::stream() << "Server's authorization data schema version " - "does not match that of the data in the dump file. Server's schema" - " version: " << _serverAuthzVersion << ", schema version in dump: " - << _dumpFileAuthzVersion, - _serverAuthzVersion == _dumpFileAuthzVersion); - } - conn().insert(_curns, obj); - } - - // wait for insert (or update) to propagate to "w" nodes (doesn't warn if w used - // without replset) - if (mongoRestoreGlobalParams.w > 0) { - string err = conn().getLastError(_curdb, false, false, mongoRestoreGlobalParams.w); - if (!err.empty()) { - toolError() << err << std::endl; - } - } - } - -private: - - BSONObj parseMetadataFile(string filePath) { - long long fileSize = boost::filesystem::file_size(filePath); - ifstream file(filePath.c_str(), ios_base::in); - - boost::scoped_array<char> buf(new char[fileSize + 1]); - file.read(buf.get(), fileSize); - buf[fileSize] = '\0'; - - int objSize; - BSONObj obj; - obj = fromjson(buf.get(), &objSize); - return obj; - } - - // Compares 2 BSONObj representing collection options. Returns true if the objects - // represent different options. Ignores the "create" field. - bool optionsSame(BSONObj obj1, BSONObj obj2) { - int nfields = 0; - BSONObjIterator i(obj1); - while ( i.more() ) { - BSONElement e = i.next(); - if (!obj2.hasField(e.fieldName())) { - if (strcmp(e.fieldName(), "create") == 0) { - continue; - } else { - return false; - } - } - nfields++; - if (e != obj2[e.fieldName()]) { - return false; - } - } - return nfields == obj2.nFields(); - } - - void createCollectionWithOptions(BSONObj obj) { - BSONObjIterator i(obj); - - // Rebuild obj as a command object for the "create" command. - // - {create: <name>} comes first, where <name> is the new name for the collection - // - elements with type Undefined get skipped over - BSONObjBuilder bo; - bo.append("create", _curcoll); - while (i.more()) { - BSONElement e = i.next(); - - if (strcmp(e.fieldName(), "create") == 0) { - continue; - } - - if (e.type() == Undefined) { - toolInfoLog() << _curns << ": skipping undefined field: " << e.fieldName() - << std::endl; - continue; - } - - bo.append(e); - } - obj = bo.obj(); - - BSONObj fields = BSON("options" << 1); - scoped_ptr<DBClientCursor> cursor(conn().query(_curdb + ".system.namespaces", Query(BSON("name" << _curns)), 0, 0, &fields)); - - bool createColl = true; - if (cursor->more()) { - createColl = false; - BSONObj nsObj = cursor->next(); - if (!nsObj.hasField("options") || !optionsSame(obj, nsObj["options"].Obj())) { - toolError() << "WARNING: collection " << _curns - << " exists with different options than are in the metadata.json file and" - << " not using --drop. Options in the metadata file will be ignored." - << std::endl; - } - } - - if (!createColl) { - return; - } - - BSONObj info; - if (!conn().runCommand(_curdb, obj, info)) { - uasserted(15936, "Creating collection " + _curns + " failed. Errmsg: " + info["errmsg"].String()); - } else { - toolInfoLog() << "\tCreated collection " << _curns << " with options: " - << obj.jsonString() << std::endl; - } - } - - /* We must handle if the dbname or collection name is different at restore time than what was dumped. - If keepCollName is true, however, we keep the same collection name that's in the index object. - */ - void createIndex(BSONObj indexObj, bool keepCollName) { - BSONObjBuilder bo; - BSONObjIterator i(indexObj); - while ( i.more() ) { - BSONElement e = i.next(); - if (strcmp(e.fieldName(), "ns") == 0) { - NamespaceString n(e.String()); - string s = _curdb + "." + (keepCollName ? n.coll().toString() : _curcoll); - bo.append("ns", s); - } - // Remove index version number - else if (strcmp(e.fieldName(), "v") != 0 || mongoRestoreGlobalParams.keepIndexVersion) { - bo.append(e); - } - } - BSONObj o = bo.obj(); - if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(0))) { - toolInfoLog() << "\tCreating index: " << o << std::endl; - } - conn().insert( _curdb + ".system.indexes" , o ); - - // We're stricter about errors for indexes than for regular data - BSONObj err = conn().getLastErrorDetailed(_curdb, false, false, mongoRestoreGlobalParams.w); - - if (err.hasField("err") && !err["err"].isNull()) { - if (err["err"].str() == "norepl" && mongoRestoreGlobalParams.w > 1) { - toolError() << "Cannot specify write concern for non-replicas" << std::endl; - } - else { - string errCode; - - if (err.hasField("code")) { - errCode = str::stream() << err["code"].numberInt(); - } - - toolError() << "Error creating index " << o["ns"].String() << ": " - << errCode << " " << err["err"] << std::endl; - } - - ::abort(); - } - - massert(16441, str::stream() << "Error calling getLastError: " << err["errmsg"], - err["ok"].trueValue()); - } -}; - -REGISTER_MONGO_TOOL(Restore); diff --git a/src/mongo/tools/shim.cpp b/src/mongo/tools/shim.cpp deleted file mode 100644 index 2badc79261c..00000000000 --- a/src/mongo/tools/shim.cpp +++ /dev/null @@ -1,469 +0,0 @@ -// shim.cpp - -/** -* Copyright (C) 2014 MongoDB, INC. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see <http://www.gnu.org/licenses/>. -* -* As a special exception, the copyright holders give permission to link the -* code of portions of this program with the OpenSSL library under certain -* conditions as described in each individual source file and distribute -* linked combinations including the program with the OpenSSL library. You -* must comply with the GNU Affero General Public License in all respects -* for all of the code used other than as permitted herein. If you modify -* file(s) with this exception, you may extend this exception to your -* version of the file(s), but you are not obligated to do so. If you do not -* wish to do so, delete this exception statement from your version. If you -* delete this exception statement from all source files in the program, -* then also delete it in the license file. -*/ - -#include "mongo/platform/basic.h" - -#include <boost/scoped_ptr.hpp> - -#include <boost/filesystem/convenience.hpp> -#include <boost/filesystem/operations.hpp> -#include <fstream> -#include <iostream> -#include <memory> - -#include "mongo/bson/bsonobjbuilder.h" -#include "mongo/client/dbclientcursor.h" -#include "mongo/db/catalog/collection.h" -#include "mongo/db/catalog/database.h" -#include "mongo/db/catalog/database_holder.h" -#include "mongo/db/json.h" -#include "mongo/db/operation_context_impl.h" -#include "mongo/db/storage/record_store.h" -#include "mongo/tools/mongoshim_options.h" -#include "mongo/tools/tool.h" -#include "mongo/tools/tool_logger.h" -#include "mongo/util/assert_util.h" -#include "mongo/util/options_parser/option_section.h" -#include "mongo/util/scopeguard.h" - -using std::auto_ptr; -using std::ios_base; -using std::ofstream; -using std::string; -using std::vector; - -using namespace mongo; - - - -/** - * Mongoshim mode handlers - */ -class ShimHandler { -public: - virtual ~ShimHandler() { } - - /** - * Returns true if this mode requires an output stream - */ - virtual bool requiresOutputStream() const = 0; - - /** - * If true, processes documents read from input in gotObject() - doRun() will not be called. - */ - virtual bool acceptsInputDocuments() const = 0; - - /** - * Process input document. - * Results may be written to output stream if provided. - */ - virtual void handleSingleInputDocument(const BSONObj& obj, std::ostream* out) const = 0; - - /** - * Generate results for this mode and write results to output stream if applicable. - * During processing, only one of handleSingleInputDocument() or generateOutputDocuments() - * will be called as determined by the result of acceptsInputDocuments(). - */ - virtual void generateOutputDocuments(std::ostream* out) const = 0; -}; - - - -/** - * Find mode. - */ -class FindShimHandler : public ShimHandler { -public: - FindShimHandler(DBClientBase& connection, const string& ns) - : _connection(connection), _ns(ns) { } - - virtual bool requiresOutputStream() const { return true; } - virtual bool acceptsInputDocuments() const { return false; } - virtual void handleSingleInputDocument(const BSONObj& obj, std::ostream* out) const { } - virtual void generateOutputDocuments(std::ostream* out) const { - invariant(out); - Query q(mongoShimGlobalParams.query); - if (mongoShimGlobalParams.sort != "") { - BSONObj sortSpec = mongo::fromjson(mongoShimGlobalParams.sort); - q.sort(sortSpec); - } - - auto_ptr<DBClientCursor> cursor = - _connection.query(_ns, q, mongoShimGlobalParams.limit, mongoShimGlobalParams.skip, - NULL, 0, QueryOption_NoCursorTimeout); - - while ( cursor->more() ) { - BSONObj obj = cursor->next(); - out->write( obj.objdata(), obj.objsize() ); - } - } - -private: - DBClientBase& _connection; - string _ns; -}; - - - -/** - * Insert mode. - */ -class InsertShimHandler : public ShimHandler { -public: - InsertShimHandler(DBClientBase& connection, const string& ns) - : _connection(connection), _ns(ns) { - if (mongoShimGlobalParams.drop) { - invariant(!_ns.empty()); - _connection.dropCollection(_ns); - } - } - - virtual bool requiresOutputStream() const { return false; } - virtual bool acceptsInputDocuments() const { return true; } - virtual void handleSingleInputDocument(const BSONObj& obj, std::ostream* out) const { - _connection.insert(_ns, obj); - } - virtual void generateOutputDocuments(std::ostream* out) const { } - -private: - DBClientBase& _connection; - string _ns; -}; - - - -/** - * Upsert mode. - */ -class UpsertShimHandler : public ShimHandler { -public: - UpsertShimHandler(DBClientBase& connection, const string& ns) - : _connection(connection), _ns(ns) { } - - virtual bool requiresOutputStream() const { return false; } - virtual bool acceptsInputDocuments() const { return true; } - virtual void handleSingleInputDocument(const BSONObj& obj, std::ostream* out) const { - BSONObjBuilder b; - invariant(!mongoShimGlobalParams.upsertFields.empty()); - for (vector<string>::const_iterator it = mongoShimGlobalParams.upsertFields.begin(), - end = mongoShimGlobalParams.upsertFields.end(); it != end; ++it) { - BSONElement e = obj.getFieldDotted(it->c_str()); - // If we cannot construct a valid query using the provided upsertFields, - // insert the object and skip the rest of the fields. - if (e.eoo()) { - _connection.insert(_ns, obj); - return; - } - b.appendAs(e, *it); - } - Query query(b.obj()); - bool upsert = true; - bool multi = false; - _connection.update(_ns, query, obj, upsert, multi); - } - virtual void generateOutputDocuments(std::ostream* out) const { } - -private: - DBClientBase& _connection; - string _ns; -}; - - - -/** - * Remove mode. - */ -class RemoveShimHandler : public ShimHandler { -public: - RemoveShimHandler(DBClientBase& connection, const string& ns) - : _connection(connection), _ns(ns) { } - - virtual bool requiresOutputStream() const { return false; } - virtual bool acceptsInputDocuments() const { return false; } - virtual void handleSingleInputDocument(const BSONObj& obj, std::ostream* out) const { } - virtual void generateOutputDocuments(std::ostream* out) const { - // Removes all documents matching query - bool justOne = false; - _connection.remove(_ns, mongoShimGlobalParams.query, justOne); - } - -private: - DBClientBase& _connection; - string _ns; -}; - - - -/** - * Repair mode. - * Writes valid objects in collection to output. - */ -class RepairShimHandler : public ShimHandler { -public: - RepairShimHandler(DBClientBase& connection, const string& ns) - : _connection(connection), _ns(ns) { } - - virtual bool requiresOutputStream() const { return true; } - virtual bool acceptsInputDocuments() const { return false; } - virtual void handleSingleInputDocument(const BSONObj& obj, std::ostream* out) const { } - virtual void generateOutputDocuments(std::ostream* out) const { - invariant(out); - toolInfoLog() << "going to try to recover data from: " << _ns << std::endl; - - OperationContextImpl txn; - - Database* db = dbHolder().openDb(&txn, toolGlobalParams.db); - if (!db) { - toolError() << "Database does not exist: " << toolGlobalParams.db << std::endl; - return; - } - - Collection* collection = db->getCollection(&txn, _ns); - if (!collection) { - toolError() << "Collection does not exist: " << toolGlobalParams.coll << std::endl; - return; - } - - toolInfoLog() << "nrecords: " << collection->numRecords(&txn) - << " datasize: " << collection->dataSize(&txn); - try { - WriteUnitOfWork wunit(&txn); - - boost::scoped_ptr<RecordIterator> iter( - collection->getRecordStore()->getIteratorForRepair(&txn)); - for (DiskLoc currLoc = iter->getNext(); !currLoc.isNull(); currLoc = iter->getNext()) { - if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { - toolInfoLog() << currLoc; - } - - BSONObj obj; - try { - obj = collection->docFor(&txn, currLoc); - - // If this is a corrupted object, just skip it, but do not abort the scan - // - if (!obj.valid()) { - continue; - } - - if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { - toolInfoLog() << obj; - } - - // Write valid object to output stream. - out->write(obj.objdata(), obj.objsize()); - } - catch (std::exception& ex) { - toolError() << "found invalid document @ " << currLoc << " " << ex.what(); - if ( ! obj.isEmpty() ) { - try { - toolError() << "first element: " << obj.firstElement(); - } - catch ( std::exception& ) { - toolError() << "unable to log invalid document @ " << currLoc; - } - } - } - } - - wunit.commit(); - } - catch (DBException& e) { - toolError() << "ERROR recovering: " << _ns << " " << e.toString(); - } - } - -private: - DBClientBase& _connection; - string _ns; -}; - - - -/** - * Command mode. - */ -class CommandShimHandler : public ShimHandler { -public: - CommandShimHandler(DBClientBase& connection) - : _connection(connection) { } - - virtual bool requiresOutputStream() const { return true; } - virtual bool acceptsInputDocuments() const { return true; } - virtual void handleSingleInputDocument(const BSONObj& obj, std::ostream* out) const { - invariant(out); - // Every document read from input is a separate command object. - BSONObj res; - bool ok = _connection.runCommand(toolGlobalParams.db, obj, res); - if (!ok) { - toolError() << "Failed to run command " << obj << ": " << res; - } - invariant(res.isValid()); - out->write(res.objdata(), res.objsize()); - } - virtual void generateOutputDocuments(std::ostream* out) const { } - -private: - DBClientBase& _connection; -}; - - - -class Shim : public BSONTool { -public: - Shim() : BSONTool() { } - - virtual void printHelp( ostream & out ) { - printMongoShimHelp(&out); - } - - virtual void gotObject(const BSONObj& obj, std::ostream* out) { - invariant(_shimHandler.get()); - _shimHandler->handleSingleInputDocument(obj, out); - } - - int doRun() { - // Flush stdout before returning from doRun(). - // XXX: This seems to be an issue under RHEL55 but not under other OSes or more recent - // versions of Linux - ON_BLOCK_EXIT(&std::ostream::flush, &cout); - - // getNS() could throw an exception. - string ns; - try { - if (mongoShimGlobalParams.mode != ShimMode::kCommand) { - ns = getNS(); - } - } - catch (...) { - printHelp(cerr); - return EXIT_FAILURE; - } - - switch (mongoShimGlobalParams.mode) { - case ShimMode::kFind: - _shimHandler.reset(new FindShimHandler(conn(), ns)); - break; - case ShimMode::kInsert: - _shimHandler.reset(new InsertShimHandler(conn(), ns)); - break; - case ShimMode::kUpsert: - _shimHandler.reset(new UpsertShimHandler(conn(), ns)); - break; - case ShimMode::kRemove: - _shimHandler.reset(new RemoveShimHandler(conn(), ns)); - break; - case ShimMode::kRepair: - _shimHandler.reset(new RepairShimHandler(conn(), ns)); - break; - case ShimMode::kCommand: - _shimHandler.reset(new CommandShimHandler(conn())); - break; - case ShimMode::kNumShimModes: - invariant(false); - } - - // Initialize output stream if handler needs it. - ostream *out = NULL; - auto_ptr<ofstream> fileStream; - if (_shimHandler->requiresOutputStream()) { - fileStream = _createOutputFile(); - if (fileStream.get()) { - if (!fileStream->good()) { - toolError() << "couldn't open [" << mongoShimGlobalParams.outputFile << "]"; - return EXIT_FAILURE; - } - out = fileStream.get(); - } - else { - // Write results to stdout by default. - out = &cout; - } - } - - // Skip doRun() if handler needs to read documents from input. - // The handler may still write results to output stream. - if (_shimHandler->acceptsInputDocuments()) { - // --inputDocuments and --in are used primarily for testing. - if (!mongoShimGlobalParams.inputDocuments.isEmpty()) { - BSONElement firstElement = mongoShimGlobalParams.inputDocuments.firstElement(); - if (firstElement.type() != Array) { - toolError() << "first element of --inputDocuments has to be an array: " - << firstElement; - return EXIT_FAILURE; - } - BSONObjIterator i(firstElement.Obj()); - while ( i.more() ) { - BSONElement e = i.next(); - if (!e.isABSONObj()) { - toolError() << "skipping non-object in input documents: " << e; - continue; - } - gotObject(e.Obj(), out); - } - } - else if (mongoShimGlobalParams.inputFileSpecified) { - processFile(mongoShimGlobalParams.inputFile, out); - } - else { - processFile("-", out); - } - } - else { - // 'out' may be NULL if not required by handler (eg. "remove" mode). - _shimHandler->generateOutputDocuments(out); - } - - return EXIT_SUCCESS; - } - -private: - - /** - * Returns a valid filestream if output file is specified and is not "-". - */ - auto_ptr<ofstream> _createOutputFile() { - auto_ptr<ofstream> fileStream; - if (mongoShimGlobalParams.outputFileSpecified && mongoShimGlobalParams.outputFile != "-") { - size_t idx = mongoShimGlobalParams.outputFile.rfind("/"); - if (idx != string::npos) { - string dir = mongoShimGlobalParams.outputFile.substr(0 , idx + 1); - boost::filesystem::create_directories(dir); - } - fileStream.reset(new ofstream(mongoShimGlobalParams.outputFile.c_str(), - ios_base::out | ios_base::binary)); - } - return fileStream; - } - - auto_ptr<ShimHandler> _shimHandler; -}; - -REGISTER_MONGO_TOOL(Shim); diff --git a/src/mongo/tools/shim_mode.cpp b/src/mongo/tools/shim_mode.cpp deleted file mode 100644 index 23a43b49138..00000000000 --- a/src/mongo/tools/shim_mode.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/* Copyright 2014 MongoDB Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/platform/basic.h" - -#include "mongo/tools/shim_mode.h" - -#include "mongo/util/assert_util.h" - -namespace mongo { - - std::string ShimMode::toString() const { - switch (_value) { - case kFind: return "find"; - case kInsert: return "insert"; - case kUpsert: return "upsert"; - case kRemove: return "remove"; - case kRepair: return "repair"; - case kCommand: return "command"; - case kNumShimModes: return "total"; - // No default. Compiler should complain if there's a shim mode that's not handled. - } - invariant(false); - } - -} // mongo diff --git a/src/mongo/tools/shim_mode.h b/src/mongo/tools/shim_mode.h deleted file mode 100644 index ae5ad6bee8a..00000000000 --- a/src/mongo/tools/shim_mode.h +++ /dev/null @@ -1,63 +0,0 @@ -/* Copyright 2014 MongoDB Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#pragma once - -#include <string> - -namespace mongo { - - /** - * Mongoshim modes. - */ - class ShimMode { - public: - enum Value { - kFind = 0, // Reads documents from collection. - kInsert = 1, // Inserts documents into collection. - kUpsert = 2, // Updates/inserts documents in collection. - kRemove = 3, // Removes documents from collection. - kRepair = 4, // Reads uncorrupted documents from collection record store. - kCommand = 5, // Runs a command on the database. - kNumShimModes - }; - - /* implicit */ ShimMode(Value value) : _value(value) {} - - operator Value() const { return _value; } - - /** - * Returns string representation shim mode. - * Used to generate list of choices for command line option. - */ - std::string toString() const; - - private: - Value _value; - }; - -} // namespace mongo diff --git a/src/mongo/tools/stat.cpp b/src/mongo/tools/stat.cpp deleted file mode 100644 index 5e161d2e3c5..00000000000 --- a/src/mongo/tools/stat.cpp +++ /dev/null @@ -1,531 +0,0 @@ -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see <http://www.gnu.org/licenses/>. -* -* As a special exception, the copyright holders give permission to link the -* code of portions of this program with the OpenSSL library under certain -* conditions as described in each individual source file and distribute -* linked combinations including the program with the OpenSSL library. You -* must comply with the GNU Affero General Public License in all respects -* for all of the code used other than as permitted herein. If you modify -* file(s) with this exception, you may extend this exception to your -* version of the file(s), but you are not obligated to do so. If you do not -* wish to do so, delete this exception statement from your version. If you -* delete this exception statement from all source files in the program, -* then also delete it in the license file. -*/ - -#include "mongo/platform/basic.h" - -#include <boost/thread/thread.hpp> -#include <fstream> -#include <iomanip> -#include <iostream> - -#include "mongo/base/init.h" -#include "mongo/client/dbclientcursor.h" -#include "mongo/client/sasl_client_authenticate.h" -#include "mongo/db/json.h" -#include "mongo/s/type_shard.h" -#include "mongo/tools/stat_util.h" -#include "mongo/tools/mongostat_options.h" -#include "mongo/tools/tool.h" -#include "mongo/util/net/httpclient.h" -#include "mongo/util/options_parser/option_section.h" -#include "mongo/util/text.h" - -namespace mongo { - - class Stat : public Tool { - public: - - Stat() : Tool() { - _autoreconnect = true; - } - - virtual void printHelp( ostream & out ) { - printMongoStatHelp(&out); - } - - BSONObj stats() { - if (mongoStatGlobalParams.http) { - HttpClient c; - HttpClient::Result r; - - string url; - { - stringstream ss; - ss << "http://" << toolGlobalParams.connectionString; - if (toolGlobalParams.connectionString.find( ":" ) == string::npos) - ss << ":28017"; - ss << "/_status"; - url = ss.str(); - } - - if ( c.get( url , &r ) != 200 ) { - toolError() << "error (http): " << r.getEntireResponse() << std::endl; - return BSONObj(); - } - - BSONObj x = fromjson( r.getBody() ); - BSONElement e = x["serverStatus"]; - if ( e.type() != Object ) { - toolError() << "BROKEN: " << x << std::endl; - return BSONObj(); - } - return e.embeddedObjectUserCheck(); - } - BSONObj out; - if (!conn().simpleCommand(toolGlobalParams.db, &out, "serverStatus")) { - toolError() << "error: " << out << std::endl; - return BSONObj(); - } - return out.getOwned(); - } - - int run() { - _statUtil.setAll(mongoStatGlobalParams.allFields); - _statUtil.setSeconds(mongoStatGlobalParams.sleep); - if (mongoStatGlobalParams.many) - return runMany(); - return runNormal(); - } - - static void printHeaders( const BSONObj& o ) { - BSONObjIterator i(o); - while ( i.more() ) { - BSONElement e = i.next(); - BSONObj x = e.Obj(); - cout << setw( x["width"].numberInt() ) << e.fieldName() << ' '; - } - cout << endl; - } - - static void printData( const BSONObj& o , const BSONObj& headers ) { - - BSONObjIterator i(headers); - while ( i.more() ) { - BSONElement e = i.next(); - BSONObj h = e.Obj(); - int w = h["width"].numberInt(); - - BSONElement data; - { - BSONElement temp = o[e.fieldName()]; - if ( temp.isABSONObj() ) - data = temp.Obj()["data"]; - } - - if ( data.type() == String ) - cout << setw(w) << data.String(); - else if ( data.type() == NumberDouble ) - cout << setw(w) << setprecision(3) << data.number(); - else if ( data.type() == NumberInt ) - cout << setw(w) << data.numberInt(); - else if ( data.eoo() ) - cout << setw(w) << ""; - else - cout << setw(w) << "???"; - - cout << ' '; - } - cout << endl; - } - - int runNormal() { - int rowNum = 0; - - BSONObj prev = stats(); - if ( prev.isEmpty() ) - return -1; - - int maxLockedDbWidth = 0; - - while (mongoStatGlobalParams.rowCount == 0 || - rowNum < mongoStatGlobalParams.rowCount) { - sleepsecs((int)ceil(_statUtil.getSeconds())); - BSONObj now; - try { - now = stats(); - } - catch ( std::exception& e ) { - toolError() << "can't get data: " << e.what() << std::endl; - continue; - } - - if ( now.isEmpty() ) - return -2; - - try { - - BSONObj out = _statUtil.doRow( prev , now ); - - // adjust width up as longer 'locked db' values appear - setMaxLockedDbWidth( &out, &maxLockedDbWidth ); - if (mongoStatGlobalParams.showHeaders && rowNum % 10 == 0) { - printHeaders( out ); - } - - printData( out , out ); - - } - catch ( AssertionException& e ) { - toolError() << "\nerror: " << e.what() << "\n" - << now - << std::endl; - } - - prev = now; - rowNum++; - } - return 0; - } - - /* Get the size of the 'locked db' field from a row of stats. If - * smaller than the current column width, set to the max. If - * greater, set the maxWidth to that value. - */ - void setMaxLockedDbWidth( const BSONObj* o, int* maxWidth ) { - BSONElement e = o->getField("locked db"); - if ( e.isABSONObj() ) { - BSONObj x = e.Obj(); - if ( x["width"].numberInt() < *maxWidth ) { - // TODO somthing less horrible. - BSONElement width = x["width"]; - invariant(width.type() == NumberInt); - char* nonConstPtr = const_cast<char*>(width.value()); - *reinterpret_cast<int*>(nonConstPtr) = *maxWidth; - } - else { - *maxWidth = x["width"].numberInt(); - } - } - } - - struct ServerState { - ServerState() : lock( "Stat::ServerState" ) {} - string host; - scoped_ptr<boost::thread> thr; - - mongo::mutex lock; - - BSONObj prev; - BSONObj now; - time_t lastUpdate; - vector<BSONObj> shards; - - string error; - bool mongos; - - BSONObj authParams; - }; - - static void serverThread( shared_ptr<ServerState> state , int sleepTime) { - try { - DBClientConnection conn( true ); - conn._logLevel = logger::LogSeverity::Debug(1); - string errmsg; - if ( ! conn.connect( HostAndPort(state->host) , errmsg ) ) - state->error = errmsg; - long long cycleNumber = 0; - - if (! (state->authParams["user"].str().empty()) ) - conn.auth(state->authParams); - - while ( ++cycleNumber ) { - try { - BSONObj out; - if ( conn.simpleCommand( "admin" , &out , "serverStatus" ) ) { - scoped_lock lk( state->lock ); - state->error = ""; - state->lastUpdate = time(0); - state->prev = state->now; - state->now = out.getOwned(); - } - else { - str::stream errorStream; - errorStream << "serverStatus failed"; - BSONElement errorField = out["errmsg"]; - if (errorField.type() == String) - errorStream << ": " << errorField.str(); - scoped_lock lk( state->lock ); - state->error = errorStream; - state->lastUpdate = time(0); - } - - if ( out["shardCursorType"].type() == Object || - out["process"].str() == "mongos" ) { - state->mongos = true; - if ( cycleNumber % 10 == 1 ) { - auto_ptr<DBClientCursor> c = conn.query( ShardType::ConfigNS , BSONObj() ); - vector<BSONObj> shards; - while ( c->more() ) { - shards.push_back( c->nextSafe().getOwned() ); - } - scoped_lock lk( state->lock ); - state->shards = shards; - } - } - } - catch ( std::exception& e ) { - scoped_lock lk( state->lock ); - state->error = e.what(); - } - - sleepsecs( sleepTime ); - } - - - } - catch ( std::exception& e ) { - toolError() << "serverThread (" << state->host << ") fatal error : " << e.what() - << std::endl; - } - catch ( ... ) { - toolError() << "serverThread (" << state->host << ") fatal error" << std::endl; - } - } - - typedef map<string,shared_ptr<ServerState> > StateMap; - - bool _add( StateMap& threads , string host ) { - shared_ptr<ServerState>& state = threads[host]; - if ( state ) - return false; - - state.reset( new ServerState() ); - state->host = host; - /* For each new thread, pass in a thread state object and the delta between samples */ - state->thr.reset( new boost::thread( stdx::bind( serverThread, - state, - (int)ceil(_statUtil.getSeconds()) ) ) ); - state->authParams = BSON(saslCommandUserFieldName << toolGlobalParams.username - << saslCommandPasswordFieldName << toolGlobalParams.password - << saslCommandUserDBFieldName << getAuthenticationDatabase() - << saslCommandMechanismFieldName - << toolGlobalParams.authenticationMechanism); - return true; - } - - /** - * @param hosts [ "a.foo.com" , "b.foo.com" ] - */ - bool _addAll( StateMap& threads , const BSONObj& hosts ) { - BSONObjIterator i( hosts ); - bool added = false; - while ( i.more() ) { - bool me = _add( threads , i.next().String() ); - added = added || me; - } - return added; - } - - bool _discover( StateMap& threads , const string& host , const shared_ptr<ServerState>& ss ) { - - BSONObj info = ss->now; - - bool found = false; - - if ( info["repl"].isABSONObj() ) { - BSONObj x = info["repl"].Obj(); - if ( x["hosts"].isABSONObj() ) - if ( _addAll( threads , x["hosts"].Obj() ) ) - found = true; - if ( x["passives"].isABSONObj() ) - if ( _addAll( threads , x["passives"].Obj() ) ) - found = true; - } - - if ( ss->mongos ) { - for ( unsigned i=0; i<ss->shards.size(); i++ ) { - BSONObj x = ss->shards[i]; - - string errmsg; - ConnectionString cs = ConnectionString::parse( x["host"].String() , errmsg ); - if ( errmsg.size() ) { - toolError() << errmsg << std::endl; - continue; - } - - vector<HostAndPort> v = cs.getServers(); - for ( unsigned i=0; i<v.size(); i++ ) { - if ( _add( threads , v[i].toString() ) ) - found = true; - } - } - } - - return found; - } - - int runMany() { - StateMap threads; - - { - string orig = "localhost"; - bool showPorts = false; - if (toolGlobalParams.hostSet) { - orig = toolGlobalParams.host; - } - - if (orig.find(":") != string::npos || toolGlobalParams.portSet) - showPorts = true; - - StringSplitter ss( orig.c_str() , "," ); - while ( ss.more() ) { - string host = ss.next(); - if ( showPorts && host.find( ":" ) == string::npos) { - // port supplied, but not for this host. use default. - StringBuilder sb; - if (toolGlobalParams.portSet) { - sb << host << ":" << toolGlobalParams.port; - } - else { - sb << host << ":27017"; - } - host = sb.str(); - } - _add( threads , host ); - } - } - - sleepsecs(1); - - int row = 0; - int maxLockedDbWidth = 0; - - while (mongoStatGlobalParams.rowCount == 0 || row < mongoStatGlobalParams.rowCount) { - sleepsecs( (int)ceil(_statUtil.getSeconds()) ); - - // collect data - vector<Row> rows; - for ( map<string,shared_ptr<ServerState> >::iterator i=threads.begin(); i!=threads.end(); ++i ) { - scoped_lock lk( i->second->lock ); - - if ( i->second->error.size() ) { - rows.push_back( Row( i->first , i->second->error ) ); - } - else if ( i->second->prev.isEmpty() || i->second->now.isEmpty() ) { - rows.push_back( Row( i->first ) ); - } - else { - BSONObj out = _statUtil.doRow( i->second->prev , i->second->now ); - rows.push_back( Row( i->first , out ) ); - } - - if (mongoStatGlobalParams.discover && ! i->second->now.isEmpty()) { - if ( _discover( threads , i->first , i->second ) ) - break; - } - } - - // compute some stats - unsigned longestHost = 0; - BSONObj biggest; - for ( unsigned i=0; i<rows.size(); i++ ) { - if ( rows[i].host.size() > longestHost ) - longestHost = rows[i].host.size(); - if ( rows[i].data.nFields() > biggest.nFields() ) - biggest = rows[i].data; - - // adjust width up as longer 'locked db' values appear - setMaxLockedDbWidth( &rows[i].data, &maxLockedDbWidth ); - } - - { - // check for any headers not in biggest - - // TODO: we put any new headers at end, - // ideally we would interleave - - set<string> seen; - - BSONObjBuilder b; - - { - // iterate biggest - BSONObjIterator i( biggest ); - while ( i.more() ) { - BSONElement e = i.next(); - seen.insert( e.fieldName() ); - b.append( e ); - } - } - - // now do the rest - for ( unsigned j=0; j<rows.size(); j++ ) { - BSONObjIterator i( rows[j].data ); - while ( i.more() ) { - BSONElement e = i.next(); - if ( seen.count( e.fieldName() ) ) - continue; - seen.insert( e.fieldName() ); - b.append( e ); - } - - } - - biggest = b.obj(); - - } - - // display data - - cout << endl; - - // header - if (row++ % 5 == 0 && mongoStatGlobalParams.showHeaders && !biggest.isEmpty()) { - cout << setw( longestHost ) << "" << "\t"; - printHeaders( biggest ); - } - - // rows - for ( unsigned i=0; i<rows.size(); i++ ) { - cout << setw( longestHost ) << rows[i].host << "\t"; - if ( rows[i].err.size() ) - cout << rows[i].err << endl; - else if ( rows[i].data.isEmpty() ) - cout << "no data" << endl; - else - printData( rows[i].data , biggest ); - } - - } - - return 0; - } - - StatUtil _statUtil; - - struct Row { - Row( string h , string e ) { - host = h; - err = e; - } - - Row( string h ) { - host = h; - } - - Row( string h , BSONObj d ) { - host = h; - data = d; - } - string host; - string err; - BSONObj data; - }; - }; - REGISTER_MONGO_TOOL(Stat); -} diff --git a/src/mongo/tools/stat_util.cpp b/src/mongo/tools/stat_util.cpp deleted file mode 100644 index 7117d7ce511..00000000000 --- a/src/mongo/tools/stat_util.cpp +++ /dev/null @@ -1,365 +0,0 @@ -// stat_util.cpp - -/** - * Copyright (C) 2008 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include <iomanip> - -#include "stat_util.h" -#include "mongo/util/mongoutils/str.h" - -using namespace mongoutils; - -namespace mongo { - - StatUtil::StatUtil( double seconds , bool all ) : - _seconds( seconds ) , - _all( all ) - - { - - } - - bool StatUtil::_in( const BSONElement& me , const BSONElement& arr ) { - if ( me.type() != String || arr.type() != Array ) - return false; - - string s = me.String(); - BSONForEach(e, arr.Obj()) { - if ( e.type() == String && s == e.String() ) - return true; - } - return false; - } - - BSONObj StatUtil::doRow( const BSONObj& a , const BSONObj& b ) { - BSONObjBuilder result; - - bool isMongos = b["shardCursorType"].type() == Object || b["process"].String() == "mongos"; - - if ( a["opcounters"].isABSONObj() && b["opcounters"].isABSONObj() ) { - BSONObj ax = a["opcounters"].embeddedObject(); - BSONObj bx = b["opcounters"].embeddedObject(); - - BSONObj ar = a["opcountersRepl"].isABSONObj() ? a["opcountersRepl"].embeddedObject() : BSONObj(); - BSONObj br = b["opcountersRepl"].isABSONObj() ? b["opcountersRepl"].embeddedObject() : BSONObj(); - - BSONObjIterator i( bx ); - while ( i.more() ) { - BSONElement e = i.next(); - if ( ar.isEmpty() || br.isEmpty() ) { - _append( result , e.fieldName() , 6 , (int)diff( e.fieldName() , ax , bx ) ); - } - else { - string f = e.fieldName(); - - int m = (int)diff( f , ax , bx ); - int r = (int)diff( f , ar , br ); - - string myout; - - if ( f == "command" ) { - myout = str::stream() << m << "|" << r; - } - else if ( f == "getmore" ) { - myout = str::stream() << m; - } - else if ( m && r ) { - // this is weird... - myout = str::stream() << m << "|" << r; - } - else if ( m ) { - myout = str::stream() << m; - } - else if ( r ) { - myout = str::stream() << "*" << r; - } - else { - myout = "*0"; - } - - _append( result , f , 6 , myout ); - } - } - } - - if ( b["backgroundFlushing"].type() == Object ) { - BSONObj ax = a["backgroundFlushing"].embeddedObject(); - BSONObj bx = b["backgroundFlushing"].embeddedObject(); - _append( result , "flushes" , 6 , (int)diff( "flushes" , ax , bx ) ); - } - - if ( b.getFieldDotted("mem.supported").trueValue() ) { - BSONObj bx = b["mem"].embeddedObject(); - BSONObjIterator i( bx ); - if (!isMongos) - _appendMem( result , "mapped" , 6 , bx["mapped"].numberInt() ); - _appendMem( result , "vsize" , 6 , bx["virtual"].numberInt() ); - _appendMem( result , "res" , 6 , bx["resident"].numberInt() ); - - if ( !isMongos && _all ) - _appendMem( result , "non-mapped" , 6 , bx["virtual"].numberInt() - bx["mapped"].numberInt() ); - } - - if ( b["extra_info"].type() == Object ) { - BSONObj ax = a["extra_info"].embeddedObject(); - BSONObj bx = b["extra_info"].embeddedObject(); - if ( ax["page_faults"].type() ) - _append( result , "faults" , 6 , (int)diff( "page_faults" , ax , bx ) ); - } - - if (!isMongos) { - - if ( b["locks"].isABSONObj() ) { - // report either the global lock % or the db with the highest lock % + the global lock - NamespaceStats prevStats = parseServerStatusLocks( a ); - NamespaceStats curStats = parseServerStatusLocks( b ); - vector<NamespaceDiff> diffs = computeDiff( prevStats , curStats ); - - if ( diffs.size() == 0 ) { - _append( result , "locked %" , 8 , 0 ); - } - else { - - // diff() divides the result by _seconds, need total uptime here - double uptimeMillis = diff( "uptimeMillis" , a , b ) * _seconds; - unsigned idx = diffs.size()-1; - - double lockToReport = diffs[idx].write; - if ( diffs[idx].ns != "." ) { - for ( unsigned i = 0; i < diffs.size(); i++ ) { - if ( diffs[i].ns == "." ) - lockToReport += diffs[i].write; - } - } - - stringstream ss; - ss.setf( ios::fixed ); - ss << diffs[idx].ns << ":" << setprecision(1) << ( 100.0 * lockToReport / uptimeMillis ) << "%"; - // set the width to be the greater of the field header size or the actual field length, e.g., 'mylongdb:89.1%' - _append( result , "locked db" , ( ss.str().size() > 10 ? ss.str().size() : 10 ) , ss.str() ); - } - } - else { - _append( result , "locked %" , 8 , percent( "globalLock.totalTime" , "globalLock.lockTime" , a , b ) ); - } - - - _append( result , "idx miss %" , 8 , percent( "indexCounters.btree.accesses" , "indexCounters.btree.misses" , a , b ) ); - } - - if ( b.getFieldDotted( "globalLock.currentQueue" ).type() == Object ) { - int r = b.getFieldDotted( "globalLock.currentQueue.readers" ).numberInt(); - int w = b.getFieldDotted( "globalLock.currentQueue.writers" ).numberInt(); - stringstream temp; - temp << r << "|" << w; - _append( result , "qr|qw" , 9 , temp.str() ); - } - - if ( b.getFieldDotted( "globalLock.activeClients" ).type() == Object ) { - int r = b.getFieldDotted( "globalLock.activeClients.readers" ).numberInt(); - int w = b.getFieldDotted( "globalLock.activeClients.writers" ).numberInt(); - stringstream temp; - temp << r << "|" << w; - _append( result , "ar|aw" , 7 , temp.str() ); - } - - if ( a["network"].isABSONObj() && b["network"].isABSONObj() ) { - BSONObj ax = a["network"].embeddedObject(); - BSONObj bx = b["network"].embeddedObject(); - _appendNet( result , "netIn" , diff( "bytesIn" , ax , bx ) ); - _appendNet( result , "netOut" , diff( "bytesOut" , ax , bx ) ); - } - - _append( result , "conn" , 5 , b.getFieldDotted( "connections.current" ).numberInt() ); - - if ( b["repl"].type() == Object ) { - - BSONObj x = b["repl"].embeddedObject(); - bool isReplSet = x["setName"].type() == String; - - stringstream ss; - - if ( isReplSet ) { - string setName = x["setName"].String(); - _append( result , "set" , setName.size() , setName ); - } - - if ( x["ismaster"].trueValue() ) - ss << "PRI"; - else if ( x["secondary"].trueValue() ) - ss << "SEC"; - else if ( x["isreplicaset"].trueValue() ) - ss << "REC"; - else if ( x["arbiterOnly"].trueValue() ) - ss << "ARB"; - else if ( _in( x["me"] , x["passives"] ) ) - ss << "PSV"; - else if ( isReplSet ) - ss << "UNK"; - else - ss << "SLV"; - - _append( result , "repl" , 4 , ss.str() ); - - } - else if ( isMongos ) { - _append( result , "repl" , 4 , "RTR" ); - } - - { - struct tm t; - time_t_to_Struct( time(0), &t , true ); - stringstream temp; - temp << setfill('0') << setw(2) << t.tm_hour - << ":" - << setfill('0') << setw(2) << t.tm_min - << ":" - << setfill('0') << setw(2) << t.tm_sec; - _append( result , "time" , 10 , temp.str() ); - } - return result.obj(); - } - - - - double StatUtil::percent( const char * outof , const char * val , const BSONObj& a , const BSONObj& b ) { - double x = ( b.getFieldDotted( val ).number() - a.getFieldDotted( val ).number() ); - double y = ( b.getFieldDotted( outof ).number() - a.getFieldDotted( outof ).number() ); - if ( y == 0 ) - return 0; - double p = x / y; - p = (double)((int)(p * 1000)) / 10; - return p; - } - - - - double StatUtil::diff( const string& name , const BSONObj& a , const BSONObj& b ) { - BSONElement x = a.getFieldDotted( name.c_str() ); - BSONElement y = b.getFieldDotted( name.c_str() ); - if ( ! x.isNumber() || ! y.isNumber() ) - return -1; - return ( y.number() - x.number() ) / _seconds; - } - - - void StatUtil::_appendNet( BSONObjBuilder& result , const string& name , double diff ) { - // I think 1000 is correct for megabit, but I've seen conflicting things (ERH 11/2010) - const double div = 1000; - - string unit = "b"; - - if ( diff >= div ) { - unit = "k"; - diff /= div; - } - - if ( diff >= div ) { - unit = "m"; - diff /= div; - } - - if ( diff >= div ) { - unit = "g"; - diff /= div; - } - - string out = str::stream() << (int)diff << unit; - _append( result , name , 6 , out ); - } - - - - void StatUtil::_appendMem( BSONObjBuilder& result , const string& name , unsigned width , double sz ) { - string unit = "m"; - if ( sz > 1024 ) { - unit = "g"; - sz /= 1024; - } - - if ( sz >= 1000 ) { - string s = str::stream() << (int)sz << unit; - _append( result , name , width , s ); - return; - } - - stringstream ss; - ss << setprecision(3) << sz << unit; - _append( result , name , width , ss.str() ); - } - - NamespaceStats StatUtil::parseServerStatusLocks( const BSONObj& serverStatus ) { - NamespaceStats stats; - - if ( ! serverStatus["locks"].isABSONObj() ) { - cout << "locks doesn't exist, old mongod? (<2.2?)" << endl; - return stats; - } - - BSONObj locks = serverStatus["locks"].Obj(); - - BSONObjIterator i( locks ); - while ( i.more() ) { - BSONElement e = i.next(); - - NamespaceInfo& s = stats[e.fieldName()]; - s.ns = e.fieldName(); - - BSONObj temp = e.Obj()["timeLockedMicros"].Obj(); - s.read = ( temp["r"].numberLong() + temp["R"].numberLong() ) / 1000; - s.write = ( temp["w"].numberLong() + temp["W"].numberLong() ) / 1000; - } - - return stats; - - } - - vector<NamespaceDiff> StatUtil::computeDiff( const NamespaceStats& prev , const NamespaceStats& current ) { - vector<NamespaceDiff> data; - - for ( NamespaceStats::const_iterator i = current.begin() ; i != current.end(); ++i ) { - const string& ns = i->first; - const NamespaceInfo& b = i->second; - if ( prev.find( ns ) == prev.end() ) - continue; - const NamespaceInfo& a = prev.find( ns )->second; - - // invalid, data fixed in 1.8.0 - if ( ns[0] == '?' ) - continue; - - data.push_back( NamespaceDiff( a , b ) ); - } - - std::sort( data.begin() , data.end() ); - - return data; - } -} - diff --git a/src/mongo/tools/stat_util.h b/src/mongo/tools/stat_util.h deleted file mode 100644 index e2ad42f1c36..00000000000 --- a/src/mongo/tools/stat_util.h +++ /dev/null @@ -1,128 +0,0 @@ -// stat_util.h - -/** - * Copyright (C) 2008 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#pragma once - -#include "mongo/pch.h" -#include "../db/jsobj.h" - -namespace mongo { - - - struct NamespaceInfo { - std::string ns; - - // these need to be in millis - long long read; - long long write; - - std::string toString() const { - std::stringstream ss; - ss << ns << " r: " << read << " w: " << write; - return ss.str(); - } - - }; - - struct NamespaceDiff { - std::string ns; - - long long read; - long long write; - - NamespaceDiff( NamespaceInfo prev , NamespaceInfo now ) { - ns = prev.ns; - read = now.read - prev.read; - write = now.write - prev.write; - } - - long long total() const { return read + write; } - - bool operator<(const NamespaceDiff& r) const { - return total() < r.total(); - } - }; - - typedef std::map<std::string,NamespaceInfo> NamespaceStats; - - /** - * static methods useful for computing status from serverStatus type things - */ - class StatUtil { - public: - /** - * @param seconds - seconds between calls to serverStatus - * @param all - show all fields - */ - StatUtil( double seconds = 1 , bool all = false ); - - /** - * @param a older serverStatus - * @param b newer serverStatus - */ - BSONObj doRow( const BSONObj& a , const BSONObj& b ); - - double getSeconds() const { return _seconds; } - bool getAll() const { return _all; } - - void setSeconds( double seconds ) { _seconds = seconds; } - void setAll( bool all ) { _all = all; } - - static NamespaceStats parseServerStatusLocks( const BSONObj& serverStatus ); - static std::vector<NamespaceDiff> computeDiff( const NamespaceStats& prev , const NamespaceStats& current ); - private: - - - double percent( const char * outof , const char * val , const BSONObj& a , const BSONObj& b ); - - double diff( const std::string& name , const BSONObj& a , const BSONObj& b ); - - void _appendMem( BSONObjBuilder& result , const std::string& name , unsigned width , double sz ); - - void _appendNet( BSONObjBuilder& result , const std::string& name , double diff ); - - template<typename T> - void _append( BSONObjBuilder& result , const std::string& name , unsigned width , const T& t ) { - if ( name.size() > width ) - width = name.size(); - result.append( name , BSON( "width" << (int)width << "data" << t ) ); - } - - bool _in( const BSONElement& me , const BSONElement& arr ); - - - // ------- - - double _seconds; - bool _all; - - }; -} - diff --git a/src/mongo/tools/tool.cpp b/src/mongo/tools/tool.cpp deleted file mode 100644 index c6675da43f3..00000000000 --- a/src/mongo/tools/tool.cpp +++ /dev/null @@ -1,435 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -// Tool.cpp - -#include "mongo/tools/tool.h" - -#include <boost/filesystem/operations.hpp> -#include <fstream> -#include <iostream> -#include <limits> - -#include "mongo/base/initializer.h" -#include "mongo/base/init.h" -#include "mongo/client/dbclient_rs.h" -#include "mongo/client/sasl_client_authenticate.h" -#include "mongo/db/auth/authorization_manager.h" -#include "mongo/db/auth/authorization_manager_global.h" -#include "mongo/db/auth/authz_manager_external_state_mock.h" -#include "mongo/db/dbdirectclient.h" -#include "mongo/db/global_environment_experiment.h" -#include "mongo/db/global_environment_d.h" -#include "mongo/db/json.h" -#include "mongo/db/operation_context_impl.h" -#include "mongo/db/repl/repl_coordinator_global.h" -#include "mongo/db/repl/repl_coordinator_mock.h" -#include "mongo/db/storage_options.h" -#include "mongo/db/storage/storage_engine.h" -#include "mongo/platform/posix_fadvise.h" -#include "mongo/util/exception_filter_win32.h" -#include "mongo/util/exit.h" -#include "mongo/util/log.h" -#include "mongo/util/options_parser/option_section.h" -#include "mongo/util/password.h" -#include "mongo/util/net/ssl_options.h" -#include "mongo/util/quick_exit.h" -#include "mongo/util/signal_handlers.h" -#include "mongo/util/text.h" -#include "mongo/util/version.h" - -using namespace std; -using namespace mongo; - -namespace mongo { - - Tool::Tool() : - _autoreconnect(false), _conn(0), _slaveConn(0) { } - - Tool::~Tool() { - if ( _conn ) - delete _conn; - } - - MONGO_INITIALIZER(ToolMocks)(InitializerContext*) { - setGlobalAuthorizationManager(new AuthorizationManager( - new AuthzManagerExternalStateMock())); - repl::ReplSettings replSettings; - repl::setGlobalReplicationCoordinator(new repl::ReplicationCoordinatorMock(replSettings)); - return Status::OK(); - } - - int Tool::main( int argc , char ** argv, char ** envp ) { - setupSignalHandlers(true); - - static StaticObserver staticObserver; - - mongo::runGlobalInitializersOrDie(argc, argv, envp); - - // hide password from ps output - for (int i=0; i < (argc-1); ++i) { - if (!strcmp(argv[i], "-p") || !strcmp(argv[i], "--password")) { - char* arg = argv[i+1]; - while (*arg) { - *arg++ = 'x'; - } - } - } - - if (!toolGlobalParams.useDirectClient) { - if (toolGlobalParams.noconnection) { - // do nothing - } - else { - string errmsg; - - ConnectionString cs = ConnectionString::parse(toolGlobalParams.connectionString, - errmsg); - if ( ! cs.isValid() ) { - toolError() << "invalid hostname [" << toolGlobalParams.connectionString << "] " - << errmsg << std::endl; - quickExit(-1); - } - - _conn = cs.connect( errmsg ); - if ( ! _conn ) { - toolError() << "couldn't connect to [" << toolGlobalParams.connectionString - << "] " << errmsg << std::endl; - quickExit(-1); - } - - toolInfoOutput() << "connected to: " << toolGlobalParams.connectionString - << std::endl; - } - - } - else { - verify( lastError.get( true ) ); - - Client::initThread("tools"); - storageGlobalParams.dbpath = toolGlobalParams.dbpath; - try { - getGlobalEnvironment()->setGlobalStorageEngine(storageGlobalParams.engine); - } - catch (const DBException& ex) { - if (ex.getCode() == ErrorCodes::DBPathInUse) { - toolError() << std::endl << "If you are running a mongod on the same " - "path you should connect to that instead of direct data " - "file access" << std::endl << std::endl; - } - else { - toolError() << "Failed to initialize storage engine: " << ex.toString(); - } - dbexit( EXIT_FS ); - quickExit(EXIT_FAILURE); - } - - _txn.reset(new OperationContextImpl()); - _conn = new DBDirectClient(_txn.get()); - } - - int ret = -1; - try { - if (!toolGlobalParams.useDirectClient && !toolGlobalParams.noconnection) - auth(); - ret = run(); - } - catch ( DBException& e ) { - toolError() << "assertion: " << e.toString() << std::endl; - ret = -1; - } - catch(const boost::filesystem::filesystem_error &fse) { - /* - https://jira.mongodb.org/browse/SERVER-2904 - - Simple tools that don't access the database, such as - bsondump, aren't throwing DBExceptions, but are throwing - boost exceptions. - - The currently available set of error codes don't seem to match - boost documentation. boost::filesystem::not_found_error - (from http://www.boost.org/doc/libs/1_31_0/libs/filesystem/doc/exception.htm) - doesn't seem to exist in our headers. Also, fse.code() isn't - boost::system::errc::no_such_file_or_directory when this - happens, as you would expect. And, determined from - experimentation that the command-line argument gets turned into - "\\?" instead of "/?" !!! - */ -#if defined(_WIN32) - if (/*(fse.code() == boost::system::errc::no_such_file_or_directory) &&*/ - (fse.path1() == "\\?")) - printHelp(cerr); - else -#endif // _WIN32 - toolError() << "error: " << fse.what() << std::endl; - - ret = -1; - } - - if ( currentClient.get() ) - currentClient.get()->shutdown(); - - if (toolGlobalParams.useDirectClient) { - exitCleanly(EXIT_CLEAN); - } - - fflush(stdout); - fflush(stderr); - quickExit(ret); - } - - DBClientBase& Tool::conn( bool slaveIfPaired ) { - if ( slaveIfPaired && _conn->type() == ConnectionString::SET ) { - if (!_slaveConn) { - DBClientReplicaSet* rs = static_cast<DBClientReplicaSet*>(_conn); - _slaveConn = &rs->slaveConn(); - } - return *_slaveConn; - } - return *_conn; - } - - bool Tool::isMaster() { - if (toolGlobalParams.useDirectClient) { - return true; - } - - BSONObj info; - bool isMaster; - bool ok = conn().isMaster(isMaster, &info); - - if (ok && !isMaster) { - toolError() << "ERROR: trying to write to non-master " << conn().toString() - << std::endl; - toolError() << "isMaster info: " << info << std::endl; - return false; - } - - return true; - } - - bool Tool::isMongos() { - // TODO: when mongos supports QueryOption_Exaust add a version check (SERVER-2628) - BSONObj isdbgrid; - conn("true").simpleCommand("admin", &isdbgrid, "isdbgrid"); - return isdbgrid["isdbgrid"].trueValue(); - } - - std::string Tool::getAuthenticationDatabase() { - if (!toolGlobalParams.authenticationDatabase.empty()) { - return toolGlobalParams.authenticationDatabase; - } - - if (!toolGlobalParams.db.empty()) { - return toolGlobalParams.db; - } - - return "admin"; - } - - /** - * Validate authentication on the server for the given dbname. - */ - void Tool::auth() { - - if (toolGlobalParams.username.empty()) { - // Make sure that we don't need authentication to connect to this db - // findOne throws an AssertionException if it's not authenticated. - if (toolGlobalParams.coll.size() > 0) { - // BSONTools don't have a collection - conn().findOne(getNS(), Query("{}"), 0, QueryOption_SlaveOk); - } - - return; - } - - BSONObjBuilder authParams; - authParams << - saslCommandUserDBFieldName << getAuthenticationDatabase() << - saslCommandUserFieldName << toolGlobalParams.username << - saslCommandPasswordFieldName << toolGlobalParams.password << - saslCommandMechanismFieldName << - toolGlobalParams.authenticationMechanism; - - if (!toolGlobalParams.gssapiServiceName.empty()) { - authParams << saslCommandServiceNameFieldName << toolGlobalParams.gssapiServiceName; - } - - if (!toolGlobalParams.gssapiHostName.empty()) { - authParams << saslCommandServiceHostnameFieldName << toolGlobalParams.gssapiHostName; - } - - _conn->auth(authParams.obj()); - } - - BSONTool::BSONTool() : Tool() { } - - int BSONTool::run() { - - if (bsonToolGlobalParams.hasFilter) { - _matcher.reset(new Matcher(fromjson(bsonToolGlobalParams.filter), - MatchExpressionParser::WhereCallback())); - } - - return doRun(); - } - - long long BSONTool::processFile(const boost::filesystem::path& root, std::ostream* out) { - bool isFifoFile = boost::filesystem::status(root).type() == boost::filesystem::fifo_file; - bool isStdin = root == "-"; - - std::string fileName = root.string(); - - unsigned long long fileLength = (isFifoFile || isStdin) ? - std::numeric_limits<unsigned long long>::max() : file_size(root); - - if ( fileLength == 0 ) { - toolInfoOutput() << "file " << fileName << " empty, skipping" << std::endl; - return 0; - } - - - FILE* file = isStdin ? stdin : fopen( fileName.c_str() , "rb" ); - if ( ! file ) { - toolError() << "error opening file: " << fileName << " " << errnoWithDescription() - << std::endl; - return 0; - } - - if (!isFifoFile && !isStdin) { -#ifdef POSIX_FADV_SEQUENTIAL - posix_fadvise(fileno(file), 0, fileLength, POSIX_FADV_SEQUENTIAL); -#endif - - if (logger::globalLogDomain()->shouldLog(logger::LogSeverity::Debug(1))) { - toolInfoOutput() << "\t file size: " << fileLength << std::endl; - } - } - - unsigned long long read = 0; - unsigned long long num = 0; - unsigned long long processed = 0; - - const int BUF_SIZE = BSONObjMaxUserSize + ( 1024 * 1024 ); - boost::scoped_array<char> buf_holder(new char[BUF_SIZE]); - char * buf = buf_holder.get(); - - // no progress is available for FIFO - // only for regular files - boost::scoped_ptr<ProgressMeter> m; - if (!toolGlobalParams.quiet && !isFifoFile && !isStdin) { - m.reset(new ProgressMeter( fileLength )); - m->setUnits( "bytes" ); - } - - while ( read < fileLength ) { - size_t amt = fread(buf, 1, 4, file); - // end of fifo - if ((isFifoFile || isStdin) && ::feof(file)) { - break; - } - verify( amt == 4 ); - - int size = ((int*)buf)[0]; - uassert( 10264 , str::stream() << "invalid object size: " << size , size < BUF_SIZE ); - - amt = fread(buf+4, 1, size-4, file); - verify( amt == (size_t)( size - 4 ) ); - - BSONObj o( buf ); - if (bsonToolGlobalParams.objcheck) { - const Status status = validateBSON(buf, size); - if (!status.isOK()) { - toolError() << "INVALID OBJECT - going to try and print out " << std::endl; - toolError() << "size: " << size << std::endl; - toolError() << "error: " << status.reason() << std::endl; - - StringBuilder sb; - try { - o.toString(sb); // using StringBuilder version to get as much as possible - } catch (...) { - toolError() << "object up to error: " << sb.str() << endl; - throw; - } - toolError() << "complete object: " << sb.str() << endl; - - // NOTE: continuing with object even though we know it is invalid. - } - } - - if (!bsonToolGlobalParams.hasFilter || _matcher->matches(o)) { - gotObject(o, out); - processed++; - } - - read += o.objsize(); - num++; - - if (m.get()) { - m->hit(o.objsize()); - } - } - - if (!isStdin) { - fclose(file); - } - - if (!isFifoFile && !isStdin) { - uassert(10265, "counts don't match", read == fileLength); - } - toolInfoOutput() << num << ((num == 1) ? " document" : " documents") - << " found" << std::endl; - if (bsonToolGlobalParams.hasFilter) { - toolInfoOutput() << processed - << ((processed == 1) ? " document" : " documents") - << " processed" << std::endl; - } - return processed; - } - -} - -#if defined(_WIN32) -// In Windows, wmain() is an alternate entry point for main(), and receives the same parameters -// as main() but encoded in Windows Unicode (UTF-16); "wide" 16-bit wchar_t characters. The -// WindowsCommandLine object converts these wide character strings to a UTF-8 coded equivalent -// and makes them available through the argv() and envp() members. This enables toolMain() -// to process UTF-8 encoded arguments and environment variables without regard to platform. -int wmain(int argc, wchar_t* argvW[], wchar_t* envpW[]) { - mongo::WindowsCommandLine wcl(argc, argvW, envpW); - auto_ptr<Tool> instance = (*Tool::createInstance)(); - int exitCode = instance->main(argc, wcl.argv(), wcl.envp()); - quickExit(exitCode); -} - -#else -int main(int argc, char* argv[], char** envp) { - auto_ptr<Tool> instance = (*Tool::createInstance)(); - quickExit(instance->main(argc, argv, envp)); -} -#endif diff --git a/src/mongo/tools/tool.h b/src/mongo/tools/tool.h deleted file mode 100644 index df6bf671491..00000000000 --- a/src/mongo/tools/tool.h +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -// Tool.h - -#pragma once - -#include <iosfwd> -#include <string> - -#if defined(_WIN32) -#include <io.h> -#endif - -#include "mongo/db/instance.h" -#include "mongo/db/matcher/matcher.h" -#include "mongo/tools/tool_logger.h" -#include "mongo/tools/tool_options.h" -#include "mongo/util/options_parser/environment.h" - -using std::string; - -namespace mongo { - - class Tool { - public: - Tool(); - virtual ~Tool(); - - static std::auto_ptr<Tool> (*createInstance)(); - - int main( int argc , char ** argv, char ** envp ); - - std::string getNS() { - if (toolGlobalParams.coll.size() == 0) { - cerr << "no collection specified!" << std::endl; - throw -1; - } - return toolGlobalParams.db + "." + toolGlobalParams.coll; - } - - std::string getAuthenticationDatabase(); - - bool isMaster(); - bool isMongos(); - - virtual int run() = 0; - - virtual void printHelp(std::ostream &out) = 0; - - protected: - - mongo::DBClientBase &conn( bool slaveIfPaired = false ); - - bool _autoreconnect; - - protected: - - mongo::DBClientBase * _conn; - mongo::DBClientBase * _slaveConn; - - scoped_ptr<OperationContext> _txn; - - private: - void auth(); - }; - - class BSONTool : public Tool { - std::auto_ptr<Matcher> _matcher; - - public: - BSONTool(); - - virtual int doRun() = 0; - virtual void gotObject(const BSONObj& obj, std::ostream* out) = 0; - - virtual int run(); - - /** - * Read BSON documents from file and invoke gotObject() for every document. - * Output stream will be passed along to gotObject(). - * If output stream is not provided, a NULL output stream will be passed to gotObject(). - */ - long long processFile(const boost::filesystem::path& file, std::ostream* out); - - }; - -} - -#define REGISTER_MONGO_TOOL(TYPENAME) \ - std::auto_ptr<Tool> createInstanceOfThisTool() {return std::auto_ptr<Tool>(new TYPENAME());} \ - std::auto_ptr<Tool> (*Tool::createInstance)() = createInstanceOfThisTool; diff --git a/src/mongo/tools/tool_logger.cpp b/src/mongo/tools/tool_logger.cpp deleted file mode 100644 index 523fda44fd5..00000000000 --- a/src/mongo/tools/tool_logger.cpp +++ /dev/null @@ -1,156 +0,0 @@ -/* Copyright 2013 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/platform/basic.h" - -#include "mongo/tools/tool_logger.h" - -#include <iostream> - -#include "mongo/base/init.h" -#include "mongo/logger/console_appender.h" -#include "mongo/logger/log_manager.h" -#include "mongo/logger/logger.h" -#include "mongo/logger/message_event.h" -#include "mongo/logger/message_event_utf8_encoder.h" -#include "mongo/tools/tool_options.h" -#include "mongo/util/log.h" - -namespace mongo { -namespace { - - /* - * Theory of operation: - * - * At process start, the loader initializes "consoleMutex" to NULL. At some point during static - * initialization, the static initialization process, running in the one and only extant thread, - * allocates a new boost::mutex on the heap and assigns consoleMutex to point to it. While - * consoleMutex is still NULL, we know that there is only one thread extant, so it is safe to - * skip locking the consoleMutex in the ErrorConsole constructor. Once the mutex is initialized, - * users of ErrorConsole can start acquiring it. - */ - - boost::mutex *consoleMutex = new boost::mutex; - -} // namespace - - ErrorConsole::ErrorConsole() : _consoleLock() { - if (consoleMutex) { - boost::unique_lock<boost::mutex> lk(*consoleMutex); - lk.swap(_consoleLock); - } - } - - std::ostream& ErrorConsole::out() { return std::cerr; } - -namespace { - - logger::MessageLogDomain* toolErrorOutput = NULL; - logger::MessageLogDomain* toolNonErrorOutput = NULL; - logger::MessageLogDomain* toolNonErrorDecoratedOutput = NULL; - -} // namespace - -MONGO_INITIALIZER_GENERAL(ToolLogRedirection, - ("GlobalLogManager", "EndStartupOptionHandling"), - ("default"))(InitializerContext*) { - - using logger::MessageEventEphemeral; - using logger::MessageEventDetailsEncoder; - using logger::MessageLogDomain; - using logger::ConsoleAppender; - - toolErrorOutput = logger::globalLogManager()->getNamedDomain("toolErrorOutput"); - toolNonErrorOutput = logger::globalLogManager()->getNamedDomain("toolNonErrorOutput"); - toolNonErrorDecoratedOutput = - logger::globalLogManager()->getNamedDomain("toolNonErrorDecoratedOutput"); - - // Errors in the tools always go to stderr - toolErrorOutput->attachAppender( - MessageLogDomain::AppenderAutoPtr( - new ConsoleAppender<MessageEventEphemeral, ErrorConsole>( - new logger::MessageEventUnadornedEncoder))); - - // If we are outputting data to stdout, we may need to redirect all logging to stderr - if (!toolGlobalParams.canUseStdout) { - logger::globalLogDomain()->clearAppenders(); - logger::globalLogDomain()->attachAppender(MessageLogDomain::AppenderAutoPtr( - new ConsoleAppender<MessageEventEphemeral, ErrorConsole>( - new MessageEventDetailsEncoder))); - } - - // Only put an appender on our informational messages if we did not use --quiet - if (!toolGlobalParams.quiet) { - if (toolGlobalParams.canUseStdout) { - // If we can use stdout, we can use the ConsoleAppender with the default console - toolNonErrorOutput->attachAppender( - MessageLogDomain::AppenderAutoPtr( - new ConsoleAppender<MessageEventEphemeral>( - new logger::MessageEventUnadornedEncoder))); - - toolNonErrorDecoratedOutput->attachAppender( - MessageLogDomain::AppenderAutoPtr( - new ConsoleAppender<MessageEventEphemeral>( - new logger::MessageEventDetailsEncoder))); - } - else { - // If we cannot use stdout, we have to use ErrorConsole to redirect informational - // messages to stderr - toolNonErrorOutput->attachAppender( - MessageLogDomain::AppenderAutoPtr( - new ConsoleAppender<MessageEventEphemeral, ErrorConsole>( - new logger::MessageEventUnadornedEncoder))); - - toolNonErrorDecoratedOutput->attachAppender( - MessageLogDomain::AppenderAutoPtr( - new ConsoleAppender<MessageEventEphemeral, ErrorConsole>( - new logger::MessageEventDetailsEncoder))); - } - } - - return Status::OK(); -} - - LogstreamBuilder toolInfoLog() { - return LogstreamBuilder(toolNonErrorDecoratedOutput, - "", - logger::LogSeverity::Log()); - } - - LogstreamBuilder toolInfoOutput() { - return LogstreamBuilder(toolNonErrorOutput, - "", - logger::LogSeverity::Log()); - } - - LogstreamBuilder toolError() { - return LogstreamBuilder(toolErrorOutput, - "", - logger::LogSeverity::Log()); - } - -} // namespace mongo diff --git a/src/mongo/tools/tool_logger.h b/src/mongo/tools/tool_logger.h deleted file mode 100644 index 3bbdfa5d97b..00000000000 --- a/src/mongo/tools/tool_logger.h +++ /dev/null @@ -1,70 +0,0 @@ -/* Copyright 2013 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#pragma once - -#include <boost/thread/mutex.hpp> -#include <iosfwd> - -#include "mongo/logger/logstream_builder.h" - -namespace mongo { - - /** - * This is a version of the Console class that uses stderr for output instead of stdout. See - * the description of the Console class for other details about how this class should operate - */ - class ErrorConsole { - public: - ErrorConsole(); - - std::ostream& out(); - - private: - boost::unique_lock<boost::mutex> _consoleLock; - }; - - using logger::LogstreamBuilder; - - /* - * Informational messages. Messages sent here will go to stdout normally, stderr if data is - * being sent to stdout, and be silenced if the user specifies --quiet. - */ - LogstreamBuilder toolInfoOutput(); - /* - * Informational messages. Messages sent here will go to stdout normally, stderr if data is - * being sent to stdout, and be silenced if the user specifies --quiet. Incudes extra log - * decoration. - */ - LogstreamBuilder toolInfoLog(); - /* - * Error messages. Messages sent here should always go to stderr and not be silenced by - * --quiet. - */ - LogstreamBuilder toolError(); - -} // namespace mongo diff --git a/src/mongo/tools/tool_options.cpp b/src/mongo/tools/tool_options.cpp deleted file mode 100644 index 37819201ebe..00000000000 --- a/src/mongo/tools/tool_options.cpp +++ /dev/null @@ -1,402 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#include "mongo/tools/tool_options.h" - -#include <boost/filesystem/operations.hpp> -#include <fstream> -#include "pcrecpp.h" - -#include "mongo/base/status.h" -#include "mongo/client/sasl_client_authenticate.h" -#include "mongo/db/storage_options.h" -#include "mongo/util/log.h" -#include "mongo/util/net/sock.h" -#include "mongo/util/net/ssl_manager.h" -#include "mongo/util/net/ssl_options.h" -#include "mongo/util/options_parser/startup_options.h" -#include "mongo/util/password.h" -#include "mongo/util/text.h" -#include "mongo/util/version.h" - -namespace mongo { - - ToolGlobalParams toolGlobalParams; - BSONToolGlobalParams bsonToolGlobalParams; - - Status addGeneralToolOptions(moe::OptionSection* options) { - options->addOptionChaining("help", "help", moe::Switch, "produce help message"); - - options->addOptionChaining("verbose", "verbose,v", moe::Switch, - "be more verbose (include multiple times " - "for more verbosity e.g. -vvvvv)"); - - options->addOptionChaining("quiet", "quiet", moe::Switch, - "silence all non error diagnostic messages"); - - options->addOptionChaining("version", "version", moe::Switch, - "print the program's version and exit"); - - - /* support for -vv -vvvv etc. */ - for (string s = "vv"; s.length() <= 10; s.append("v")) { - options->addOptionChaining(s.c_str(), s.c_str(), moe::Switch, "verbose") - .hidden(); - } - - return Status::OK(); - } - - Status addRemoteServerToolOptions(moe::OptionSection* options) { - options->addOptionChaining("host", "host,h", moe::String, - "mongo host to connect to ( <set name>/s1,s2 for sets)") - .incompatibleWith("dbpath"); - - options->addOptionChaining("port", "port", moe::Int, - "server port. Can also use --host hostname:port") - .validRange(0, 65535); - - options->addOptionChaining("ipv6", "ipv6", moe::Switch, - "enable IPv6 support (disabled by default)"); - - -#ifdef MONGO_SSL - Status ret = addSSLClientOptions(options); - if (!ret.isOK()) { - return ret; - } -#endif - - options->addOptionChaining("username", "username,u", moe::String, - "username"); - - // We ask a user for a password if they pass in an empty string or pass --password with no - // argument. This must be handled when the password value is checked. - // - // Desired behavior: - // --username test // Continue with username "test" and no password - // --username test --password test // Continue with username "test" and password "test" - // --username test --password // Continue with username "test" and prompt for password - // --username test --password "" // Continue with username "test" and prompt for password - // - // To do this we pass moe::Value(std::string("")) as the "implicit value" of this option - options->addOptionChaining("password", "password,p", moe::String, "password") - .setImplicit(moe::Value(std::string(""))); - - options->addOptionChaining("authenticationDatabase", "authenticationDatabase", moe::String, - "user source (defaults to dbname)") - .setDefault(moe::Value(std::string(""))); - - options->addOptionChaining("authenticationMechanism", "authenticationMechanism", - moe::String, "authentication mechanism") - .setDefault(moe::Value(std::string("MONGODB-CR"))); - - options->addOptionChaining("gssapiServiceName", "gssapiServiceName", - moe::String, - "Service name to use when authenticating using GSSAPI/Kerberos") - .setDefault(moe::Value(std::string(saslDefaultServiceName))); - - options->addOptionChaining("gssapiHostName", "gssapiHostName", moe::String, - "Remote host name to use for purpose of GSSAPI/Kerberos authentication"); - - return Status::OK(); - } - - Status addLocalServerToolOptions(moe::OptionSection* options) { - options->addOptionChaining("dbpath", "dbpath", moe::String, - "directly access mongod database files in the given path, instead of " - "connecting to a mongod server - needs to lock the data directory, " - "so cannot be used if a mongod is currently accessing the same path") - .incompatibleWith("host"); - - options->addOptionChaining("directoryperdb", "directoryperdb", moe::Switch, - "each db is in a separate directory (relevant only if dbpath specified)"); - - options->addOptionChaining("journal", "journal", moe::Switch, - "enable journaling (relevant only if dbpath specified)"); - - - return Status::OK(); - } - Status addSpecifyDBCollectionToolOptions(moe::OptionSection* options) { - options->addOptionChaining("db", "db,d", moe::String, "database to use"); - - options->addOptionChaining("collection", "collection,c", moe::String, - "collection to use (some commands)"); - - - return Status::OK(); - } - - Status addFieldOptions(moe::OptionSection* options) { - options->addOptionChaining("fields", "fields,f", moe::String, - "comma separated list of field names e.g. -f name,age"); - - options->addOptionChaining("fieldFile", "fieldFile", moe::String, - "file with field names - 1 per line"); - - - return Status::OK(); - } - - Status addBSONToolOptions(moe::OptionSection* options) { - options->addOptionChaining("objcheck", "objcheck", moe::Switch, - "validate object before inserting (default)"); - - options->addOptionChaining("noobjcheck", "noobjcheck", moe::Switch, - "don't validate object before inserting"); - - options->addOptionChaining("filter", "filter", moe::String, - "filter to apply before inserting"); - - - return Status::OK(); - } - - std::string getParam(std::string name, std::string def) { - if (moe::startupOptionsParsed.count(name)) { - return moe::startupOptionsParsed[name.c_str()].as<string>(); - } - return def; - } - int getParam(std::string name, int def) { - if (moe::startupOptionsParsed.count(name)) { - return moe::startupOptionsParsed[name.c_str()].as<int>(); - } - return def; - } - bool hasParam(std::string name) { - return moe::startupOptionsParsed.count(name); - } - - void printToolVersionString(std::ostream &out) { - out << toolGlobalParams.name << " version " << mongo::versionString; - if (mongo::versionString[strlen(mongo::versionString)-1] == '-') - out << " (commit " << mongo::gitVersion() << ")"; - out << std::endl; - } - - bool handlePreValidationGeneralToolOptions(const moe::Environment& params) { - if (moe::startupOptionsParsed.count("version")) { - printToolVersionString(std::cout); - return false; - } - return true; - } - - extern bool directoryperdb; - - Status storeGeneralToolOptions(const moe::Environment& params, - const std::vector<std::string>& args) { - - toolGlobalParams.name = args[0]; - - storageGlobalParams.prealloc = false; - - // The default value may vary depending on compile options, but for tools - // we want durability to be disabled. - storageGlobalParams.dur = false; - - // Set authentication parameters - if (params.count("authenticationDatabase")) { - toolGlobalParams.authenticationDatabase = - params["authenticationDatabase"].as<string>(); - } - - if (params.count("authenticationMechanism")) { - toolGlobalParams.authenticationMechanism = - params["authenticationMechanism"].as<string>(); - } - - if (params.count("gssapiServiceName")) { - toolGlobalParams.gssapiServiceName = params["gssapiServiceName"].as<string>(); - } - - if (params.count("gssapiHostName")) { - toolGlobalParams.gssapiHostName = params["gssapiHostName"].as<string>(); - } - if (params.count("verbose")) { - logger::globalLogDomain()->setMinimumLoggedSeverity(logger::LogSeverity::Debug(1)); - } - - for (string s = "vv"; s.length() <= 12; s.append("v")) { - if (params.count(s)) { - logger::globalLogDomain()->setMinimumLoggedSeverity( - logger::LogSeverity::Debug(s.length())); - } - } - - toolGlobalParams.quiet = params.count("quiet"); - -#ifdef MONGO_SSL - Status ret = storeSSLClientOptions(params); - if (!ret.isOK()) { - return ret; - } -#endif - - if (args.empty()) { - return Status(ErrorCodes::InternalError, "Cannot get binary name: argv array is empty"); - } - - // setup binary name - toolGlobalParams.name = args[0]; - size_t i = toolGlobalParams.name.rfind('/'); - if (i != string::npos) { - toolGlobalParams.name = toolGlobalParams.name.substr(i + 1); - } - toolGlobalParams.db = "test"; - toolGlobalParams.coll = ""; - toolGlobalParams.noconnection = false; - - if (params.count("db")) - toolGlobalParams.db = params["db"].as<string>(); - - if (params.count("collection")) - toolGlobalParams.coll = params["collection"].as<string>(); - - if (params.count("username")) - toolGlobalParams.username = params["username"].as<string>(); - - if (params.count("password")) { - toolGlobalParams.password = params["password"].as<string>(); - if (toolGlobalParams.password.empty()) { - toolGlobalParams.password = askPassword(); - } - } - - if (params.count("ipv6")) { - enableIPv6(); - } - - toolGlobalParams.dbpath = getParam("dbpath"); - toolGlobalParams.useDirectClient = hasParam("dbpath"); - if (toolGlobalParams.useDirectClient && params.count("journal")) { - storageGlobalParams.dur = true; - } - - if (!toolGlobalParams.useDirectClient) { - toolGlobalParams.connectionString = "127.0.0.1"; - if (params.count("host")) { - toolGlobalParams.hostSet = true; - toolGlobalParams.host = params["host"].as<string>(); - toolGlobalParams.connectionString = params["host"].as<string>(); - } - - if (params.count("port")) { - toolGlobalParams.portSet = true; - toolGlobalParams.port = params["port"].as<int>(); - StringBuilder sb; - sb << toolGlobalParams.connectionString << ':' << toolGlobalParams.port; - toolGlobalParams.connectionString = sb.str(); - } - } - else { - if (params.count("directoryperdb")) { - storageGlobalParams.directoryperdb = true; - } - - toolGlobalParams.connectionString = "DIRECT"; - } - - return Status::OK(); - } - - Status storeFieldOptions(const moe::Environment& params, - const std::vector<std::string>& args) { - - toolGlobalParams.fieldsSpecified = false; - - if (hasParam("fields")) { - toolGlobalParams.fieldsSpecified = true; - - string fields_arg = getParam("fields"); - pcrecpp::StringPiece input(fields_arg); - - string f; - pcrecpp::RE re("([#\\w\\.\\s\\-]+),?" ); - while ( re.Consume( &input, &f ) ) { - toolGlobalParams.fields.push_back( f ); - } - return Status::OK(); - } - - if (hasParam("fieldFile")) { - toolGlobalParams.fieldsSpecified = true; - - string fn = getParam("fieldFile"); - if (!boost::filesystem::exists(fn)) { - StringBuilder sb; - sb << "file: " << fn << " doesn't exist"; - return Status(ErrorCodes::InternalError, sb.str()); - } - - const int BUF_SIZE = 1024; - char line[1024 + 128]; - std::ifstream file(fn.c_str()); - - while (file.rdstate() == std::ios_base::goodbit) { - file.getline(line, BUF_SIZE); - const char * cur = line; - while (isspace(cur[0])) cur++; - if (cur[0] == '\0') - continue; - - toolGlobalParams.fields.push_back(cur); - } - return Status::OK(); - } - - return Status::OK(); - } - - - Status storeBSONToolOptions(const moe::Environment& params, - const std::vector<std::string>& args) { - - bsonToolGlobalParams.objcheck = true; - - if (hasParam("objcheck") && hasParam("noobjcheck")) { - return Status(ErrorCodes::BadValue, "can't have both --objcheck and --noobjcheck"); - } - - if (hasParam("objcheck")) { - bsonToolGlobalParams.objcheck = true; - } - else if (hasParam("noobjcheck")) { - bsonToolGlobalParams.objcheck = false; - } - - if (hasParam("filter")) { - bsonToolGlobalParams.hasFilter = true; - bsonToolGlobalParams.filter = getParam("filter"); - } - - return Status::OK(); - } -} diff --git a/src/mongo/tools/tool_options.h b/src/mongo/tools/tool_options.h deleted file mode 100644 index f93cb03f7e7..00000000000 --- a/src/mongo/tools/tool_options.h +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (C) 2010 10gen Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License, version 3, - * as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see <http://www.gnu.org/licenses/>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the GNU Affero General Public License in all respects - * for all of the code used other than as permitted herein. If you modify - * file(s) with this exception, you may extend this exception to your - * version of the file(s), but you are not obligated to do so. If you do not - * wish to do so, delete this exception statement from your version. If you - * delete this exception statement from all source files in the program, - * then also delete it in the license file. - */ - -#pragma once - -#include <iosfwd> -#include <string> -#include <vector> - -#include "mongo/base/status.h" - -namespace mongo { - - namespace optionenvironment { - class OptionSection; - class Environment; - } // namespace optionenvironment - - namespace moe = mongo::optionenvironment; - - struct ToolGlobalParams { - - ToolGlobalParams() : canUseStdout(true), hostSet(false), portSet(false) { } - std::string name; - - std::string db; - std::string coll; - - std::string username; - std::string password; - std::string authenticationDatabase; - std::string authenticationMechanism; - std::string gssapiServiceName; - std::string gssapiHostName; - - bool quiet; - bool canUseStdout; - bool noconnection; - - std::vector<std::string> fields; - bool fieldsSpecified; - - std::string host; // --host - bool hostSet; - int port; // --port - bool portSet; - std::string connectionString; // --host and --port after processing - std::string dbpath; - bool useDirectClient; - }; - - extern ToolGlobalParams toolGlobalParams; - - struct BSONToolGlobalParams { - bool objcheck; - std::string filter; - bool hasFilter; - }; - - extern BSONToolGlobalParams bsonToolGlobalParams; - - Status addGeneralToolOptions(moe::OptionSection* options); - - Status addRemoteServerToolOptions(moe::OptionSection* options); - - Status addLocalServerToolOptions(moe::OptionSection* options); - - Status addSpecifyDBCollectionToolOptions(moe::OptionSection* options); - - Status addBSONToolOptions(moe::OptionSection* options); - - Status addFieldOptions(moe::OptionSection* options); - - // Legacy interface for getting options in tools - // TODO: Remove this when we use the new interface everywhere - std::string getParam(std::string name, std::string def=""); - int getParam(std::string name, int def); - bool hasParam(std::string name); - - /** - * Handle options that should come before validation, such as "help". - * - * Returns false if an option was found that implies we should prematurely exit with success. - */ - bool handlePreValidationGeneralToolOptions(const moe::Environment& params); - - Status storeGeneralToolOptions(const moe::Environment& params, - const std::vector<std::string>& args); - - Status storeFieldOptions(const moe::Environment& params, - const std::vector<std::string>& args); - - Status storeBSONToolOptions(const moe::Environment& params, - const std::vector<std::string>& args); -} diff --git a/src/mongo/tools/top.cpp b/src/mongo/tools/top.cpp deleted file mode 100644 index fc8ccc105c2..00000000000 --- a/src/mongo/tools/top.cpp +++ /dev/null @@ -1,184 +0,0 @@ -/** -* Copyright (C) 2008 10gen Inc. -* -* This program is free software: you can redistribute it and/or modify -* it under the terms of the GNU Affero General Public License, version 3, -* as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU Affero General Public License for more details. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see <http://www.gnu.org/licenses/>. -* -* As a special exception, the copyright holders give permission to link the -* code of portions of this program with the OpenSSL library under certain -* conditions as described in each individual source file and distribute -* linked combinations including the program with the OpenSSL library. You -* must comply with the GNU Affero General Public License in all respects -* for all of the code used other than as permitted herein. If you modify -* file(s) with this exception, you may extend this exception to your -* version of the file(s), but you are not obligated to do so. If you do not -* wish to do so, delete this exception statement from your version. If you -* delete this exception statement from all source files in the program, -* then also delete it in the license file. -*/ - -#include "mongo/platform/basic.h" - -#include <fstream> -#include <iomanip> -#include <iostream> - -#include "mongo/db/json.h" -#include "mongo/tools/stat_util.h" -#include "mongo/tools/mongotop_options.h" -#include "mongo/tools/tool.h" -#include "mongo/util/options_parser/option_section.h" - -namespace mongo { - - class TopTool : public Tool { - public: - - TopTool() : Tool() { - _autoreconnect = true; - } - - virtual void printHelp( ostream & out ) { - printMongoTopHelp(&out); - } - - NamespaceStats getData() { - if (mongoTopGlobalParams.useLocks) - return getDataLocks(); - return getDataTop(); - } - - NamespaceStats getDataLocks() { - - BSONObj out; - if (!conn().simpleCommand(toolGlobalParams.db, &out, "serverStatus")) { - toolError() << "error: " << out << std::endl; - return NamespaceStats(); - } - - return StatUtil::parseServerStatusLocks( out.getOwned() ); - } - - NamespaceStats getDataTop() { - NamespaceStats stats; - - BSONObj out; - if (!conn().simpleCommand(toolGlobalParams.db, &out, "top")) { - toolError() << "error: " << out << std::endl; - return stats; - } - - if ( ! out["totals"].isABSONObj() ) { - toolError() << "error: invalid top\n" << out << std::endl; - return stats; - } - - out = out["totals"].Obj().getOwned(); - - BSONObjIterator i( out ); - while ( i.more() ) { - BSONElement e = i.next(); - if ( ! e.isABSONObj() ) - continue; - - NamespaceInfo& s = stats[e.fieldName()]; - s.ns = e.fieldName(); - s.read = e.Obj()["readLock"].Obj()["time"].numberLong() / 1000; - s.write = e.Obj()["writeLock"].Obj()["time"].numberLong() / 1000; - } - - return stats; - } - - void printDiff( const NamespaceStats& prev , const NamespaceStats& now ) { - if ( prev.size() == 0 || now.size() == 0 ) { - cout << "." << endl; - return; - } - - vector<NamespaceDiff> data = StatUtil::computeDiff( prev , now ); - - unsigned longest = 30; - - for ( unsigned i=0; i < data.size(); i++ ) { - const string& ns = data[i].ns; - - if (!mongoTopGlobalParams.useLocks && ns.find('.') == string::npos) - continue; - - if ( ns.size() > longest ) - longest = ns.size(); - } - - int numberWidth = 10; - - cout << "\n" - << setw(longest) << (mongoTopGlobalParams.useLocks ? "db" : "ns") - << setw(numberWidth+2) << "total" - << setw(numberWidth+2) << "read" - << setw(numberWidth+2) << "write" - << "\t\t" << terseCurrentTime() - << endl; - for ( int i=data.size()-1; i>=0 && data.size() - i < 10 ; i-- ) { - - if (!mongoTopGlobalParams.useLocks && data[i].ns.find('.') == string::npos) - continue; - - cout << setw(longest) << data[i].ns - << setw(numberWidth) << setprecision(3) << data[i].total() << "ms" - << setw(numberWidth) << setprecision(3) << data[i].read << "ms" - << setw(numberWidth) << setprecision(3) << data[i].write << "ms" - << endl; - } - - } - - int run() { - if (isMongos()) { - toolError() << "mongotop only works on instances of mongod." << std::endl; - return EXIT_FAILURE; - } - - NamespaceStats prev = getData(); - - while ( true ) { - sleepsecs(mongoTopGlobalParams.sleep); - - NamespaceStats now; - try { - now = getData(); - } - catch ( std::exception& e ) { - toolError() << "can't get data: " << e.what() << std::endl; - continue; - } - - if ( now.size() == 0 ) - return -2; - - try { - printDiff( prev , now ); - } - catch ( AssertionException& e ) { - toolError() << "\nerror: " << e.what() << std::endl; - } - - prev = now; - } - - return 0; - } - }; - - REGISTER_MONGO_TOOL(TopTool); - -} |