summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorNathan Myers <ncm@cantrip.org>2017-05-15 11:27:06 -0400
committerNathan Myers <nathan.myers@10gen.com>2017-05-15 17:44:30 -0400
commit5d5767455f884144a5895d5fcc64fcdc97ae50fe (patch)
tree19607fb2c889c008d27c69dee668570f883c8c9b /src
parentc88c4809c2440d286ed0fc29e1e8d684f015e563 (diff)
downloadmongo-5d5767455f884144a5895d5fcc64fcdc97ae50fe.tar.gz
SERVER-27760 Delete old range deleter sources
Diffstat (limited to 'src')
-rw-r--r--src/mongo/db/db.cpp1
-rw-r--r--src/mongo/db/range_deleter.cpp592
-rw-r--r--src/mongo/db/range_deleter.h334
-rw-r--r--src/mongo/db/range_deleter_db_env.h65
-rw-r--r--src/mongo/db/range_deleter_mock_env.cpp146
-rw-r--r--src/mongo/db/range_deleter_mock_env.h167
-rw-r--r--src/mongo/db/range_deleter_service.cpp49
-rw-r--r--src/mongo/db/range_deleter_service.h39
-rw-r--r--src/mongo/db/range_deleter_test.cpp339
-rw-r--r--src/mongo/db/s/cleanup_orphaned_cmd.cpp1
-rw-r--r--src/mongo/db/s/migration_destination_manager.cpp1
-rw-r--r--src/mongo/db/s/migration_destination_manager_legacy_commands.cpp1
-rw-r--r--src/mongo/db/s/move_chunk_command.cpp1
-rw-r--r--src/mongo/db/stats/range_deleter_server_status.cpp98
14 files changed, 0 insertions, 1834 deletions
diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index 34f18c08e80..2981e14ac43 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -79,7 +79,6 @@
#include "mongo/db/op_observer_impl.h"
#include "mongo/db/operation_context.h"
#include "mongo/db/query/internal_plans.h"
-#include "mongo/db/range_deleter_service.h"
#include "mongo/db/repair_database.h"
#include "mongo/db/repl/oplog.h"
#include "mongo/db/repl/repl_settings.h"
diff --git a/src/mongo/db/range_deleter.cpp b/src/mongo/db/range_deleter.cpp
deleted file mode 100644
index 19af20ffae1..00000000000
--- a/src/mongo/db/range_deleter.cpp
+++ /dev/null
@@ -1,592 +0,0 @@
-/**
- * Copyright (C) 2013 10gen Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the GNU Affero General Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kSharding
-
-#include "mongo/platform/basic.h"
-
-#include "mongo/db/range_deleter.h"
-
-#include <boost/date_time/posix_time/posix_time_duration.hpp>
-#include <memory>
-
-#include "mongo/db/client.h"
-#include "mongo/db/repl/replication_coordinator_global.h"
-#include "mongo/db/service_context.h"
-#include "mongo/db/write_concern_options.h"
-#include "mongo/stdx/memory.h"
-#include "mongo/util/concurrency/idle_thread_block.h"
-#include "mongo/util/exit.h"
-#include "mongo/util/log.h"
-#include "mongo/util/mongoutils/str.h"
-#include "mongo/util/time_support.h"
-#include "mongo/util/timer.h"
-
-using std::unique_ptr;
-using std::endl;
-using std::set;
-using std::pair;
-using std::string;
-
-namespace {
-const long int kNotEmptyTimeoutMillis = 200;
-const long long int kMaxCursorCheckIntervalMillis = 500;
-const size_t kDeleteJobsHistory = 10; // entries
-
-/**
- * Removes an element from the container that holds a pointer type, and deletes the
- * pointer as well. Returns true if the element was found.
- */
-template <typename ContainerType, typename ContainerElementType>
-bool deletePtrElement(ContainerType* container, ContainerElementType elem) {
- typename ContainerType::iterator iter = container->find(elem);
-
- if (iter == container->end()) {
- return false;
- }
-
- delete *iter;
- container->erase(iter);
- return true;
-}
-}
-
-namespace mongo {
-
-namespace duration = boost::posix_time;
-
-static void logCursorsWaiting(RangeDeleteEntry* entry) {
- // We always log the first cursors waiting message (so we have cursor ids in the logs).
- // After 15 minutes (the cursor timeout period), we start logging additional messages at
- // a 1 minute interval.
- static const auto kLogCursorsThreshold = Minutes{15};
- static const auto kLogCursorsInterval = Minutes{1};
-
- Date_t currentTime = jsTime();
- Milliseconds elapsedMillisSinceQueued{0};
-
- // We always log the first message when lastLoggedTime == 0
- if (entry->lastLoggedTS != Date_t()) {
- if (currentTime > entry->stats.queueStartTS)
- elapsedMillisSinceQueued = currentTime - entry->stats.queueStartTS;
-
- // Not logging, threshold not passed
- if (elapsedMillisSinceQueued < kLogCursorsThreshold)
- return;
-
- Milliseconds elapsedMillisSinceLog{0};
- if (currentTime > entry->lastLoggedTS)
- elapsedMillisSinceLog = currentTime - entry->lastLoggedTS;
-
- // Not logging, logged a short time ago
- if (elapsedMillisSinceLog < kLogCursorsInterval)
- return;
- }
-
- str::stream cursorList;
- for (std::set<CursorId>::const_iterator it = entry->cursorsToWait.begin();
- it != entry->cursorsToWait.end();
- ++it) {
- if (it != entry->cursorsToWait.begin())
- cursorList << ", ";
- cursorList << *it;
- }
-
- log() << "waiting for open cursors before removing range "
- << "[" << redact(entry->options.range.minKey) << ", "
- << redact(entry->options.range.maxKey) << ") "
- << "in " << entry->options.range.ns
- << (entry->lastLoggedTS == Date_t()
- ? string("")
- : string(str::stream() << ", elapsed secs: "
- << durationCount<Seconds>(elapsedMillisSinceQueued)))
- << ", cursor ids: [" << string(cursorList) << "]";
-
- entry->lastLoggedTS = currentTime;
-}
-
-struct RangeDeleter::NSMinMax {
- NSMinMax(std::string ns, const BSONObj min, const BSONObj max) : ns(ns), min(min), max(max) {}
-
- std::string ns;
-
- // Inclusive lower range.
- BSONObj min;
-
- // Exclusive upper range.
- BSONObj max;
-};
-
-bool RangeDeleter::NSMinMaxCmp::operator()(const NSMinMax* lhs, const NSMinMax* rhs) const {
- const int nsComp = lhs->ns.compare(rhs->ns);
-
- if (nsComp < 0) {
- return true;
- }
-
- if (nsComp > 0) {
- return false;
- }
-
- return compareRanges(lhs->min, lhs->max, rhs->min, rhs->max) < 0;
-}
-
-RangeDeleter::RangeDeleter(RangeDeleterEnv* env)
- : _env(env), // ownership xfer
- _stopRequested(false),
- _deletesInProgress(0) {}
-
-RangeDeleter::~RangeDeleter() {
- for (TaskList::iterator it = _notReadyQueue.begin(); it != _notReadyQueue.end(); ++it) {
- delete (*it);
- }
-
- for (TaskList::iterator it = _taskQueue.begin(); it != _taskQueue.end(); ++it) {
- delete (*it);
- }
-
- for (NSMinMaxSet::iterator it = _deleteSet.begin(); it != _deleteSet.end(); ++it) {
- delete (*it);
- }
-
- for (std::deque<DeleteJobStats*>::iterator it = _statsHistory.begin();
- it != _statsHistory.end();
- ++it) {
- delete (*it);
- }
-}
-
-void RangeDeleter::startWorkers() {
- if (!_worker) {
- _worker.reset(new stdx::thread(stdx::bind(&RangeDeleter::doWork, this)));
- }
-}
-
-void RangeDeleter::stopWorkers() {
- {
- stdx::lock_guard<stdx::mutex> sl(_stopMutex);
- _stopRequested = true;
- }
-
- if (_worker) {
- _worker->join();
- }
-
- stdx::unique_lock<stdx::mutex> sl(_queueMutex);
- while (_deletesInProgress > 0) {
- _nothingInProgressCV.wait(sl);
- }
-}
-
-bool RangeDeleter::queueDelete(OperationContext* opCtx,
- const RangeDeleterOptions& options,
- Notification<void>* doneSignal,
- std::string* errMsg) {
- string dummy;
- if (errMsg == NULL)
- errMsg = &dummy;
-
- const string& ns(options.range.ns);
- const BSONObj& min(options.range.minKey);
- const BSONObj& max(options.range.maxKey);
-
- unique_ptr<RangeDeleteEntry> toDelete(new RangeDeleteEntry(options));
- toDelete->doneSignal = doneSignal;
-
- {
- stdx::lock_guard<stdx::mutex> sl(_queueMutex);
- if (_stopRequested) {
- *errMsg = "deleter is already stopped.";
- return false;
- }
-
- if (!canEnqueue_inlock(ns, min, max, errMsg)) {
- return false;
- }
-
- _deleteSet.insert(new NSMinMax(ns, min.getOwned(), max.getOwned()));
- }
-
- if (options.waitForOpenCursors) {
- _env->getCursorIds(opCtx, ns, &toDelete->cursorsToWait);
- }
-
- toDelete->stats.queueStartTS = jsTime();
-
- if (!toDelete->cursorsToWait.empty())
- logCursorsWaiting(toDelete.get());
-
- {
- stdx::lock_guard<stdx::mutex> sl(_queueMutex);
-
- if (toDelete->cursorsToWait.empty()) {
- toDelete->stats.queueEndTS = jsTime();
- _taskQueue.push_back(toDelete.release());
- _taskQueueNotEmptyCV.notify_one();
- } else {
- _notReadyQueue.push_back(toDelete.release());
- }
- }
-
- return true;
-}
-
-namespace {
-const int kWTimeoutMillis = 60 * 60 * 1000;
-
-bool _waitForMajority(OperationContext* opCtx, std::string* errMsg) {
- const WriteConcernOptions writeConcern(
- WriteConcernOptions::kMajority, WriteConcernOptions::SyncMode::UNSET, kWTimeoutMillis);
-
- repl::ReplicationCoordinator::StatusAndDuration replStatus =
- repl::getGlobalReplicationCoordinator()->awaitReplicationOfLastOpForClient(opCtx,
- writeConcern);
- if (!replStatus.status.isOK()) {
- *errMsg = str::stream() << "rangeDeleter failed while waiting for replication after "
- << durationCount<Seconds>(replStatus.duration) << " seconds due to "
- << replStatus.status.toString();
- log() << *errMsg;
- }
-
- return replStatus.status.isOK();
-}
-} // namespace
-
-bool RangeDeleter::deleteNow(OperationContext* opCtx,
- const RangeDeleterOptions& options,
- string* errMsg) {
- if (stopRequested()) {
- *errMsg = "deleter is already stopped.";
- return false;
- }
-
- string dummy;
- if (errMsg == NULL)
- errMsg = &dummy;
-
- const string& ns(options.range.ns);
- const BSONObj& min(options.range.minKey);
- const BSONObj& max(options.range.maxKey);
-
- NSMinMax deleteRange(ns, min, max);
- {
- stdx::lock_guard<stdx::mutex> sl(_queueMutex);
- if (!canEnqueue_inlock(ns, min, max, errMsg)) {
- return false;
- }
-
- _deleteSet.insert(&deleteRange);
-
- // Note: count for pending deletes is an integral part of the shutdown story.
- // Therefore, to simplify things, there is no "pending" state for deletes in
- // deleteNow, the state transition is simply inProgress -> done.
- _deletesInProgress++;
- }
-
- RangeDeleteEntry taskDetails(options);
- if (options.waitForOpenCursors) {
- _env->getCursorIds(opCtx, ns, &taskDetails.cursorsToWait);
- }
-
- long long checkIntervalMillis = 5;
-
-
- taskDetails.stats.queueStartTS = jsTime();
-
- for (; !taskDetails.cursorsToWait.empty(); sleepmillis(checkIntervalMillis)) {
- logCursorsWaiting(&taskDetails);
-
- set<CursorId> cursorsNow;
- _env->getCursorIds(opCtx, ns, &cursorsNow);
-
- set<CursorId> cursorsLeft;
- std::set_intersection(taskDetails.cursorsToWait.begin(),
- taskDetails.cursorsToWait.end(),
- cursorsNow.begin(),
- cursorsNow.end(),
- std::inserter(cursorsLeft, cursorsLeft.end()));
-
- taskDetails.cursorsToWait.swap(cursorsLeft);
-
- if (stopRequested()) {
- *errMsg = "deleter was stopped.";
-
- stdx::lock_guard<stdx::mutex> sl(_queueMutex);
- _deleteSet.erase(&deleteRange);
-
- _deletesInProgress--;
-
- if (_deletesInProgress == 0) {
- _nothingInProgressCV.notify_one();
- }
-
- return false;
- }
-
- if (checkIntervalMillis < kMaxCursorCheckIntervalMillis) {
- checkIntervalMillis *= 2;
- }
- }
- taskDetails.stats.queueEndTS = jsTime();
-
- taskDetails.stats.deleteStartTS = jsTime();
- bool result = _env->deleteRange(opCtx, taskDetails, &taskDetails.stats.deletedDocCount, errMsg);
-
- taskDetails.stats.deleteEndTS = jsTime();
-
- if (result) {
- taskDetails.stats.waitForReplStartTS = jsTime();
- result = _waitForMajority(opCtx, errMsg);
- taskDetails.stats.waitForReplEndTS = jsTime();
- }
-
- {
- stdx::lock_guard<stdx::mutex> sl(_queueMutex);
- _deleteSet.erase(&deleteRange);
-
- _deletesInProgress--;
-
- if (_deletesInProgress == 0) {
- _nothingInProgressCV.notify_one();
- }
- }
-
- recordDelStats(new DeleteJobStats(taskDetails.stats));
- return result;
-}
-
-std::vector<std::unique_ptr<DeleteJobStats>> RangeDeleter::getStatsHistory() const {
- std::vector<std::unique_ptr<DeleteJobStats>> stats;
- stats.reserve(kDeleteJobsHistory);
-
- stdx::lock_guard<stdx::mutex> sl(_statsHistoryMutex);
- for (std::deque<DeleteJobStats*>::const_iterator it = _statsHistory.begin();
- it != _statsHistory.end();
- ++it) {
- stats.push_back(stdx::make_unique<DeleteJobStats>(**it));
- }
- return stats;
-}
-
-BSONObj RangeDeleter::toBSON() const {
- stdx::lock_guard<stdx::mutex> sl(_queueMutex);
-
- BSONObjBuilder builder;
-
- BSONArrayBuilder notReadyBuilder(builder.subarrayStart("notReady"));
- for (TaskList::const_iterator iter = _notReadyQueue.begin(); iter != _notReadyQueue.end();
- ++iter) {
- notReadyBuilder.append((*iter)->toBSON());
- }
- notReadyBuilder.doneFast();
-
- BSONArrayBuilder readyBuilder(builder.subarrayStart("ready"));
- for (TaskList::const_iterator iter = _taskQueue.begin(); iter != _taskQueue.end(); ++iter) {
- readyBuilder.append((*iter)->toBSON());
- }
- readyBuilder.doneFast();
-
- return builder.obj();
-}
-
-void RangeDeleter::doWork() {
- Client::initThreadIfNotAlready("RangeDeleter");
- Client* client = &cc();
-
- while (!globalInShutdownDeprecated() && !stopRequested()) {
- string errMsg;
-
- RangeDeleteEntry* nextTask = NULL;
-
- {
- stdx::unique_lock<stdx::mutex> sl(_queueMutex);
- while (_taskQueue.empty()) {
- {
- MONGO_IDLE_THREAD_BLOCK;
- _taskQueueNotEmptyCV.wait_for(
- sl, Milliseconds(kNotEmptyTimeoutMillis).toSystemDuration());
- }
-
- if (stopRequested()) {
- log() << "stopping range deleter worker" << endl;
- return;
- }
-
- if (_taskQueue.empty()) {
- // Try to check if some deletes are ready and move them to the
- // ready queue.
-
- TaskList::iterator iter = _notReadyQueue.begin();
- while (iter != _notReadyQueue.end()) {
- RangeDeleteEntry* entry = *iter;
-
- set<CursorId> cursorsNow;
- if (entry->options.waitForOpenCursors) {
- auto opCtx = client->makeOperationContext();
- _env->getCursorIds(opCtx.get(), entry->options.range.ns, &cursorsNow);
- }
-
- set<CursorId> cursorsLeft;
- std::set_intersection(entry->cursorsToWait.begin(),
- entry->cursorsToWait.end(),
- cursorsNow.begin(),
- cursorsNow.end(),
- std::inserter(cursorsLeft, cursorsLeft.end()));
-
- entry->cursorsToWait.swap(cursorsLeft);
-
- if (entry->cursorsToWait.empty()) {
- (*iter)->stats.queueEndTS = jsTime();
- _taskQueue.push_back(*iter);
- _taskQueueNotEmptyCV.notify_one();
- iter = _notReadyQueue.erase(iter);
- } else {
- logCursorsWaiting(entry);
- ++iter;
- }
- }
- }
- }
-
- if (stopRequested()) {
- log() << "stopping range deleter worker" << endl;
- return;
- }
-
- nextTask = _taskQueue.front();
- _taskQueue.pop_front();
-
- _deletesInProgress++;
- }
-
- {
- auto opCtx = client->makeOperationContext();
- nextTask->stats.deleteStartTS = jsTime();
- bool delResult = _env->deleteRange(
- opCtx.get(), *nextTask, &nextTask->stats.deletedDocCount, &errMsg);
- nextTask->stats.deleteEndTS = jsTime();
-
- if (delResult) {
- nextTask->stats.waitForReplStartTS = jsTime();
-
- if (!_waitForMajority(opCtx.get(), &errMsg)) {
- warning() << "Error encountered while waiting for replication: " << errMsg;
- }
-
- nextTask->stats.waitForReplEndTS = jsTime();
- } else {
- warning() << "Error encountered while trying to delete range: " << redact(errMsg);
- }
- }
-
- {
- stdx::lock_guard<stdx::mutex> sl(_queueMutex);
-
- NSMinMax setEntry(nextTask->options.range.ns,
- nextTask->options.range.minKey,
- nextTask->options.range.maxKey);
- deletePtrElement(&_deleteSet, &setEntry);
- _deletesInProgress--;
-
- if (nextTask->doneSignal) {
- nextTask->doneSignal->set();
- }
- }
-
- recordDelStats(new DeleteJobStats(nextTask->stats));
- delete nextTask;
- nextTask = NULL;
- }
-}
-
-bool RangeDeleter::canEnqueue_inlock(StringData ns,
- const BSONObj& min,
- const BSONObj& max,
- string* errMsg) const {
- NSMinMax toDelete(ns.toString(), min, max);
- if (_deleteSet.count(&toDelete) > 0) {
- *errMsg = str::stream() << "ns: " << ns << ", min: " << min << ", max: " << max
- << " is already being processed for deletion.";
- return false;
- }
-
- return true;
-}
-
-bool RangeDeleter::stopRequested() const {
- stdx::lock_guard<stdx::mutex> sl(_stopMutex);
- return _stopRequested;
-}
-
-size_t RangeDeleter::getTotalDeletes() const {
- stdx::lock_guard<stdx::mutex> sl(_queueMutex);
- return _deleteSet.size();
-}
-
-size_t RangeDeleter::getPendingDeletes() const {
- stdx::lock_guard<stdx::mutex> sl(_queueMutex);
- return _notReadyQueue.size() + _taskQueue.size();
-}
-
-size_t RangeDeleter::getDeletesInProgress() const {
- stdx::lock_guard<stdx::mutex> sl(_queueMutex);
- return _deletesInProgress;
-}
-
-void RangeDeleter::recordDelStats(DeleteJobStats* newStat) {
- stdx::lock_guard<stdx::mutex> sl(_statsHistoryMutex);
- if (_statsHistory.size() == kDeleteJobsHistory) {
- delete _statsHistory.front();
- _statsHistory.pop_front();
- }
-
- _statsHistory.push_back(newStat);
-}
-
-RangeDeleteEntry::RangeDeleteEntry(const RangeDeleterOptions& options)
- : options(options), doneSignal(nullptr) {}
-
-BSONObj RangeDeleteEntry::toBSON() const {
- BSONObjBuilder builder;
- builder.append("ns", options.range.ns);
- builder.append("min", options.range.minKey);
- builder.append("max", options.range.maxKey);
- BSONArrayBuilder cursorBuilder(builder.subarrayStart("cursors"));
-
- for (std::set<CursorId>::const_iterator it = cursorsToWait.begin(); it != cursorsToWait.end();
- ++it) {
- cursorBuilder.append((long long)*it);
- }
- cursorBuilder.doneFast();
-
- return builder.done().copy();
-}
-
-RangeDeleterOptions::RangeDeleterOptions(const KeyRange& range)
- : range(range), fromMigrate(false), onlyRemoveOrphanedDocs(false), waitForOpenCursors(false) {}
-}
diff --git a/src/mongo/db/range_deleter.h b/src/mongo/db/range_deleter.h
deleted file mode 100644
index c15b07df1af..00000000000
--- a/src/mongo/db/range_deleter.h
+++ /dev/null
@@ -1,334 +0,0 @@
-/**
- * Copyright (C) 2013 10gen Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the GNU Affero General Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include <deque>
-#include <set>
-#include <string>
-#include <vector>
-
-#include "mongo/base/disallow_copying.h"
-#include "mongo/base/string_data.h"
-#include "mongo/db/clientcursor.h"
-#include "mongo/db/jsobj.h"
-#include "mongo/db/range_arithmetic.h"
-#include "mongo/db/write_concern_options.h"
-#include "mongo/stdx/mutex.h"
-#include "mongo/stdx/thread.h"
-#include "mongo/util/concurrency/mutex.h"
-#include "mongo/util/concurrency/notification.h"
-#include "mongo/util/time_support.h"
-
-namespace mongo {
-
-class OperationContext;
-struct DeleteJobStats;
-struct RangeDeleteEntry;
-struct RangeDeleterEnv;
-struct RangeDeleterOptions;
-
-/**
- * Class for deleting documents for a given namespace and range. It contains a queue of
- * jobs to be deleted. Deletions can be "immediate", in which case they are going to be put
- * in front of the queue and acted on promptly, or "lazy", in which they would be acted
- * upon when they get to the head of the queue.
- *
- * Threading assumptions:
- *
- * This class has (currently) one worker thread attacking the queue, one
- * job at a time. If we want an immediate deletion, that job is going to
- * be performed on the thread that is requesting it.
- *
- * All calls regarding deletion are synchronized.
- *
- * Life cycle:
- * RangeDeleter* deleter = new RangeDeleter(new ...);
- * deleter->startWorkers();
- * ...
- * getGlobalServiceContext()->killAllOperations(); // stop all deletes
- * deleter->stopWorkers();
- * delete deleter;
- */
-class RangeDeleter {
- MONGO_DISALLOW_COPYING(RangeDeleter);
-
-public:
- /**
- * Creates a new deleter and uses an environment object to delegate external logic like
- * data deletion. Takes ownership of the environment.
- */
- explicit RangeDeleter(RangeDeleterEnv* env);
-
- /**
- * Destroys this deleter. Must make sure that no threads are working on this queue. Use
- * stopWorkers to stop the internal workers, it is an error not to do so.
- */
- ~RangeDeleter();
-
- //
- // Thread management methods
- //
-
- /**
- * Starts the background thread to work on this queue. Does nothing if the worker
- * thread is already active.
- *
- * This call is _not_ thread safe and must be issued before any other call.
- */
- void startWorkers();
-
- /**
- * Stops the background thread working on this queue. This will block if there are
- * tasks that are being deleted, but will leave the pending tasks in the queue.
- *
- * Steps:
- * 1. Stop accepting new queued deletes.
- * 2. Stop all idle workers.
- * 3. Waits for all threads to finish any task that is in progress (but see note
- * below).
- *
- * Note:
- *
- * + restarting this deleter with startWorkers after stopping it is not supported.
- *
- * + the worker thread could be running a call in the environment. The thread is
- * only going to be returned when the environment decides so. In production,
- * KillCurrentOp::killAll can be used to get the thread back from the environment.
- */
- void stopWorkers();
-
- //
- // Queue manipulation methods - can be called by anyone.
- //
-
- /**
- * Adds a new delete to the queue.
- *
- * If notifyDone is not NULL, it will be signaled after the delete is completed.
- * Note that this will happen only if the delete was actually queued.
- *
- * Returns true if the task is queued and false If the given range is blacklisted,
- * is already queued, or stopWorkers() was called.
- */
- bool queueDelete(OperationContext* opCtx,
- const RangeDeleterOptions& options,
- Notification<void>* doneSignal,
- std::string* errMsg);
-
- /**
- * Removes the documents specified by the range. Unlike queueTask, this call
- * blocks and the deletion is performed by the current thread.
- *
- * Returns true if the deletion was performed. False if the range is blacklisted,
- * was already queued, or stopWorkers() was called.
- */
- bool deleteNow(OperationContext* opCtx,
- const RangeDeleterOptions& options,
- std::string* errMsg);
-
- //
- // Introspection methods
- //
-
- // Note: original contents of stats will be cleared. Caller owns the returned stats.
- std::vector<std::unique_ptr<DeleteJobStats>> getStatsHistory() const;
-
- size_t getTotalDeletes() const;
- size_t getPendingDeletes() const;
- size_t getDeletesInProgress() const;
-
- //
- // Methods meant to be only used for testing. Should be treated like private
- // methods.
- //
-
- /** Returns a BSON representation of the queue contents. For debugging only. */
- BSONObj toBSON() const;
-
-private:
- // Ownership is transferred to here.
- void recordDelStats(DeleteJobStats* newStat);
-
-
- struct NSMinMax;
-
- struct NSMinMaxCmp {
- bool operator()(const NSMinMax* lhs, const NSMinMax* rhs) const;
- };
-
- typedef std::deque<RangeDeleteEntry*> TaskList; // owned here
-
- typedef std::set<NSMinMax*, NSMinMaxCmp> NSMinMaxSet; // owned here
-
- /** Body of the worker thread */
- void doWork();
-
- /** Returns true if the range doesn't intersect with one other range */
- bool canEnqueue_inlock(StringData ns,
- const BSONObj& min,
- const BSONObj& max,
- std::string* errMsg) const;
-
- /** Returns true if stopWorkers() was called. This call is synchronized. */
- bool stopRequested() const;
-
- std::unique_ptr<RangeDeleterEnv> _env;
-
- // Initially not active. Must be started explicitly.
- std::unique_ptr<stdx::thread> _worker;
-
- // Protects _stopRequested.
- mutable stdx::mutex _stopMutex;
-
- // If set, no other delete taks should be accepted.
- bool _stopRequested;
-
- // No delete is in progress. Used to make sure that there is no activity
- // in this deleter, and therefore is safe to destroy it. Must be used in
- // conjunction with _stopRequested.
- stdx::condition_variable _nothingInProgressCV;
-
- // Protects all the data structure below this.
- mutable stdx::mutex _queueMutex;
-
- // _taskQueue has a task ready to work on.
- stdx::condition_variable _taskQueueNotEmptyCV;
-
- // Queue for storing the list of ranges that have cursors pending on it.
- //
- // Note: pointer life cycle is not handled here.
- TaskList _notReadyQueue;
-
- // Queue for storing the list of ranges that are ready to be removed.
- //
- // Note: pointer life cycle is not handled here.
- TaskList _taskQueue;
-
- // Set of all deletes - deletes waiting for cursors, waiting to be acted upon
- // and in progress. Includes both queued and immediate deletes.
- //
- // queued delete life cycle: new @ queuedDelete, delete @ doWork
- // deleteNow life cycle: deleteNow stack variable
- NSMinMaxSet _deleteSet;
-
- // Keeps track of number of tasks that are in progress, including the inline deletes.
- size_t _deletesInProgress;
-
- // Protects _statsHistory
- mutable stdx::mutex _statsHistoryMutex;
- std::deque<DeleteJobStats*> _statsHistory;
-};
-
-
-/**
- * Simple class for storing statistics for the RangeDeleter.
- */
-struct DeleteJobStats {
- Date_t queueStartTS;
- Date_t queueEndTS;
- Date_t deleteStartTS;
- Date_t deleteEndTS;
- Date_t waitForReplStartTS;
- Date_t waitForReplEndTS;
-
- long long int deletedDocCount;
-
- DeleteJobStats() : deletedDocCount(0) {}
-};
-
-struct RangeDeleterOptions {
- RangeDeleterOptions(const KeyRange& range);
-
- const KeyRange range;
-
- WriteConcernOptions writeConcern;
- std::string removeSaverReason;
- bool fromMigrate;
- bool onlyRemoveOrphanedDocs;
- bool waitForOpenCursors;
-};
-
-/**
- * For internal use only.
- */
-struct RangeDeleteEntry {
- RangeDeleteEntry(const RangeDeleterOptions& options);
-
- const RangeDeleterOptions options;
-
- // Sets of cursors to wait to close until this can be ready
- // for deletion.
- std::set<CursorId> cursorsToWait;
-
- // Not owned here.
- // Important invariant: Can only be set and used by one thread.
- Notification<void>* doneSignal;
-
- // Time since the last time we reported this object.
- Date_t lastLoggedTS;
-
- DeleteJobStats stats;
-
- // For debugging only
- BSONObj toBSON() const;
-};
-
-/**
- * Class for encapsulating logic used by the RangeDeleter class to perform its tasks.
- */
-struct RangeDeleterEnv {
- virtual ~RangeDeleterEnv() {}
-
- /**
- * Deletes the documents from the given range. This method should be
- * responsible for making sure that the proper contexts are setup
- * to be able to perform deletions.
- *
- * Must be a synchronous call. Docs should be deleted after call ends.
- * Must not throw Exceptions.
- */
- virtual bool deleteRange(OperationContext* opCtx,
- const RangeDeleteEntry& taskDetails,
- long long int* deletedDocs,
- std::string* errMsg) = 0;
-
- /**
- * Gets the list of open cursors on a given namespace. The openCursors is an
- * output parameter that will contain all the cursors open after this is called.
- * Assume that openCursors is empty when passed in.
- *
- * Must be a synchronous call. CursorIds should be populated after call.
- * Must not throw exception.
- */
- virtual void getCursorIds(OperationContext* opCtx,
- StringData ns,
- std::set<CursorId>* openCursors) = 0;
-};
-
-} // namespace mongo
diff --git a/src/mongo/db/range_deleter_db_env.h b/src/mongo/db/range_deleter_db_env.h
deleted file mode 100644
index a246573eda6..00000000000
--- a/src/mongo/db/range_deleter_db_env.h
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Copyright (C) 2013 10gen Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the GNU Affero General Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include "mongo/db/range_deleter.h"
-
-namespace mongo {
-
-/**
- * This class implements the deleter methods to be used for a shard.
- */
-struct RangeDeleterDBEnv : public RangeDeleterEnv {
- /**
- * Deletes the documents from the given range synchronously.
- *
- * The keyPattern will be used to determine the right index to use to perform
- * the deletion and it can be a prefix of an existing index. Caller is responsible
- * of making sure that both min and max is a prefix of keyPattern.
- *
- * Note that secondaryThrottle will be ignored if current process is not part
- * of a replica set.
- *
- * docsDeleted would contain the number of docs deleted if the deletion was successful.
- *
- * Does not throw Exceptions.
- */
- virtual bool deleteRange(OperationContext* opCtx,
- const RangeDeleteEntry& taskDetails,
- long long int* deletedDocs,
- std::string* errMsg);
-
- /**
- * Gets the list of open cursors on a given namespace.
- */
- virtual void getCursorIds(OperationContext* opCtx,
- StringData ns,
- std::set<CursorId>* openCursors);
-};
-}
diff --git a/src/mongo/db/range_deleter_mock_env.cpp b/src/mongo/db/range_deleter_mock_env.cpp
deleted file mode 100644
index 9ac49945346..00000000000
--- a/src/mongo/db/range_deleter_mock_env.cpp
+++ /dev/null
@@ -1,146 +0,0 @@
-/**
- * Copyright (C) 2013 10gen Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the GNU Affero General Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/range_deleter_mock_env.h"
-
-namespace mongo {
-
-using std::set;
-using std::string;
-
-bool DeletedRangeCmp::operator()(const DeletedRange& lhs, const DeletedRange& rhs) const {
- const int nsComp = lhs.ns.compare(rhs.ns);
-
- if (nsComp < 0) {
- return true;
- }
-
- if (nsComp > 0) {
- return false;
- }
-
- return compareRanges(lhs.min, lhs.max, rhs.min, rhs.max) < 0;
-}
-
-RangeDeleterMockEnv::RangeDeleterMockEnv()
- : _pauseDelete(false), _pausedCount(0), _getCursorsCallCount(0) {}
-
-void RangeDeleterMockEnv::addCursorId(StringData ns, CursorId id) {
- stdx::lock_guard<stdx::mutex> sl(_cursorMapMutex);
- _cursorMap[ns.toString()].insert(id);
-}
-
-void RangeDeleterMockEnv::removeCursorId(StringData ns, CursorId id) {
- stdx::lock_guard<stdx::mutex> sl(_cursorMapMutex);
- _cursorMap[ns.toString()].erase(id);
-}
-
-void RangeDeleterMockEnv::pauseDeletes() {
- stdx::lock_guard<stdx::mutex> sl(_pauseDeleteMutex);
- _pauseDelete = true;
-}
-
-void RangeDeleterMockEnv::resumeOneDelete() {
- stdx::lock_guard<stdx::mutex> sl(_pauseDeleteMutex);
- _pauseDelete = false;
- _pausedCV.notify_one();
-}
-
-void RangeDeleterMockEnv::waitForNthGetCursor(uint64_t nthCall) {
- stdx::unique_lock<stdx::mutex> sl(_envStatMutex);
- while (_getCursorsCallCount < nthCall) {
- _cursorsCallCountUpdatedCV.wait(sl);
- }
-}
-
-void RangeDeleterMockEnv::waitForNthPausedDelete(uint64_t nthPause) {
- stdx::unique_lock<stdx::mutex> sl(_pauseDeleteMutex);
- while (_pausedCount < nthPause) {
- _pausedDeleteChangeCV.wait(sl);
- }
-}
-
-bool RangeDeleterMockEnv::deleteOccured() const {
- stdx::lock_guard<stdx::mutex> sl(_deleteListMutex);
- return !_deleteList.empty();
-}
-
-DeletedRange RangeDeleterMockEnv::getLastDelete() const {
- stdx::lock_guard<stdx::mutex> sl(_deleteListMutex);
- return _deleteList.back();
-}
-
-bool RangeDeleterMockEnv::deleteRange(OperationContext* opCtx,
- const RangeDeleteEntry& taskDetails,
- long long int* deletedDocs,
- string* errMsg) {
- {
- stdx::unique_lock<stdx::mutex> sl(_pauseDeleteMutex);
- bool wasInitiallyPaused = _pauseDelete;
-
- if (_pauseDelete) {
- _pausedCount++;
- _pausedDeleteChangeCV.notify_one();
- }
-
- while (_pauseDelete) {
- _pausedCV.wait(sl);
- }
-
- _pauseDelete = wasInitiallyPaused;
- }
-
- {
- stdx::lock_guard<stdx::mutex> sl(_deleteListMutex);
-
- DeletedRange entry;
- entry.ns = taskDetails.options.range.ns;
- entry.min = taskDetails.options.range.minKey.getOwned();
- entry.max = taskDetails.options.range.maxKey.getOwned();
- entry.shardKeyPattern = taskDetails.options.range.keyPattern.getOwned();
-
- _deleteList.push_back(entry);
- }
-
- return true;
-}
-
-void RangeDeleterMockEnv::getCursorIds(OperationContext* opCtx, StringData ns, set<CursorId>* in) {
- {
- stdx::lock_guard<stdx::mutex> sl(_cursorMapMutex);
- const set<CursorId>& _cursors = _cursorMap[ns.toString()];
- std::copy(_cursors.begin(), _cursors.end(), inserter(*in, in->begin()));
- }
-
- {
- stdx::lock_guard<stdx::mutex> sl(_envStatMutex);
- _getCursorsCallCount++;
- _cursorsCallCountUpdatedCV.notify_one();
- }
-}
-}
diff --git a/src/mongo/db/range_deleter_mock_env.h b/src/mongo/db/range_deleter_mock_env.h
deleted file mode 100644
index f57ef8b1d5e..00000000000
--- a/src/mongo/db/range_deleter_mock_env.h
+++ /dev/null
@@ -1,167 +0,0 @@
-/**
- * Copyright (C) 2013 10gen Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the GNU Affero General Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include <map>
-#include <set>
-#include <string>
-
-#include "mongo/db/range_deleter.h"
-#include "mongo/stdx/mutex.h"
-
-namespace mongo {
-
-struct DeletedRange {
- std::string ns;
- BSONObj min;
- BSONObj max;
- BSONObj shardKeyPattern;
-};
-
-/**
- * Comparator function object compatible with set.
- */
-struct DeletedRangeCmp {
- bool operator()(const DeletedRange& lhs, const DeletedRange& rhs) const;
-};
-
-/**
- * Mock environment for RangeDeleter with knobs for pausing/resuming
- * deletes, setting open cursors IDs per namespace and the ability to
- * record the history of deletes performed through this environment.
- */
-class RangeDeleterMockEnv : public mongo::RangeDeleterEnv {
-public:
- RangeDeleterMockEnv();
-
- //
- // Environment modification methods.
- //
-
- /**
- * Adds an id to the current set of cursors in the given namespace.
- */
- void addCursorId(StringData ns, CursorId id);
-
- /**
- * Removes the id from the set of open cursors in the given namespace.
- */
- void removeCursorId(StringData ns, CursorId id);
-
- //
- // Environment synchronization methods.
- //
-
- /**
- * Blocks all new deletes from proceeding.
- */
- void pauseDeletes();
-
- /**
- * Unblocks one paused delete.
- */
- void resumeOneDelete();
-
- /**
- * Blocks until the getCursor method was called and terminated at least the
- * specified number of times for the entire lifetime of this deleter.
- */
- void waitForNthGetCursor(uint64_t nthCall);
-
- /**
- * Blocks until the deleteRange method was called and at the same time paused
- * at least the specified number of times for the entire lifetime of this deleter.
- */
- void waitForNthPausedDelete(uint64_t nthPause);
-
- //
- // Environment introspection methods.
- //
-
- /**
- * Returns true if deleteRange was called at least once.
- */
- bool deleteOccured() const;
-
- /**
- * Returns the last delete. Undefined if deleteOccured is false.
- */
- DeletedRange getLastDelete() const;
-
- //
- // Environment methods.
- //
-
- /**
- * Basic implementation of delete that matches the signature for
- * RangeDeleterEnv::deleteRange. This does not actually perform the delete
- * but simply keeps a record of it. Can also be paused by pauseDeletes and
- * resumed with resumeDeletes.
- */
- bool deleteRange(OperationContext* opCtx,
- const RangeDeleteEntry& taskDetails,
- long long int* deletedDocs,
- std::string* errMsg);
-
- /**
- * Basic implementation of gathering open cursors that matches the signature for
- * RangeDeleterEnv::getCursorIds. The cursors returned can be modified with
- * the setCursorId and clearCursorMap methods.
- */
- void getCursorIds(OperationContext* opCtx, StringData ns, std::set<CursorId>* in);
-
-private:
- // mutex acquisition ordering:
- // _envStatMutex -> _pauseDeleteMutex -> _deleteListMutex -> _cursorMapMutex
-
- mutable stdx::mutex _deleteListMutex;
- std::vector<DeletedRange> _deleteList;
-
- stdx::mutex _cursorMapMutex;
- std::map<std::string, std::set<CursorId>> _cursorMap;
-
- // Protects _pauseDelete & _pausedCount
- stdx::mutex _pauseDeleteMutex;
- stdx::condition_variable _pausedCV;
- bool _pauseDelete;
-
- // Number of times a delete gets paused.
- uint64_t _pausedCount;
- // _pausedCount < nthPause (used by waitForNthPausedDelete)
- stdx::condition_variable _pausedDeleteChangeCV;
-
- // Protects all variables below this line.
- stdx::mutex _envStatMutex;
-
- // Keeps track of the number of times getCursorIds was called.
- uint64_t _getCursorsCallCount;
- // _getCursorsCallCount < nthCall (used by waitForNthGetCursor)
- stdx::condition_variable _cursorsCallCountUpdatedCV;
-};
-}
diff --git a/src/mongo/db/range_deleter_service.cpp b/src/mongo/db/range_deleter_service.cpp
deleted file mode 100644
index bbe7dc72680..00000000000
--- a/src/mongo/db/range_deleter_service.cpp
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Copyright (C) 2013 10gen Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the GNU Affero General Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/db/range_deleter_service.h"
-
-#include "mongo/base/init.h"
-#include "mongo/db/range_deleter_db_env.h"
-
-namespace {
-
-mongo::RangeDeleter* _deleter = NULL;
-}
-
-namespace mongo {
-
-MONGO_INITIALIZER(RangeDeleterInit)(InitializerContext* context) {
- _deleter = new RangeDeleter(new RangeDeleterDBEnv);
- return Status::OK();
-}
-
-RangeDeleter* getDeleter() {
- return _deleter;
-}
-}
diff --git a/src/mongo/db/range_deleter_service.h b/src/mongo/db/range_deleter_service.h
deleted file mode 100644
index d654e870350..00000000000
--- a/src/mongo/db/range_deleter_service.h
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Copyright (C) 2013 10gen Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the GNU Affero General Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#pragma once
-
-#include "mongo/db/range_deleter.h"
-
-namespace mongo {
-
-/**
- * Gets the global instance of the deleter and starts it.
- */
-RangeDeleter* getDeleter();
-}
diff --git a/src/mongo/db/range_deleter_test.cpp b/src/mongo/db/range_deleter_test.cpp
deleted file mode 100644
index 633743c330a..00000000000
--- a/src/mongo/db/range_deleter_test.cpp
+++ /dev/null
@@ -1,339 +0,0 @@
-/**
- * Copyright (C) 2013 10gen Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the GNU Affero General Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/platform/basic.h"
-
-#include <string>
-
-#include "mongo/db/client.h"
-#include "mongo/db/field_parser.h"
-#include "mongo/db/operation_context.h"
-#include "mongo/db/range_deleter.h"
-#include "mongo/db/range_deleter_mock_env.h"
-#include "mongo/db/repl/repl_settings.h"
-#include "mongo/db/repl/replication_coordinator_mock.h"
-#include "mongo/db/service_context.h"
-#include "mongo/db/service_context_d_test_fixture.h"
-#include "mongo/db/service_context_noop.h"
-#include "mongo/db/write_concern_options.h"
-#include "mongo/stdx/functional.h"
-#include "mongo/stdx/future.h"
-#include "mongo/stdx/thread.h"
-#include "mongo/unittest/unittest.h"
-#include "mongo/util/scopeguard.h"
-
-namespace mongo {
-namespace {
-
-using std::string;
-
-// The range deleter cursor close wait interval increases exponentially from 5 milliseconds to an
-// upper bound of 500 msec. Three seconds should be enough time for changes in the cursors set to be
-// noticed.
-const Seconds MAX_IMMEDIATE_DELETE_WAIT(3);
-
-const mongo::repl::ReplSettings replSettings = {};
-
-class RangeDeleterTest : public unittest::Test {
-public:
- ServiceContext* getServiceContext() {
- return getGlobalServiceContext();
- }
- OperationContext* getOpCtx() {
- return _opCtx.get();
- }
-
-protected:
- RangeDeleterTest() : env(new RangeDeleterMockEnv()), deleter(env) {}
-
- RangeDeleterMockEnv* env;
- RangeDeleter deleter;
-
-private:
- void setUp() override {
- setGlobalServiceContext(stdx::make_unique<ServiceContextNoop>());
- mongo::repl::ReplicationCoordinator::set(
- getServiceContext(),
- stdx::make_unique<mongo::repl::ReplicationCoordinatorMock>(getServiceContext(),
- replSettings));
- _client = getServiceContext()->makeClient("RangeDeleterTest");
- _opCtx = _client->makeOperationContext();
- deleter.startWorkers();
- }
-
- ServiceContext::UniqueClient _client;
- ServiceContext::UniqueOperationContext _opCtx;
-};
-
-using ImmediateDelete = RangeDeleterTest;
-using MixedDeletes = RangeDeleterTest;
-using QueuedDelete = RangeDeleterTest;
-
-// Should not be able to queue deletes if deleter workers were not started.
-TEST_F(QueuedDelete, CantAfterStop) {
- deleter.stopWorkers();
-
- string errMsg;
- ASSERT_FALSE(
- deleter.queueDelete(getOpCtx(),
- RangeDeleterOptions(KeyRange(
- "test.user", BSON("x" << 120), BSON("x" << 200), BSON("x" << 1))),
- NULL /* notifier not needed */,
- &errMsg));
- ASSERT_FALSE(errMsg.empty());
- ASSERT_FALSE(env->deleteOccured());
-}
-
-// Should not start delete if the set of cursors that were open when the
-// delete was queued is still open.
-TEST_F(QueuedDelete, ShouldWaitCursor) {
- const string ns("test.user");
- env->addCursorId(ns, 345);
-
- Notification<void> doneSignal;
- RangeDeleterOptions deleterOptions(
- KeyRange(ns, BSON("x" << 0), BSON("x" << 10), BSON("x" << 1)));
- deleterOptions.waitForOpenCursors = true;
-
- ASSERT_TRUE(
- deleter.queueDelete(getOpCtx(), deleterOptions, &doneSignal, NULL /* errMsg not needed */));
-
- env->waitForNthGetCursor(1u);
-
- ASSERT_EQUALS(1U, deleter.getPendingDeletes());
- ASSERT_FALSE(env->deleteOccured());
-
- // Set the open cursors to a totally different sets of cursorIDs.
- env->addCursorId(ns, 200);
- env->removeCursorId(ns, 345);
-
- doneSignal.get(getOpCtx());
-
- ASSERT_TRUE(env->deleteOccured());
- const DeletedRange deletedChunk(env->getLastDelete());
-
- ASSERT_EQUALS(ns, deletedChunk.ns);
- ASSERT_BSONOBJ_EQ(deletedChunk.min, BSON("x" << 0));
- ASSERT_BSONOBJ_EQ(deletedChunk.max, BSON("x" << 10));
-
- deleter.stopWorkers();
-}
-
-// Should terminate when stop is requested.
-TEST_F(QueuedDelete, StopWhileWaitingCursor) {
- const string ns("test.user");
- env->addCursorId(ns, 345);
-
- Notification<void> doneSignal;
- RangeDeleterOptions deleterOptions(
- KeyRange(ns, BSON("x" << 0), BSON("x" << 10), BSON("x" << 1)));
- deleterOptions.waitForOpenCursors = true;
- ASSERT_TRUE(
- deleter.queueDelete(getOpCtx(), deleterOptions, &doneSignal, NULL /* errMsg not needed */));
-
-
- env->waitForNthGetCursor(1u);
-
- deleter.stopWorkers();
- ASSERT_FALSE(env->deleteOccured());
-}
-
-// Should not start delete if the set of cursors that were open when the
-// deleteNow method is called is still open.
-TEST_F(ImmediateDelete, ShouldWaitCursor) {
- const string ns("test.user");
- env->addCursorId(ns, 345);
-
- string errMsg;
- RangeDeleterOptions deleterOption(
- KeyRange(ns, BSON("x" << 0), BSON("x" << 10), BSON("x" << 1)));
- deleterOption.waitForOpenCursors = true;
-
- stdx::packaged_task<bool()> deleterTask(
- [&] { return deleter.deleteNow(getOpCtx(), deleterOption, &errMsg); });
- stdx::future<bool> deleterFuture = deleterTask.get_future();
- stdx::thread deleterThread(std::move(deleterTask));
-
- auto guard = MakeGuard([&] {
- deleter.stopWorkers();
- deleterThread.join();
- });
-
- env->waitForNthGetCursor(1u);
-
- // Note: immediate deletes has no pending state, it goes directly to inProgress
- // even while waiting for cursors.
- ASSERT_EQUALS(1U, deleter.getDeletesInProgress());
-
- ASSERT_FALSE(env->deleteOccured());
-
- // Set the open cursors to a totally different sets of cursorIDs.
- env->addCursorId(ns, 200);
- env->removeCursorId(ns, 345);
-
- ASSERT_TRUE(stdx::future_status::ready ==
- deleterFuture.wait_for(MAX_IMMEDIATE_DELETE_WAIT.toSystemDuration()));
- ASSERT_TRUE(deleterFuture.get());
- ASSERT_TRUE(env->deleteOccured());
-
- const DeletedRange deletedChunk(env->getLastDelete());
-
- ASSERT_EQUALS(ns, deletedChunk.ns);
- ASSERT_BSONOBJ_EQ(deletedChunk.min, BSON("x" << 0));
- ASSERT_BSONOBJ_EQ(deletedChunk.max, BSON("x" << 10));
- ASSERT_BSONOBJ_EQ(deletedChunk.shardKeyPattern, BSON("x" << 1));
-}
-
-// Should terminate when stop is requested.
-TEST_F(ImmediateDelete, StopWhileWaitingCursor) {
- const string ns("test.user");
- env->addCursorId(ns, 345);
-
- string errMsg;
- RangeDeleterOptions deleterOption(
- KeyRange(ns, BSON("x" << 0), BSON("x" << 10), BSON("x" << 1)));
- deleterOption.waitForOpenCursors = true;
-
- stdx::packaged_task<bool()> deleterTask(
- [&] { return deleter.deleteNow(getOpCtx(), deleterOption, &errMsg); });
- stdx::future<bool> deleterFuture = deleterTask.get_future();
- stdx::thread deleterThread(std::move(deleterTask));
-
- auto join_thread_guard = MakeGuard([&] { deleterThread.join(); });
- auto stop_deleter_guard = MakeGuard([&] { deleter.stopWorkers(); });
-
- env->waitForNthGetCursor(1u);
-
- // Note: immediate deletes has no pending state, it goes directly to inProgress
- // even while waiting for cursors.
- ASSERT_EQUALS(1U, deleter.getDeletesInProgress());
-
- ASSERT_FALSE(env->deleteOccured());
-
- stop_deleter_guard.Execute();
-
- ASSERT_TRUE(stdx::future_status::ready ==
- deleterFuture.wait_for(MAX_IMMEDIATE_DELETE_WAIT.toSystemDuration()));
- ASSERT_FALSE(deleterFuture.get());
- ASSERT_FALSE(env->deleteOccured());
-}
-
-// Tests the interaction of multiple deletes queued with different states.
-// Starts by adding a new delete task, waits for the worker to work on it,
-// and then adds 2 more task, one of which is ready to be deleted, while the
-// other one is waiting for an open cursor. The test then makes sure that the
-// deletes are performed in the right order.
-TEST_F(MixedDeletes, MultipleDeletes) {
- const string blockedNS("foo.bar");
- const string ns("test.user");
-
- env->addCursorId(blockedNS, 345);
- env->pauseDeletes();
-
- Notification<void> doneSignal1;
- RangeDeleterOptions deleterOption1(
- KeyRange(ns, BSON("x" << 10), BSON("x" << 20), BSON("x" << 1)));
- deleterOption1.waitForOpenCursors = true;
- ASSERT_TRUE(deleter.queueDelete(
- getOpCtx(), deleterOption1, &doneSignal1, NULL /* don't care errMsg */));
-
- env->waitForNthPausedDelete(1u);
-
- // Make sure that the delete is already in progress before proceeding.
- ASSERT_EQUALS(1U, deleter.getDeletesInProgress());
-
- Notification<void> doneSignal2;
- RangeDeleterOptions deleterOption2(
- KeyRange(blockedNS, BSON("x" << 20), BSON("x" << 30), BSON("x" << 1)));
- deleterOption2.waitForOpenCursors = true;
- ASSERT_TRUE(deleter.queueDelete(
- getOpCtx(), deleterOption2, &doneSignal2, NULL /* don't care errMsg */));
-
- Notification<void> doneSignal3;
- RangeDeleterOptions deleterOption3(
- KeyRange(ns, BSON("x" << 30), BSON("x" << 40), BSON("x" << 1)));
- deleterOption3.waitForOpenCursors = true;
- ASSERT_TRUE(deleter.queueDelete(
- getOpCtx(), deleterOption3, &doneSignal3, NULL /* don't care errMsg */));
-
- // Now, the setup is:
- // { x: 10 } => { x: 20 } in progress.
- // { x: 20 } => { x: 30 } waiting for cursor id 345.
- // { x: 30 } => { x: 40 } waiting to be picked up by worker.
-
- // Make sure that the current state matches the setup.
- ASSERT_EQUALS(3U, deleter.getTotalDeletes());
- ASSERT_EQUALS(2U, deleter.getPendingDeletes());
- ASSERT_EQUALS(1U, deleter.getDeletesInProgress());
-
- // Let the first delete proceed.
- env->resumeOneDelete();
- doneSignal1.get(getOpCtx());
-
- ASSERT_TRUE(env->deleteOccured());
-
- // { x: 10 } => { x: 20 } should be the first one since it is already in
- // progress before the others are queued.
- DeletedRange deleted1(env->getLastDelete());
-
- ASSERT_EQUALS(ns, deleted1.ns);
- ASSERT_BSONOBJ_EQ(deleted1.min, BSON("x" << 10));
- ASSERT_BSONOBJ_EQ(deleted1.max, BSON("x" << 20));
- ASSERT_BSONOBJ_EQ(deleted1.shardKeyPattern, BSON("x" << 1));
-
- // Let the second delete proceed.
- env->resumeOneDelete();
- doneSignal3.get(getOpCtx());
-
- DeletedRange deleted2(env->getLastDelete());
-
- // { x: 30 } => { x: 40 } should be next since there are still
- // cursors open for blockedNS.
-
- ASSERT_EQUALS(ns, deleted2.ns);
- ASSERT_BSONOBJ_EQ(deleted2.min, BSON("x" << 30));
- ASSERT_BSONOBJ_EQ(deleted2.max, BSON("x" << 40));
- ASSERT_BSONOBJ_EQ(deleted2.shardKeyPattern, BSON("x" << 1));
-
- env->removeCursorId(blockedNS, 345);
- // Let the last delete proceed.
- env->resumeOneDelete();
- doneSignal2.get(getOpCtx());
-
- DeletedRange deleted3(env->getLastDelete());
-
- ASSERT_EQUALS(blockedNS, deleted3.ns);
- ASSERT_BSONOBJ_EQ(deleted3.min, BSON("x" << 20));
- ASSERT_BSONOBJ_EQ(deleted3.max, BSON("x" << 30));
- ASSERT_BSONOBJ_EQ(deleted3.shardKeyPattern, BSON("x" << 1));
-
- deleter.stopWorkers();
-}
-
-} // unnamed namespace
-} // namespace mongo
diff --git a/src/mongo/db/s/cleanup_orphaned_cmd.cpp b/src/mongo/db/s/cleanup_orphaned_cmd.cpp
index a7a9519300b..8ce1e5d08f4 100644
--- a/src/mongo/db/s/cleanup_orphaned_cmd.cpp
+++ b/src/mongo/db/s/cleanup_orphaned_cmd.cpp
@@ -43,7 +43,6 @@
#include "mongo/db/jsobj.h"
#include "mongo/db/namespace_string.h"
#include "mongo/db/range_arithmetic.h"
-#include "mongo/db/range_deleter_service.h"
#include "mongo/db/repl/replication_coordinator_global.h"
#include "mongo/db/s/chunk_move_write_concern_options.h"
#include "mongo/db/s/collection_metadata.h"
diff --git a/src/mongo/db/s/migration_destination_manager.cpp b/src/mongo/db/s/migration_destination_manager.cpp
index beb61bfd3db..b7329e10fc3 100644
--- a/src/mongo/db/s/migration_destination_manager.cpp
+++ b/src/mongo/db/s/migration_destination_manager.cpp
@@ -47,7 +47,6 @@
#include "mongo/db/namespace_string.h"
#include "mongo/db/operation_context.h"
#include "mongo/db/ops/delete.h"
-#include "mongo/db/range_deleter_service.h"
#include "mongo/db/repl/repl_client_info.h"
#include "mongo/db/repl/replication_coordinator_global.h"
#include "mongo/db/s/collection_metadata.h"
diff --git a/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp b/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp
index 05c21710bbf..383af210483 100644
--- a/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp
+++ b/src/mongo/db/s/migration_destination_manager_legacy_commands.cpp
@@ -42,7 +42,6 @@
#include "mongo/db/auth/privilege.h"
#include "mongo/db/commands.h"
#include "mongo/db/jsobj.h"
-#include "mongo/db/range_deleter_service.h"
#include "mongo/db/s/chunk_move_write_concern_options.h"
#include "mongo/db/s/sharding_state.h"
#include "mongo/s/chunk_version.h"
diff --git a/src/mongo/db/s/move_chunk_command.cpp b/src/mongo/db/s/move_chunk_command.cpp
index 315055b0ce5..7f5500497d6 100644
--- a/src/mongo/db/s/move_chunk_command.cpp
+++ b/src/mongo/db/s/move_chunk_command.cpp
@@ -37,7 +37,6 @@
#include "mongo/db/auth/authorization_session.h"
#include "mongo/db/commands.h"
#include "mongo/db/db_raii.h"
-#include "mongo/db/range_deleter_service.h"
#include "mongo/db/repl/repl_client_info.h"
#include "mongo/db/s/chunk_move_write_concern_options.h"
#include "mongo/db/s/collection_metadata.h"
diff --git a/src/mongo/db/stats/range_deleter_server_status.cpp b/src/mongo/db/stats/range_deleter_server_status.cpp
deleted file mode 100644
index 43e919d160a..00000000000
--- a/src/mongo/db/stats/range_deleter_server_status.cpp
+++ /dev/null
@@ -1,98 +0,0 @@
-/**
- * Copyright (C) 2014 MongoDB Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License, version 3,
- * as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- * As a special exception, the copyright holders give permission to link the
- * code of portions of this program with the OpenSSL library under certain
- * conditions as described in each individual source file and distribute
- * linked combinations including the program with the OpenSSL library. You
- * must comply with the GNU Affero General Public License in all respects for
- * all of the code used other than as permitted herein. If you modify file(s)
- * with this exception, you may extend this exception to your version of the
- * file(s), but you are not obligated to do so. If you do not wish to do so,
- * delete this exception statement from your version. If you delete this
- * exception statement from all source files in the program, then also delete
- * it in the license file.
- */
-
-#include "mongo/base/owned_pointer_vector.h"
-#include "mongo/db/commands/server_status.h"
-#include "mongo/db/range_deleter_service.h"
-
-namespace mongo {
-
-/**
- * Server status section for RangeDeleter.
- *
- * Sample format:
- *
- * rangeDeleter: {
- * lastDeleteStats: [
- * {
- * deleteDocs: NumberLong(5);
- * queueStart: ISODate("2014-06-11T22:45:30.221Z"),
- * queueEnd: ISODate("2014-06-11T22:45:30.221Z"),
- * deleteStart: ISODate("2014-06-11T22:45:30.221Z"),
- * deleteEnd: ISODate("2014-06-11T22:45:30.221Z"),
- * waitForReplStart: ISODate("2014-06-11T22:45:30.221Z"),
- * waitForReplEnd: ISODate("2014-06-11T22:45:30.221Z")
- * }
- * ]
- * }
- */
-class RangeDeleterServerStatusSection : public ServerStatusSection {
-public:
- RangeDeleterServerStatusSection() : ServerStatusSection("rangeDeleter") {}
- bool includeByDefault() const {
- return false;
- }
-
- BSONObj generateSection(OperationContext* opCtx, const BSONElement& configElement) const {
- RangeDeleter* deleter = getDeleter();
- if (!deleter) {
- return BSONObj();
- }
-
- BSONObjBuilder result;
-
- auto statsList = deleter->getStatsHistory();
- BSONArrayBuilder oldStatsBuilder;
- for (auto it = statsList.begin(); it != statsList.end(); ++it) {
- BSONObjBuilder entryBuilder;
- entryBuilder.append("deletedDocs", (*it)->deletedDocCount);
-
- if ((*it)->queueEndTS > Date_t()) {
- entryBuilder.append("queueStart", (*it)->queueStartTS);
- entryBuilder.append("queueEnd", (*it)->queueEndTS);
- }
-
- if ((*it)->deleteEndTS > Date_t()) {
- entryBuilder.append("deleteStart", (*it)->deleteStartTS);
- entryBuilder.append("deleteEnd", (*it)->deleteEndTS);
-
- if ((*it)->waitForReplEndTS > Date_t()) {
- entryBuilder.append("waitForReplStart", (*it)->waitForReplStartTS);
- entryBuilder.append("waitForReplEnd", (*it)->waitForReplEndTS);
- }
- }
-
- oldStatsBuilder.append(entryBuilder.obj());
- }
- result.append("lastDeleteStats", oldStatsBuilder.arr());
-
- return result.obj();
- }
-
-} rangeDeleterServerStatusSection;
-}