1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
|
// record_store.h
/**
* Copyright (C) 2013 10gen Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* As a special exception, the copyright holders give permission to link the
* code of portions of this program with the OpenSSL library under certain
* conditions as described in each individual source file and distribute
* linked combinations including the program with the OpenSSL library. You
* must comply with the GNU Affero General Public License in all respects for
* all of the code used other than as permitted herein. If you modify file(s)
* with this exception, you may extend this exception to your version of the
* file(s), but you are not obligated to do so. If you do not wish to do so,
* delete this exception statement from your version. If you delete this
* exception statement from all source files in the program, then also delete
* it in the license file.
*/
#pragma once
#include <boost/optional.hpp>
#include "mongo/base/owned_pointer_vector.h"
#include "mongo/bson/mutable/damage_vector.h"
#include "mongo/db/exec/collection_scan_common.h"
#include "mongo/db/record_id.h"
#include "mongo/db/storage/record_data.h"
#include "mongo/db/storage/record_fetcher.h"
namespace mongo {
class CappedCallback;
class Collection;
struct CompactOptions;
struct CompactStats;
class DocWriter;
class MAdvise;
class NamespaceDetails;
class OperationContext;
class RecordFetcher;
class RecordStoreCompactAdaptor;
class RecordStore;
struct ValidateResults;
class ValidateAdaptor;
/**
* Allows inserting a Record "in-place" without creating a copy ahead of time.
*/
class DocWriter {
public:
virtual ~DocWriter() {}
virtual void writeDocument(char* buf) const = 0;
virtual size_t documentSize() const = 0;
virtual bool addPadding() const {
return true;
}
};
/**
* @see RecordStore::updateRecord
*/
class UpdateNotifier {
public:
virtual ~UpdateNotifier() {}
virtual Status recordStoreGoingToMove(OperationContext* txn,
const RecordId& oldLocation,
const char* oldBuffer,
size_t oldSize) = 0;
virtual Status recordStoreGoingToUpdateInPlace(OperationContext* txn, const RecordId& loc) = 0;
};
/**
* The data items stored in a RecordStore.
*/
struct Record {
RecordId id;
RecordData data;
};
struct BsonRecord {
RecordId id;
const BSONObj* docPtr;
};
/**
* Retrieves Records from a RecordStore.
*
* A cursor is constructed with a direction flag with the following effects:
* - The direction that next() moves.
* - If a restore cannot return to the saved position, cursors will be positioned on the
* closest position *after* the query in the direction of the scan.
*
* A cursor is tied to a transaction, such as the OperationContext or a WriteUnitOfWork
* inside that context. Any cursor acquired inside a transaction is invalid outside
* of that transaction, instead use the save and restore methods to reestablish the cursor.
*
* Any method other than invalidate and the save methods may throw WriteConflictException. If
* that happens, the cursor may not be used again until it has been saved and successfully
* restored. If next() or restore() throw a WCE the cursor's position will be the same as before
* the call (strong exception guarantee). All other methods leave the cursor in a valid state
* but with an unspecified position (basic exception guarantee). If any exception other than
* WCE is thrown, the cursor must be destroyed, which is guaranteed not to leak any resources.
*
* Any returned unowned BSON is only valid until the next call to any method on this
* interface.
*
* Implementations may override any default implementation if they can provide a more
* efficient implementation.
*
* Storage engines only need to implement the derived SeekableRecordCursor, but may choose
* to implement this simpler interface for cursors used for repair or random traversal.
*
* IMPORTANT NOTE FOR DOCUMENT-LOCKING ENGINES: If you implement capped collections with a
* "visibility" system such that documents that exist in your snapshot but were inserted after
* the last uncommitted document are hidden, you must follow the following rules:
* - next() must never return invisible documents.
* - If next() on a forward cursor hits an invisible document, it should behave as if it hit
* the end of the collection.
* - When next() on a reverse cursor seeks to the end of the collection it must return the
* newest visible document. This should only return boost::none if there are no visible
* documents in the collection.
* - SeekableRecordCursor::seekExact() must ignore the visibility filter and return the requested
* document even if it is supposed to be invisible.
* TODO SERVER-18934 Handle this above the storage engine layer so storage engines don't have to
* deal with capped visibility.
*/
class RecordCursor {
public:
virtual ~RecordCursor() = default;
/**
* Moves forward and returns the new data or boost::none if there is no more data.
* Continues returning boost::none once it reaches EOF.
*/
virtual boost::optional<Record> next() = 0;
//
// Saving and restoring state
//
/**
* Prepares for state changes in underlying data in a way that allows the cursor's
* current position to be restored.
*
* It is safe to call save multiple times in a row.
* No other method (excluding destructor) may be called until successfully restored.
*/
virtual void save() = 0;
/**
* Recovers from potential state changes in underlying data.
*
* Returns false if it is invalid to continue using this iterator. This usually means that
* capped deletes have caught up to the position of this iterator and continuing could
* result in missed data.
*
* If the former position no longer exists, but it is safe to continue iterating, the
* following call to next() will return the next closest position in the direction of the
* scan, if any.
*
* This handles restoring after either save() or SeekableRecordCursor::saveUnpositioned().
*/
virtual bool restore() = 0;
/**
* Detaches from the OperationContext and releases any storage-engine state.
*
* It is only legal to call this when in a "saved" state. While in the "detached" state, it is
* only legal to call reattachToOperationContext or the destructor. It is not legal to call
* detachFromOperationContext() while already in the detached state.
*/
virtual void detachFromOperationContext() = 0;
/**
* Reattaches to the OperationContext and reacquires any storage-engine state.
*
* It is only legal to call this in the "detached" state. On return, the cursor is left in a
* "saved" state, so callers must still call restoreState to use this object.
*/
virtual void reattachToOperationContext(OperationContext* opCtx) = 0;
/**
* Inform the cursor that this id is being invalidated. Must be called between save and restore.
* The txn is that of the operation causing the invalidation, not the txn using the cursor.
*
* WARNING: Storage engines other than MMAPv1 should use the default implementation,
* and not depend on this being called.
*/
virtual void invalidate(OperationContext* txn, const RecordId& id) {}
//
// RecordFetchers
//
// Storage engines which do not support document-level locking hold locks at collection or
// database granularity. As an optimization, these locks can be yielded when a record needs
// to be fetched from secondary storage. If this method returns non-NULL, then it indicates
// that the query system layer should yield its locks, following the protocol defined by the
// RecordFetcher class, so that a potential page fault is triggered out of the lock.
//
// Storage engines which support document-level locking need not implement this.
//
// TODO see if these can be replaced by WriteConflictException.
//
/**
* Returns a RecordFetcher if needed for a call to next() or none if unneeded.
*/
virtual std::unique_ptr<RecordFetcher> fetcherForNext() const {
return {};
}
};
/**
* Adds explicit seeking of records. This functionality is separated out from RecordCursor,
* because some cursors, such as repair cursors, are not required to support seeking.
*
* Warning: MMAPv1 cannot detect if RecordIds are valid. Therefore callers should only pass
* potentially deleted RecordIds to seek methods if they know that MMAPv1 is not the current
* storage engine. All new storage engines must support detecting the existence of Records.
*
*/
class SeekableRecordCursor : public RecordCursor {
public:
/**
* Seeks to a Record with the provided id.
*
* If an exact match can't be found, boost::none will be returned and the resulting position
* of the cursor is unspecified.
*/
virtual boost::optional<Record> seekExact(const RecordId& id) = 0;
/**
* Prepares for state changes in underlying data without necessarily saving the current
* state.
*
* The cursor's position when restored is unspecified. Caller is expected to seek rather
* than call next() following the restore.
*
* It is safe to call saveUnpositioned multiple times in a row.
* No other method (excluding destructor) may be called until successfully restored.
*/
virtual void saveUnpositioned() {
save();
}
/**
* Returns a RecordFetcher if needed to fetch the provided Record or none if unneeded.
*/
virtual std::unique_ptr<RecordFetcher> fetcherForId(const RecordId& id) const {
return {};
}
};
/**
* A RecordStore provides an abstraction used for storing documents in a collection,
* or entries in an index. In storage engines implementing the KVEngine, record stores
* are also used for implementing catalogs.
*
* Many methods take an OperationContext parameter. This contains the RecoveryUnit, with
* all RecordStore specific transaction information, as well as the LockState. Methods that take
* an OperationContext may throw a WriteConflictException.
*/
class RecordStore {
MONGO_DISALLOW_COPYING(RecordStore);
public:
RecordStore(StringData ns) : _ns(ns.toString()) {}
virtual ~RecordStore() {}
// META
// name of the RecordStore implementation
virtual const char* name() const = 0;
virtual const std::string& ns() const {
return _ns;
}
/**
* The dataSize is an approximation of the sum of the sizes (in bytes) of the
* documents or entries in the recordStore.
*/
virtual long long dataSize(OperationContext* txn) const = 0;
/**
* Total number of record in the RecordStore. You may need to cache it, so this call
* takes constant time, as it is called often.
*/
virtual long long numRecords(OperationContext* txn) const = 0;
virtual bool isCapped() const = 0;
virtual void setCappedCallback(CappedCallback*) {
invariant(false);
}
/**
* @param extraInfo - optional more debug info
* @param level - optional, level of debug info to put in (higher is more)
* @return total estimate size (in bytes) on stable storage
*/
virtual int64_t storageSize(OperationContext* txn,
BSONObjBuilder* extraInfo = NULL,
int infoLevel = 0) const = 0;
// CRUD related
/**
* Get the RecordData at loc, which must exist.
*
* If unowned data is returned, it is valid until the next modification of this Record or
* the lock on this collection is released.
*
* In general, prefer findRecord or RecordCursor::seekExact since they can tell you if a
* record has been removed.
*/
virtual RecordData dataFor(OperationContext* txn, const RecordId& loc) const {
RecordData data;
invariant(findRecord(txn, loc, &data));
return data;
}
/**
* @param out - If the record exists, the contents of this are set.
* @return true iff there is a Record for loc
*
* If unowned data is returned, it is valid until the next modification of this Record or
* the lock on this collection is released.
*
* In general prefer RecordCursor::seekExact since it can avoid copying data in more
* storageEngines.
*
* Warning: MMAPv1 cannot detect if RecordIds are valid. Therefore callers should only pass
* potentially deleted RecordIds to seek methods if they know that MMAPv1 is not the current
* storage engine. All new storage engines must support detecting the existence of Records.
*/
virtual bool findRecord(OperationContext* txn, const RecordId& loc, RecordData* out) const {
auto cursor = getCursor(txn);
auto record = cursor->seekExact(loc);
if (!record)
return false;
record->data.makeOwned(); // Unowned data expires when cursor goes out of scope.
*out = std::move(record->data);
return true;
}
virtual void deleteRecord(OperationContext* txn, const RecordId& dl) = 0;
virtual StatusWith<RecordId> insertRecord(OperationContext* txn,
const char* data,
int len,
bool enforceQuota) = 0;
virtual StatusWith<RecordId> insertRecord(OperationContext* txn,
const DocWriter* doc,
bool enforceQuota) = 0;
virtual Status insertRecords(OperationContext* txn,
std::vector<Record>* records,
bool enforceQuota) {
for (auto& record : *records) {
StatusWith<RecordId> res =
insertRecord(txn, record.data.data(), record.data.size(), enforceQuota);
if (!res.isOK())
return res.getStatus();
record.id = res.getValue();
}
return Status::OK();
}
/**
* @param notifier - Only used by record stores which do not support doc-locking.
* In the case of a document move, this is called after the document
* has been written to the new location, but before it is deleted from
* the old location.
* In the case of an in-place update, this is called just before the
* in-place write occurs.
* @return Status or RecordId, RecordId might be different
*
* For capped record stores, the record size will never change.
*/
virtual StatusWith<RecordId> updateRecord(OperationContext* txn,
const RecordId& oldLocation,
const char* data,
int len,
bool enforceQuota,
UpdateNotifier* notifier) = 0;
/**
* @return Returns 'false' if this record store does not implement
* 'updatewithDamages'. If this method returns false, 'updateWithDamages' must not be
* called, and all updates must be routed through 'updateRecord' above. This allows the
* update framework to avoid doing the work of damage tracking if the underlying record
* store cannot utilize that information.
*/
virtual bool updateWithDamagesSupported() const = 0;
/**
* Updates the record positioned at 'loc' in-place using the deltas described by 'damages'. The
* 'damages' vector describes contiguous ranges of 'damageSource' from which to copy and apply
* byte-level changes to the data.
*
* @return the updated version of the record. If unowned data is returned, then it is valid
* until the next modification of this Record or the lock on the collection has been released.
*/
virtual StatusWith<RecordData> updateWithDamages(OperationContext* txn,
const RecordId& loc,
const RecordData& oldRec,
const char* damageSource,
const mutablebson::DamageVector& damages) = 0;
/**
* Returns a new cursor over this record store.
*
* The cursor is logically positioned before the first (or last if !forward) Record in the
* collection so that Record will be returned on the first call to next(). Implementations
* are allowed to lazily seek to the first Record when next() is called rather than doing
* it on construction.
*/
virtual std::unique_ptr<SeekableRecordCursor> getCursor(OperationContext* txn,
bool forward = true) const = 0;
/**
* Constructs a cursor over a potentially corrupted store, which can be used to salvage
* damaged records. The iterator might return every record in the store if all of them
* are reachable and not corrupted. Returns NULL if not supported.
*/
virtual std::unique_ptr<RecordCursor> getCursorForRepair(OperationContext* txn) const {
return {};
}
/**
* Constructs a cursor over a record store that returns documents in a randomized order, and
* allows storage engines to provide a more efficient way of random sampling of a record store
* than MongoDB's default sampling methods, which is used when this method returns {}.
*
* This method may be implemented using a pseudo-random walk over B-trees or a similar approach.
* Different cursors should return documents in a different order. Random cursors may return
* the same document more than once and, as a result, may return more documents than exist in
* the record store. Implementations should avoid obvious biases toward older, newer, larger
* smaller or other specific classes of documents.
*/
virtual std::unique_ptr<RecordCursor> getRandomCursor(OperationContext* txn) const {
return {};
}
/**
* Returns many RecordCursors that partition the RecordStore into many disjoint sets.
* Iterating all returned RecordCursors is equivalent to iterating the full store.
*/
virtual std::vector<std::unique_ptr<RecordCursor>> getManyCursors(OperationContext* txn) const {
std::vector<std::unique_ptr<RecordCursor>> out(1);
out[0] = getCursor(txn);
return out;
}
// higher level
/**
* removes all Records
*/
virtual Status truncate(OperationContext* txn) = 0;
/**
* Truncate documents newer than the document at 'end' from the capped
* collection. The collection cannot be completely emptied using this
* function. An assertion will be thrown if that is attempted.
* @param inclusive - Truncate 'end' as well iff true
* XXX: this will go away soon, just needed to move for now
*/
virtual void temp_cappedTruncateAfter(OperationContext* txn, RecordId end, bool inclusive) = 0;
/**
* does this RecordStore support the compact operation?
*
* If you return true, you must provide implementations of all compact methods.
*/
virtual bool compactSupported() const {
return false;
}
/**
* Does compact() leave RecordIds alone or can they change.
*
* Only called if compactSupported() returns true.
*/
virtual bool compactsInPlace() const {
invariant(false);
}
/**
* Attempt to reduce the storage space used by this RecordStore.
*
* Only called if compactSupported() returns true.
* No RecordStoreCompactAdaptor will be passed if compactsInPlace() returns true.
*/
virtual Status compact(OperationContext* txn,
RecordStoreCompactAdaptor* adaptor,
const CompactOptions* options,
CompactStats* stats) {
invariant(false);
}
/**
* @param full - does more checks
* @param scanData - scans each document
* @return OK if the validate run successfully
* OK will be returned even if corruption is found
* deatils will be in result
*/
virtual Status validate(OperationContext* txn,
bool full,
bool scanData,
ValidateAdaptor* adaptor,
ValidateResults* results,
BSONObjBuilder* output) = 0;
/**
* @param scaleSize - amount by which to scale size metrics
* appends any custom stats from the RecordStore or other unique stats
*/
virtual void appendCustomStats(OperationContext* txn,
BSONObjBuilder* result,
double scale) const = 0;
/**
* Load all data into cache.
* What cache depends on implementation.
*
* If the underlying storage engine does not support the operation,
* returns ErrorCodes::CommandNotSupported
*
* @param output (optional) - where to put detailed stats
*/
virtual Status touch(OperationContext* txn, BSONObjBuilder* output) const {
return Status(ErrorCodes::CommandNotSupported,
"this storage engine does not support touch");
}
/**
* Return the RecordId of an oplog entry as close to startingPosition as possible without
* being higher. If there are no entries <= startingPosition, return RecordId().
*
* If you don't implement the oplogStartHack, just use the default implementation which
* returns boost::none.
*/
virtual boost::optional<RecordId> oplogStartHack(OperationContext* txn,
const RecordId& startingPosition) const {
return boost::none;
}
/**
* When we write to an oplog, we call this so that if the storage engine
* supports doc locking, it can manage the visibility of oplog entries to ensure
* they are ordered.
*
* Since this is called inside of a WriteUnitOfWork while holding a std::mutex, it is
* illegal to acquire any LockManager locks inside of this function.
*/
virtual Status oplogDiskLocRegister(OperationContext* txn, const Timestamp& opTime) {
return Status::OK();
}
/**
* Called after a repair operation is run with the recomputed numRecords and dataSize.
*/
virtual void updateStatsAfterRepair(OperationContext* txn,
long long numRecords,
long long dataSize) = 0;
protected:
std::string _ns;
};
class RecordStoreCompactAdaptor {
public:
virtual ~RecordStoreCompactAdaptor() {}
virtual bool isDataValid(const RecordData& recData) = 0;
virtual size_t dataSize(const RecordData& recData) = 0;
virtual void inserted(const RecordData& recData, const RecordId& newLocation) = 0;
};
struct ValidateResults {
ValidateResults() {
valid = true;
}
bool valid;
std::vector<std::string> errors;
std::vector<std::string> warnings;
};
/**
* This is so when a RecordStore is validating all records
* it can call back to someone to check if a record is valid.
* The actual data contained in a Record is totally opaque to the implementation.
*/
class ValidateAdaptor {
public:
virtual ~ValidateAdaptor() {}
virtual Status validate(const RecordData& recordData, size_t* dataSize) = 0;
};
}
|