1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
|
load("jstests/libs/parallelTester.js"); // for Thread.
function backupData(mongo, destinationDirectory) {
let backupCursor = openBackupCursor(mongo);
let metadata = getBackupCursorMetadata(backupCursor);
copyBackupCursorFiles(
backupCursor, /*namespacesToSkip=*/[], metadata.dbpath, destinationDirectory);
backupCursor.close();
return metadata;
}
function openBackupCursor(mongo) {
// Opening a backup cursor can race with taking a checkpoint, resulting in a transient
// error. Retry until it succeeds.
while (true) {
try {
return mongo.getDB("admin").aggregate([{$backupCursor: {}}]);
} catch (exc) {
jsTestLog({"Failed to open a backup cursor, retrying.": exc});
}
}
}
function extendBackupCursor(mongo, backupId, extendTo) {
return mongo.getDB("admin").aggregate(
[{$backupCursorExtend: {backupId: backupId, timestamp: extendTo}}],
{maxTimeMS: 180 * 1000});
}
function startHeartbeatThread(host, backupCursor, session, stopCounter) {
let cursorId = tojson(backupCursor._cursorid);
let lsid = tojson(session.getSessionId());
let heartbeatBackupCursor = function(host, cursorId, lsid, stopCounter) {
const conn = new Mongo(host);
const db = conn.getDB("admin");
while (stopCounter.getCount() > 0) {
let res = assert.commandWorked(db.runCommand({
getMore: eval("(" + cursorId + ")"),
collection: "$cmd.aggregate",
lsid: eval("(" + lsid + ")")
}));
sleep(10 * 1000);
}
};
heartbeater = new Thread(heartbeatBackupCursor, host, cursorId, lsid, stopCounter);
heartbeater.start();
return heartbeater;
}
function getBackupCursorMetadata(backupCursor) {
assert(backupCursor.hasNext());
let doc = backupCursor.next();
assert(doc.hasOwnProperty("metadata"));
return doc["metadata"];
}
/**
* Exhaust the backup cursor and copy all the listed files to the destination directory. If `async`
* is true, this function will spawn a Thread doing the copy work and return the thread along
* with the backup cursor metadata. The caller should `join` the thread when appropriate.
*/
function copyBackupCursorFiles(
backupCursor, namespacesToSkip, dbpath, destinationDirectory, async, fileCopiedCallback) {
resetDbpath(destinationDirectory);
mkdir(destinationDirectory + "/journal");
let copyThread = copyBackupCursorExtendFiles(
backupCursor, namespacesToSkip, dbpath, destinationDirectory, async, fileCopiedCallback);
return copyThread;
}
function copyBackupCursorExtendFiles(
cursor, namespacesToSkip, dbpath, destinationDirectory, async, fileCopiedCallback) {
let files = _cursorToFiles(cursor, namespacesToSkip, fileCopiedCallback);
let copyThread;
if (async) {
copyThread = new Thread(_copyFiles, files, dbpath, destinationDirectory, _copyFileHelper);
copyThread.start();
} else {
_copyFiles(files, dbpath, destinationDirectory, _copyFileHelper);
}
jsTestLog({
msg: "Destination",
destination: destinationDirectory,
dbpath: ls(destinationDirectory),
journal: ls(destinationDirectory + "/journal")
});
return copyThread;
}
function _cursorToFiles(cursor, namespacesToSkip, fileCopiedCallback) {
let files = [];
while (cursor.hasNext()) {
let doc = cursor.next();
assert(doc.hasOwnProperty("filename"));
if (namespacesToSkip.includes(doc.ns)) {
jsTestLog("Skipping file during backup: " + tojson(doc));
continue;
}
if (fileCopiedCallback) {
fileCopiedCallback(doc);
}
files.push(doc.filename);
}
return files;
}
function _copyFiles(files, dbpath, destinationDirectory, copyFileHelper) {
files.forEach((file) => {
let dbgDoc = copyFileHelper(file, dbpath, destinationDirectory);
dbgDoc["msg"] = "File copy";
jsTestLog(dbgDoc);
});
}
function _copyFileHelper(absoluteFilePath, sourceDbPath, destinationDirectory) {
// Ensure the dbpath ends with an OS appropriate slash.
let separator = '/';
if (_isWindows()) {
separator = '\\';
}
let lastChar = sourceDbPath[sourceDbPath.length - 1];
if (lastChar !== '/' && lastChar !== '\\') {
sourceDbPath += separator;
}
// Ensure that the full path starts with the returned dbpath.
assert.eq(0, absoluteFilePath.indexOf(sourceDbPath));
// Grab the file path relative to the dbpath. Maintain that relation when copying
// to the `hiddenDbpath`.
let relativePath = absoluteFilePath.substr(sourceDbPath.length);
let destination = destinationDirectory + separator + relativePath;
const newFileDirectory = destination.substring(0, destination.lastIndexOf(separator));
mkdir(newFileDirectory);
copyFile(absoluteFilePath, destination);
return {fileSource: absoluteFilePath, relativePath: relativePath, fileDestination: destination};
}
|
