1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
|
/**
* Utilities for testing cluster to cluster replicator.
*/
let ClusterToClusterUtil = (function() {
load("jstests/libs/namespace_utils.js");
// System databases and collections that are excluded from copying.
const excludedSystemDatabases = ["admin", "config", "local"];
const excludedSystemCollections =
["system.views", "system.profile", "system.resharding.", "system.buckets.", "system.drop."];
/**
* Perform sanity check on the namespaces to filter.
*/
function checkFilteredNamespacesInput(namespaces) {
if (namespaces) {
for (const ns of namespaces) {
const [db, coll] = getDBNameAndCollNameFromFullNamespace(ns);
assert(db && coll, `Incorrect namespace format: ${ns}`);
assert(!excludedSystemDatabases.includes(db),
"Filtered namespaces cannot contain excluded system databases");
assert(!coll.startsWith("system."));
}
}
}
/**
* Return the databases to copy from the source cluster.
*/
function getDatabasesToCopy(conn) {
const listDBRes = assert.commandWorked(conn.adminCommand(
{listDatabases: 1, filter: {name: {$nin: excludedSystemDatabases}}, nameOnly: true}));
return listDBRes.databases.map(entry => entry.name);
}
/**
* Return all the collections to copy from the source cluster, grouped by database and
* filtered by includeNamespaces and excludeNamespaces. When includeNamespaces is not
* provided, the collection infos returned will include views, so callers may need to
* explicitly check the collection type to different between collections and views.
*/
function getCollectionsToCopy(conn, includeNamespaces, excludeNamespaces) {
const collInfoMap = {};
if (includeNamespaces && includeNamespaces.length > 0) {
assert(!excludeNamespaces || excludeNamespaces.size == 0,
"Cannot have inputs for both includeNamespaces and excludeNamespaces");
checkFilteredNamespacesInput(includeNamespaces);
for (const ns of includeNamespaces) {
const [dbName, collName] = getDBNameAndCollNameFromFullNamespace(ns);
const collInfo = getCollectionInfo(conn, dbName, collName);
if (!collInfo) {
print(`Namespace to include for copy does not exist: ${dbName}.${collName}`);
continue;
}
if (!collInfoMap.hasOwnProperty(dbName)) {
collInfoMap[dbName] = [];
}
collInfoMap[dbName].push(collInfo);
}
return collInfoMap;
}
checkFilteredNamespacesInput(excludeNamespaces);
const databases = getDatabasesToCopy(conn);
databases.forEach(dbName => {
const collInfos = getCollectionsFromDatabase(conn, dbName, excludeNamespaces);
if (collInfos.length > 0) {
collInfoMap[dbName] = collInfos;
}
});
return collInfoMap;
}
/**
* Return the collection infos of the given database, excluding those in the excludeNamespaces.
*/
function getCollectionsFromDatabase(conn, dbName, excludeNamespaces = []) {
let excludedCollections = excludeNamespaces.reduce((list, ns) => {
const [db, coll] = getDBNameAndCollNameFromFullNamespace(ns);
if (db === dbName) {
list.push(coll);
}
return list;
}, [...excludedSystemCollections]);
excludedCollections = excludedCollections.map(coll => {
// If collection ends with '.', match the prefix
return coll.endsWith('.') ? new RegExp(`^${coll}`) : coll;
});
const res = assert.commandWorked(conn.getDB(dbName).runCommand(
{listCollections: 1, filter: {name: {$nin: excludedCollections}}}));
return new DBCommandCursor(db, res).toArray().sort(compareOn("name"));
}
/**
* Return the collection info of the given collection name, or null if no such collection.
*/
function getCollectionInfo(conn, dbName, collName) {
const res = assert.commandWorked(
conn.getDB(dbName).runCommand({listCollections: 1, filter: {name: collName}}));
const firstBatch = res.cursor.firstBatch;
return firstBatch.length > 0 ? firstBatch[0] : null;
}
/**
* Return the shard key information of the given collection, or null if the collection
* is not sharded.
*/
function getShardKeyInfo(conn, dbName, collName) {
return conn.getDB("config").collections.findOne({_id: `${dbName}.${collName}`});
}
return {
checkFilteredNamespacesInput,
getDatabasesToCopy,
getCollectionsToCopy,
getCollectionsFromDatabase,
getCollectionInfo,
getShardKeyInfo,
};
})();
|
