1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
|
// Test that a pipeline with a $lookup stage on a sharded foreign collection may be run on a mongod.
// This test uses the flag-guarded sharded $lookup parameter, which results in cursors being
// established from a shard instead of mongos. As of SERVER-46255, these requests will include the
// 'clientOperationKey' which is not recognizable on 'last-stable' shards.
// @tags: [requires_fcv_44]
(function() {
load("jstests/noPassthrough/libs/server_parameter_helpers.js"); // For setParameterOnAllHosts.
load("jstests/libs/discover_topology.js"); // For findDataBearingNodes.
const sharded = new ShardingTest({mongos: 1, shards: 2});
setParameterOnAllHosts(
DiscoverTopology.findNonConfigNodes(sharded.s), "internalQueryAllowShardedLookup", true);
assert.commandWorked(sharded.s.adminCommand({enableSharding: "test"}));
sharded.ensurePrimaryShard('test', sharded.shard0.shardName);
const coll = sharded.s.getDB('test').mainColl;
const foreignColl = sharded.s.getDB('test').foreignColl;
const smallColl = sharded.s.getDB("test").smallColl;
const nDocsMainColl = 10;
const nDocsForeignColl = 2 * nDocsMainColl;
for (let i = 0; i < nDocsMainColl; i++) {
assert.commandWorked(coll.insert({_id: i, collName: "mainColl", foreignId: i}));
assert.commandWorked(
foreignColl.insert({_id: 2 * i, key: i, collName: "foreignColl", data: "hello-0"}));
assert.commandWorked(
foreignColl.insert({_id: 2 * i + 1, key: i, collName: "foreignColl", data: "hello-1"}));
}
assert.commandWorked(smallColl.insert({_id: 0, collName: "smallColl"}));
const runTest = function() {
(function testSingleLookupFromShard() {
// Run a pipeline which must be merged on a shard. This should force the $lookup (on
// the sharded collection) to be run on a mongod.
pipeline = [
{
$lookup: {
localField: "foreignId",
foreignField: "key",
from: "foreignColl",
as: "foreignDoc"
}
},
{$_internalSplitPipeline: {mergeType: "anyShard"}}
];
const results = coll.aggregate(pipeline).toArray();
assert.eq(results.length, nDocsMainColl);
for (let i = 0; i < results.length; i++) {
assert.eq(results[i].foreignDoc.length, 2, results[i]);
}
})();
(function testMultipleLookupsFromShard() {
// Run two lookups in a row (both on mongod).
pipeline = [
{
$lookup: {
localField: "foreignId",
foreignField: "key",
from: "foreignColl",
as: "foreignDoc"
}
},
{
$lookup: {
from: "smallColl",
as: "smallCollDocs",
pipeline: [],
}
},
{$_internalSplitPipeline: {mergeType: "anyShard"}}
];
const results = coll.aggregate(pipeline).toArray();
assert.eq(results.length, nDocsMainColl);
for (let i = 0; i < results.length; i++) {
assert.eq(results[i].foreignDoc.length, 2, results[i]);
assert.eq(results[i].smallCollDocs.length, 1, results[i]);
}
})();
(function testUnshardedLookupWithinShardedLookup() {
// Pipeline with unsharded $lookup inside a sharded $lookup.
pipeline = [
{
$lookup: {
from: "foreignColl",
as: "foreignDoc",
pipeline: [
{$lookup: {from: "smallColl", as: "doc", pipeline: []}},
],
}
},
{$_internalSplitPipeline: {mergeType: "anyShard"}}
];
const results = coll.aggregate(pipeline).toArray();
assert.eq(results.length, nDocsMainColl);
for (let i = 0; i < results.length; i++) {
assert.eq(results[i].foreignDoc.length, nDocsForeignColl);
for (let j = 0; j < nDocsForeignColl; j++) {
// Each document pulled from the foreign collection should have one document
// from "smallColl."
assert.eq(results[i].foreignDoc[j].collName, "foreignColl");
// TODO SERVER-39016: Once a mongod is able to target the primary shard when
// reading from a non-sharded collection this should always work. Until then,
// the results of the query depend on which shard is chosen as the merging
// shard. If the primary shard is chosen, we'll get the correct results (and
// correctly find a document in "smallColl"). Otherwise if the merging shard is
// not the primary shard, the merging shard will attempt to do a local read (on
// an empty/non-existent collection), which will return nothing.
if (results[i].foreignDoc[j].doc.length === 1) {
assert.eq(results[i].foreignDoc[j].doc[0].collName, "smallColl");
} else {
assert.eq(results[i].foreignDoc[j].doc.length, 0);
}
}
}
})();
};
jsTestLog("Running test with neither collection sharded");
runTest();
jsTestLog("Running test with foreign collection sharded");
sharded.shardColl(
"foreignColl",
{_id: 1}, // shard key
{_id: 5}, // split
{_id: 5}, // move
"test", // dbName
true // waitForDelete
);
runTest();
jsTestLog("Running test with main and foreign collection sharded");
sharded.shardColl(
"mainColl",
{_id: 1}, // shard key
{_id: 5}, // split
{_id: 5}, // move
"test", // dbName
true // waitForDelete
);
runTest();
sharded.stop();
})();
|
