1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
|
/**
* This test is intended to exercise shard filtering logic. This test works by sharding a
* collection, and then inserting orphaned documents directly into one of the shards. It then runs a
* find() and makes sure that orphaned documents are filtered out.
* @tags: [
* requires_sharding,
* # TODO SERVER-71169: Implement shard filtering for CQF.
* cqf_incompatible,
* ]
*/
(function() {
"use strict";
load("jstests/libs/analyze_plan.js");
// Deliberately inserts orphans outside of migration.
TestData.skipCheckOrphans = true;
const st = new ShardingTest({shards: 2});
const collName = "test.shardfilter";
const mongosDb = st.s.getDB("test");
const mongosColl = st.s.getCollection(collName);
assert.commandWorked(st.s.adminCommand({enableSharding: "test"}));
st.ensurePrimaryShard("test", st.shard1.name);
assert.commandWorked(
st.s.adminCommand({shardCollection: collName, key: {a: 1, "b.c": 1, "d.e.f": 1}}));
// Put a chunk with no data onto shard0 in order to make sure that both shards get targeted.
assert.commandWorked(st.s.adminCommand({split: collName, middle: {a: 20, "b.c": 0, "d.e.f": 0}}));
assert.commandWorked(st.s.adminCommand({split: collName, middle: {a: 30, "b.c": 0, "d.e.f": 0}}));
assert.commandWorked(st.s.adminCommand(
{moveChunk: collName, find: {a: 25, "b.c": 0, "d.e.f": 0}, to: st.shard0.shardName}));
// Shard the collection and insert some docs.
const docs = [
{_id: 0, a: 1, b: {c: 1}, d: {e: {f: 1}}, g: 100},
{_id: 1, a: 1, b: {c: 2}, d: {e: {f: 2}}, g: 100.9},
{_id: 2, a: 1, b: {c: 3}, d: {e: {f: 3}}, g: "a"},
{_id: 3, a: 1, b: {c: 3}, d: {e: {f: 3}}, g: [1, 2, 3]},
{_id: 4, a: "a", b: {c: "b"}, d: {e: {f: "c"}}, g: null},
{_id: 5, a: 1.0, b: {c: "b"}, d: {e: {f: Infinity}}, g: NaN}
];
assert.commandWorked(mongosColl.insert(docs));
assert.eq(mongosColl.find().itcount(), 6);
// Insert some documents with valid partial shard keys to both shards. The versions of these
// documents on shard0 are orphans, since all of the data is owned by shard1.
const docsWithMissingAndNullKeys = [
{_id: 6, a: "missingParts"},
{_id: 7, a: null, b: {c: 1}, d: {e: {f: 1}}},
{_id: 8, a: "null", b: {c: null}, d: {e: {f: 1}}},
{_id: 9, a: "deepNull", b: {c: 1}, d: {e: {f: null}}},
];
assert.commandWorked(st.shard0.getCollection(collName).insert(docsWithMissingAndNullKeys));
assert.commandWorked(st.shard1.getCollection(collName).insert(docsWithMissingAndNullKeys));
// Insert orphan docs without missing or null shard keys onto shard0 and test that they get filtered
// out.
const orphanDocs = [
{_id: 10, a: 100, b: {c: 10}, d: {e: {f: 999}}, g: "a"},
{_id: 11, a: 101, b: {c: 11}, d: {e: {f: 1000}}, g: "b"}
];
assert.commandWorked(st.shard0.getCollection(collName).insert(orphanDocs));
assert.eq(mongosColl.find().itcount(), 10);
// Insert docs directly into shard0 to test that regular (non-null, non-missing) shard keys get
// filtered out.
assert.commandWorked(st.shard0.getCollection(collName).insert(docs));
assert.eq(mongosColl.find().itcount(), 10);
// Ensure that shard filtering works correctly for a query that can use the index supporting the
// shard key. In this case, shard filtering can occur before the FETCH stage, but the plan is not
// covered.
let explain = mongosColl.find({a: {$gte: 0}}).explain();
assert.eq(explain.queryPlanner.winningPlan.stage, "SHARD_MERGE", explain);
assert(planHasStage(mongosDb, explain.queryPlanner.winningPlan, "SHARDING_FILTER"), explain);
assert(isIxscan(mongosDb, explain.queryPlanner.winningPlan), explain);
assert(!isIndexOnly(mongosDb, explain.queryPlanner.winningPlan), explain);
assert.sameMembers(mongosColl.find({a: {$gte: 0}}).toArray(), [
{_id: 0, a: 1, b: {c: 1}, d: {e: {f: 1}}, g: 100},
{_id: 1, a: 1, b: {c: 2}, d: {e: {f: 2}}, g: 100.9},
{_id: 2, a: 1, b: {c: 3}, d: {e: {f: 3}}, g: "a"},
{_id: 3, a: 1, b: {c: 3}, d: {e: {f: 3}}, g: [1, 2, 3]},
{_id: 5, a: 1, b: {c: "b"}, d: {e: {f: Infinity}}, g: NaN}
]);
// In this case, shard filtering is done as part of a covered plan.
explain = mongosColl.find({a: {$gte: 0}}, {_id: 0, a: 1}).explain();
assert.eq(explain.queryPlanner.winningPlan.stage, "SHARD_MERGE", explain);
assert(planHasStage(mongosDb, explain.queryPlanner.winningPlan, "SHARDING_FILTER"), explain);
assert(isIxscan(mongosDb, explain.queryPlanner.winningPlan), explain);
assert(isIndexOnly(mongosDb, explain.queryPlanner.winningPlan), explain);
assert.sameMembers(mongosColl.find({a: {$gte: 0}}, {_id: 0, a: 1}).toArray(), [
{a: 1},
{a: 1},
{a: 1},
{a: 1},
{a: 1},
]);
// Drop the collection and shard it by a new key that has no dotted fields. Again, make sure that
// shard0 has an empty chunk.
assert(mongosColl.drop());
assert.commandWorked(st.s.adminCommand({shardCollection: collName, key: {a: 1, b: 1, c: 1, d: 1}}));
assert.commandWorked(st.s.adminCommand({split: collName, middle: {a: 20, b: 0, c: 0, d: 0}}));
assert.commandWorked(st.s.adminCommand({split: collName, middle: {a: 30, b: 0, c: 0, d: 0}}));
assert.commandWorked(st.s.adminCommand(
{moveChunk: collName, find: {a: 25, b: 0, c: 0, d: 0}, to: st.shard0.shardName}));
// Insert some data via mongos, and also insert some documents directly to shard0 to produce an
// orphans.
assert.commandWorked(mongosColl.insert([
{_id: 0, a: 0, b: 0, c: 0, d: 0},
{_id: 1, a: 1, b: 1, c: 1, d: 1},
{_id: 2, a: -1, b: -1, c: -1, d: -1},
]));
assert.commandWorked(st.shard0.getCollection(collName).insert({_id: 3, a: 0, b: 0, c: 0, d: 0}));
assert.commandWorked(st.shard0.getCollection(collName).insert({_id: 4, a: 0, b: 99, c: 0, d: 99}));
assert.commandWorked(st.shard0.getCollection(collName).insert({_id: 5, a: 0, b: 0, c: 99, d: 99}));
assert.commandWorked(st.shard0.getCollection(collName).insert({_id: 6, a: 0, b: 99, c: 99, d: 99}));
// Run a query that can use covered shard filtering where the projection involves more than one
// field of the shard key.
explain = mongosColl.find({a: {$gte: 0}}, {_id: 0, a: 1, b: 1, d: 1}).explain();
assert.eq(explain.queryPlanner.winningPlan.stage, "SHARD_MERGE", explain);
assert(planHasStage(mongosDb, explain.queryPlanner.winningPlan, "SHARDING_FILTER"), explain);
assert(isIxscan(mongosDb, explain.queryPlanner.winningPlan), explain);
assert(isIndexOnly(mongosDb, explain.queryPlanner.winningPlan), explain);
assert.sameMembers(mongosColl.find({a: {$gte: 0}}, {_id: 0, a: 1, b: 1, d: 1}).toArray(),
[{a: 0, b: 0, d: 0}, {a: 1, b: 1, d: 1}]);
// Run a query that will use a covered OR plan.
explain = mongosColl.find({$or: [{a: 0, c: 0}, {a: 25, c: 0}]}, {_id: 0, a: 1, c: 1}).explain();
assert.eq(explain.queryPlanner.winningPlan.stage, "SHARD_MERGE", explain);
assert(planHasStage(mongosDb, explain.queryPlanner.winningPlan, "SHARDING_FILTER"), explain);
assert(planHasStage(mongosDb, explain.queryPlanner.winningPlan, "OR"), explain);
assert(isIndexOnly(mongosDb, explain.queryPlanner.winningPlan), explain);
assert.sameMembers(
mongosColl.find({$or: [{a: 0, c: 0}, {a: 25, c: 0}]}, {_id: 0, a: 1, c: 1}).toArray(),
[{a: 0, c: 0}]);
// Similar case to above, but here the index scans involve a single interval of the index.
explain = mongosColl.find({$or: [{a: 0, b: 0}, {a: 25, b: 0}]}, {_id: 0, a: 1, b: 1}).explain();
assert.eq(explain.queryPlanner.winningPlan.stage, "SHARD_MERGE", explain);
assert(planHasStage(mongosDb, explain.queryPlanner.winningPlan, "SHARDING_FILTER"), explain);
assert(planHasStage(mongosDb, explain.queryPlanner.winningPlan, "OR"), explain);
assert(isIndexOnly(mongosDb, explain.queryPlanner.winningPlan), explain);
assert.sameMembers(
mongosColl.find({$or: [{a: 0, b: 0}, {a: 25, b: 0}]}, {_id: 0, a: 1, b: 1}).toArray(),
[{a: 0, b: 0}]);
st.stop();
})();
|
