1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
|
/**
* Testing of just the query layer's integration for columnar index.
* @tags: [
* # Runs explain on an aggregate command which is only compatible with readConcern local.
* assumes_read_concern_unchanged,
* # column store indexes are still under a feature flag and require full sbe
* uses_column_store_index,
* featureFlagColumnstoreIndexes,
* featureFlagSbeFull,
* # TODO SERVER-69884: featureFlag guarded tests shouldn't require explicit 'no_selinux' tag.
* no_selinux,
* ]
*/
(function() {
"use strict";
load("jstests/libs/fail_point_util.js");
load("jstests/libs/analyze_plan.js"); // For "planHasStage."
load("jstests/aggregation/extras/utils.js"); // For "resultsEq."
const coll = db.columnstore_index_correctness;
(function testColumnScanIsUsed() {
coll.drop();
coll.insert([{_id: 0, x: 42}]); // the content doesn't matter for this test
assert.commandWorked(coll.createIndex({"$**": "columnstore"}));
let explain = coll.find({}, {_id: 0, x: 1}).explain();
assert(planHasStage(db, explain, "COLUMN_SCAN"),
"Projection of existing column " + tojson(explain));
explain = coll.find({}, {_id: 0, "x.y.z": 1, a: 1}).explain();
assert(planHasStage(db, explain, "COLUMN_SCAN"),
"Projection of non-existing columns " + tojson(explain));
explain = coll.find({}, {x: 1, a: 1}).explain();
assert(planHasStage(db, explain, "COLUMN_SCAN"),
"Projection includes _id column " + tojson(explain));
explain = coll.find({}, {_id: 0}).explain();
assert(!planHasStage(db, explain, "COLUMN_SCAN"),
"Exclusive projection cannot use column scan " + tojson(explain));
})();
// Multiple tests in this file use the same dataset. Intentionally not using _id as the unique
// identifier, to avoid getting IDHACK plans when we query by it.
const docs = [
{num: 0},
{num: 1, a: null},
{num: 2, a: "scalar"},
{num: 3, a: {}},
{num: 4, a: {x: 1, b: "scalar"}},
{num: 5, a: {b: {}}},
{num: 6, a: {x: 1, b: {}}},
{num: 7, a: {x: 1, b: {x: 1}}},
{num: 8, a: {b: {c: "scalar"}}},
{num: 9, a: {b: {c: null}}},
{num: 10, a: {b: {c: [[1, 2], [{}], 2]}}},
{num: 11, a: {x: 1, b: {x: 1, c: ["scalar"]}}},
{num: 12, a: {x: 1, b: {c: {x: 1}}}},
{num: 13, a: {b: []}},
{num: 14, a: {b: [null]}},
{num: 15, a: {b: ["scalar"]}},
{num: 16, a: {b: [[]]}},
{num: 17, a: {b: [1, {}, 2]}},
{num: 18, a: {b: [[1, 2], [{}], 2]}},
{num: 19, a: {x: 1, b: [[1, 2], [{}], 2]}},
{num: 20, a: {b: [{c: "scalar"}]}},
{num: 21, a: {b: [{c: "scalar"}, {c: "scalar2"}]}},
{num: 22, a: {b: [{c: [[1, 2], [{}], 2]}]}},
{num: 23, a: {b: [1, {c: "scalar"}, 2]}},
{num: 24, a: {b: [1, {c: [[1, 2], [{}], 2]}, 2]}},
{num: 25, a: {x: 1, b: [1, {c: [[1, 2], [{}], 2]}, 2]}},
{num: 26, a: {b: [[1, 2], [{c: "scalar"}], 2]}},
{num: 27, a: {b: [[1, 2], [{c: [[1, 2], [{}], 2]}], 2]}},
{num: 28, a: {x: 1, b: [[1, 2], [{c: [[1, 2], [{}], 2]}], 2]}},
{num: 29, a: []},
{num: 30, a: [null]},
{num: 31, a: ["scalar"]},
{num: 32, a: [[]]},
{num: 33, a: [{}]},
{num: 34, a: [1, {}, 2]},
{num: 35, a: [[1, 2], [{}], 2]},
{num: 36, a: [{b: "scalar"}]},
{num: 37, a: [{b: null}]},
{num: 38, a: [1, {b: "scalar"}, 2]},
{num: 39, a: [1, {b: []}, 2]},
{num: 40, a: [1, {b: [null]}, 2]},
{num: 41, a: [1, {b: ["scalar"]}, 2]},
{num: 42, a: [1, {b: [[]]}, 2]},
{num: 43, a: [{b: []}]},
{num: 44, a: [{b: ["scalar"]}]},
{num: 45, a: [{b: [[]]}]},
{num: 46, a: [{b: {}}]},
{num: 47, a: [{b: {c: "scalar"}}]},
{num: 48, a: [{b: {c: [[1, 2], [{}], 2]}}]},
{num: 49, a: [{b: {x: 1}}]},
{num: 50, a: [{b: {x: 1, c: "scalar"}}]},
{num: 51, a: [{b: [{c: "scalar"}]}]},
{num: 52, a: [{b: [{c: ["scalar"]}]}]},
{num: 53, a: [{b: [1, {c: ["scalar"]}, 2]}]},
{num: 54, a: [{b: [{}]}]},
{num: 55, a: [{b: [[1, 2], [{}], 2]}]},
{num: 56, a: [{b: [[1, 2], [{c: "scalar"}], 2]}]},
{num: 57, a: [{b: [[1, 2], [{c: ["scalar"]}], 2]}]},
{num: 58, a: [1, {b: {}}, 2]},
{num: 59, a: [1, {b: {c: "scalar"}}, 2]},
{num: 60, a: [1, {b: {c: {x: 1}}}, 2]},
{num: 61, a: [1, {b: {c: [1, {}, 2]}}, 2]},
{num: 62, a: [1, {b: {x: 1}}, 2]},
{num: 63, a: [1, {b: {x: 1, c: "scalar"}}, 2]},
{num: 64, a: [1, {b: {x: 1, c: [[]]}}, 2]},
{num: 65, a: [1, {b: {x: 1, c: [1, {}, 2]}}, 2]},
{num: 66, a: [1, {b: [{}]}, 2]},
{num: 67, a: [1, {b: [{c: "scalar"}]}, 2]},
{num: 68, a: [1, {b: [{c: {x: 1}}]}, 2]},
{num: 69, a: [1, {b: [{c: [1, {}, 2]}]}, 2]},
{num: 70, a: [1, {b: [1, {}, 2]}, 2]},
{num: 71, a: [1, {b: [1, {c: null}, 2]}, 2]},
{num: 72, a: [1, {b: [1, {c: "scalar"}, 2]}, 2]},
{num: 73, a: [1, {b: [1, {c: [1, {}, 2]}, 2]}, 2]},
{num: 74, a: [1, {b: [[1, 2], [{}], 2]}, 2]},
{num: 75, a: [1, {b: [[1, 2], [{c: "scalar"}], 2]}, 2]},
{num: 76, a: [1, {b: [[1, 2], [{c: [1, {}, 2]}], 2]}, 2]},
{num: 77, a: [[1, 2], [{b: "scalar"}], 2]},
{num: 78, a: [[1, 2], [{b: {x: 1, c: "scalar"}}], 2]},
{num: 79, a: [[1, 2], [{b: {x: 1, c: [1, {}, 2]}}], 2]},
{num: 80, a: [[1, 2], [{b: []}], 2]},
{num: 81, a: [[1, 2], [{b: [1, {c: "scalar"}, 2]}], 2]},
{num: 82, a: [[1, 2], [{b: [[1, 2], [{c: "scalar"}], 2]}], 2]},
{num: 83, a: [[1, 2], [{b: [[1, 2], [{c: [[1, 2], [{}], 2]}], 2]}], 2]},
{num: 84, a: [{b: [{c: 1}, {}]}]},
{num: 85, a: [{b: [{c: 1}, {d: 1}]}]},
{num: 86, a: [{b: {c: 1}}, {b: {}}]},
{num: 87, a: [{b: {c: 1}}, {b: {d: 1}}]},
{num: 88, a: [{b: {c: 1}}, {}]},
{num: 89, a: [{b: {c: 1}}, {b: null}]},
{num: 90, a: [{b: {c: 1}}, {b: []}]},
{num: 91, a: [{b: []}, {b: []}]},
{num: 92, a: {b: [{c: [1, 2]}]}},
{num: 93, a: {b: {c: [1, 2]}}},
{num: 94, a: [[1, 2], [{b: [[1, 2], [{c: [[1, 2], [{}], 2]}], 2]}], 2]},
{num: 95, a: [{m: 1, n: 2}, {m: 2, o: 1}]},
];
coll.drop();
let bulk = coll.initializeUnorderedBulkOp();
for (let doc of docs) {
let insertObj = {};
Object.assign(insertObj, doc);
if (doc.num % 2 == 0) {
insertObj.optionalField = "foo";
}
bulk.insert(insertObj);
}
bulk.execute();
assert.commandWorked(coll.createIndex({"$**": "columnstore"}));
(function testProjectionOfIndependentPaths() {
const kProjection = {_id: 0, "a.b.c": 1, num: 1, optionalField: 1};
let explain = coll.find({}, kProjection).explain();
assert(planHasStage(db, explain, "COLUMN_SCAN"),
"Should have used column scan " + tojson(explain));
let results = coll.find({}, kProjection).toArray();
assert.eq(results.length, docs.length, "With no filter should have returned all docs");
for (let res of results) {
const trueResult = coll.find({num: res.num}, kProjection).hint({$natural: 1}).toArray()[0];
const originalDoc = coll.findOne({num: res.num});
assert.eq(res, trueResult, "Mismatched projection of " + tojson(originalDoc));
}
})();
// Run a similar query that projects multiple fields with a shared parent object.
(function testProjectionOfSiblingPaths() {
const kSiblingProjection = {_id: 0, "a.m": 1, "a.n": 1, num: 1};
let explain = coll.find({}, kSiblingProjection).explain();
assert(planHasStage(db, explain, "COLUMN_SCAN"),
"Should have used column scan " + tojson(explain));
let results = coll.find({}, kSiblingProjection).toArray();
assert.eq(results.length, docs.length, "With no filter should have returned all docs");
for (let res of results) {
const trueResult =
coll.find({num: res.num}, kSiblingProjection).hint({$natural: 1}).toArray()[0];
const originalDoc = coll.findOne({num: res.num});
assert.eq(res, trueResult, "Mismatched projection of " + tojson(originalDoc));
}
})();
// Run a query that tests the SERVER-67742 fix.
(function testPrefixPath() {
const kPrefixProjection = {_id: 0, "a": 1, num: 1};
// Have to use the index hint because SERVER-67264 blocks selection of CSI.
let explain = coll.find({"a.m": 1}, kPrefixProjection).hint({"$**": "columnstore"}).explain();
assert(planHasStage(db, explain, "COLUMN_SCAN"),
"Should have used column scan " + tojson(explain));
let results = coll.find({"a.m": 1}, kPrefixProjection).hint({"$**": "columnstore"}).toArray();
let trueResults = coll.find({"a.m": 1}, kPrefixProjection).hint({$natural: 1}).toArray();
assert.eq(results.length,
trueResults.length,
`Should have found the same number of docs but found:\n with index: ${
tojson(results)}\n without index: ${tojson(trueResults)}`);
for (let res of results) {
const trueResult =
coll.find({num: res.num}, kPrefixProjection).hint({$natural: 1}).toArray()[0];
const originalDoc = coll.findOne({num: res.num});
assert.eq(res, trueResult, "Mismatched projection of " + tojson(originalDoc));
}
})();
// Now test grouping semantics. Grouping limits the set of paths visible downstream which should
// allow column scan plans.
(function testGroup() {
// Sanity check that we are comparing the plans we expect to be.
let pipeline = [
{$group: {_id: "$a.b.c", docs: {$push: "$num"}}},
{$set: {docs: {$sortArray: {input: "$docs", sortBy: 1}}}}
];
let naturalExplain = coll.explain().aggregate(pipeline, {hint: {$natural: 1}});
assert(aggPlanHasStage(naturalExplain, "COLLSCAN"), naturalExplain);
let nonHintedExplain = coll.explain().aggregate(pipeline);
assert(aggPlanHasStage(nonHintedExplain, "COLUMN_SCAN"), nonHintedExplain);
assert(!aggPlanHasStage(nonHintedExplain, "PROJECTION_DEFAULT"), nonHintedExplain);
assert(!aggPlanHasStage(nonHintedExplain, "PROJECTION_SIMPLE"), nonHintedExplain);
assert(resultsEq(coll.aggregate(pipeline, {hint: {$natural: 1}}).toArray(),
coll.aggregate(pipeline).toArray()),
() => {
print(`Results mismatch for $group query. Running resultsEq with verbose`);
resultsEq(expectedResults, coll.aggregate(pipeline).toArray(), true);
});
// For readers who are taking on the massachistic task of trying to
// verify that these results are in fact expected, the major expectations are that all arrays
// are traversed and output as the "structure" EXCEPT if there's a doubly nested array without
// any intervening path as in {a: [[{b: {c: 1}}]]}.
const expectedResults = [
{_id: "scalar", docs: [8]},
{_id: ["scalar", "scalar2"], docs: [21]},
{_id: ["scalar"], docs: [11, 20, 23, 47, 50, 59, 63]},
{_id: [1, 2], docs: [93]},
{_id: [1, []], docs: [90]},
{_id: [1], docs: [86, 87, 88, 89]},
{_id: [["scalar"]], docs: [51, 67, 72]},
{_id: [[1, 2], [{}], 2], docs: [10]},
{_id: [[1, 2]], docs: [92]},
{_id: [[1, {}, 2]], docs: [61, 65]},
{_id: [[1]], docs: [84, 85]},
{_id: [[["scalar"]]], docs: [52, 53]},
{_id: [[[1, 2], [{}], 2]], docs: [22, 24, 25, 48]},
{_id: [[[1, {}, 2]]], docs: [69, 73]},
{_id: [[[]]], docs: [64]},
{_id: [[], []], docs: [91]},
// Note "$a.b.c" does not descend into double (directly nested) arrays as in 42,45. Might
// have expected [[[]]]. Similarly in 56,57, it does not find the "c" values "hidden" within
// a directly-nested array.
{_id: [[]], docs: [39, 40, 41, 42, 43, 44, 45, 54, 55, 56, 57, 66, 70, 74, 75, 76]},
{_id: [[null]], docs: [71]},
{_id: [[{x: 1}]], docs: [68]},
{
_id: [],
docs: [
13, 14, 15, 16, 17, 18, 19, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
36, 37, 38, 46, 49, 58, 62, 77, 78, 79, 80, 81, 82, 83, 94, 95
]
},
{_id: [{x: 1}], docs: [60]},
{_id: null, docs: [0, 1, 2, 3, 4, 5, 6, 7, 9]},
{_id: {x: 1}, docs: [12]},
];
assert(resultsEq(expectedResults, coll.aggregate(pipeline).toArray()), () => {
print(`Results mismatch for $group query. Actual results: ${
tojson(coll.aggregate(pipeline).toArray())} Running resultsEq with verbose`);
resultsEq(expectedResults, coll.aggregate(pipeline).toArray(), true);
});
})();
})();
|