1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
|
// Tests that $group pushdown to SBE feature works in a sharded environment for some special
// scenarios.
//
// Notes:
// - In a sharded environment, the mongos splits a $group stage into two different stages. One is a
// merge $group stage at the mongos-side which does the global aggregation and the other is a $group
// stage at the shard-side which does the partial aggregation.
// - All aggregation features are tested by aggregation test suites under a sharded environment
// through passthrough tests. So, this test suite focuses on some special scenarios like for
// example, $group is pushed down to SBE at the shard-side and some accumulators may return the
// partial aggregation results in a special format to the mongos.
//
// Needs the following tag to be excluded from linux-64-duroff build variant because running
// wiredTiger without journaling in a replica set is not supported.
// @tags: [requires_sharding]
(function() {
'use strict';
load("jstests/libs/analyze_plan.js");
// As of now, $group pushdown to SBE feature is not enabled by default. So, enables it with a
// minimal configuration of a sharded cluster.
//
// TODO Remove {setParameter: "featureFlagSBEGroupPushdown=true"} when the feature is enabled by
// default.
const st = new ShardingTest(
{config: 1, shards: 1, shardOptions: {setParameter: "featureFlagSBEGroupPushdown=true"}});
// This database name can provide multiple similar test cases with a good separate namespace and
// each test case may create a separate collection for its own dataset.
const db = st.getDB(jsTestName());
const dbAtShard = st.shard0.getDB(jsTestName());
// Makes sure that $group pushdown to SBE feature is enabled.
assert(
assert.commandWorked(dbAtShard.adminCommand({getParameter: 1, featureFlagSBEGroupPushdown: 1}))
.featureFlagSBEGroupPushdown.value);
// Makes sure that the test db is sharded and the data is stored into the only shard.
assert.commandWorked(st.s0.adminCommand({enableSharding: db.getName()}));
st.ensurePrimaryShard(db.getName(), st.shard0.shardName);
let assertShardedGroupResultsMatch = (coll, pipeline) => {
// Turns to the classic engine at the shard before figuring out its result.
assert.commandWorked(
dbAtShard.adminCommand({setParameter: 1, internalQueryForceClassicEngine: true}));
// Collects the classic engine's result as the expected result, executing the pipeline at the
// mongos.
const classicalRes =
coll.runCommand({aggregate: coll.getName(), pipeline: pipeline, cursor: {}})
.cursor.firstBatch;
// Turns to the SBE engine at the shard.
assert.commandWorked(
dbAtShard.adminCommand({setParameter: 1, internalQueryForceClassicEngine: false}));
// Verifies that the SBE engine's results are same as the expected results, executing the
// pipeline at the mongos.
const sbeRes = coll.runCommand({aggregate: coll.getName(), pipeline: pipeline, cursor: {}})
.cursor.firstBatch;
assert.sameMembers(sbeRes, classicalRes);
};
let prepareCollection = coll => {
coll.drop();
// Makes sure that the collection is sharded.
assert.commandWorked(st.s0.adminCommand({shardCollection: coll.getFullName(), key: {_id: 1}}));
return coll;
};
// A test case for a sharded $sum
let coll = prepareCollection(db.partial_sum);
// Prepares data for the 'NumberLong' sum result to overflow, when the shard sends back the partial
// sum as a doc with 'subTotal' and 'subTotalError' fields to the mongos. All data go to the only
// shard and so overflow will happen.
assert.commandWorked(
coll.insert([{a: 1, b: NumberLong("9223372036854775807")}, {a: 2, b: NumberLong("10")}]));
assertShardedGroupResultsMatch(coll, [{$group: {_id: "$a", s: {$sum: "$b"}}}]);
assertShardedGroupResultsMatch(coll, [{$group: {_id: null, s: {$sum: "$b"}}}]);
// Test cases for a sharded $stdDevPop and $stdDevSamp
coll = prepareCollection(db.partial_std_dev);
assert.commandWorked(coll.insert([
{"item": "a", "price": 10},
{"item": "b", "price": 20},
{"item": "a", "price": 5},
{"item": "b", "price": 10},
{"item": "c", "price": 5},
]));
assertShardedGroupResultsMatch(coll, [{$group: {_id: "$item", sd: {$stdDevSamp: "$price"}}}]);
assertShardedGroupResultsMatch(coll, [{$group: {_id: "$item", sd: {$stdDevSamp: "$missing"}}}]);
assertShardedGroupResultsMatch(coll, [{$group: {_id: "$item", sd: {$stdDevPop: "$price"}}}]);
assertShardedGroupResultsMatch(coll, [{$group: {_id: "$item", sd: {$stdDevPop: "$missing"}}}]);
// A test case for a sharded $avg
coll = prepareCollection(db.partial_avg);
// Prepares dataset so that each group has different numeric data types for price field which
// will excercise different code paths in generated SBE plan stages.
// Prices for group "a" are all decimals.
assert.commandWorked(coll.insert(
[{item: "a", price: NumberDecimal("10.7")}, {item: "a", price: NumberDecimal("20.3")}]));
// Prices for group "b" are one decimal and one non-decimal.
assert.commandWorked(
coll.insert([{item: "b", price: NumberDecimal("3.7")}, {item: "b", price: 2.3}]));
// Prices for group "c" are all non-decimals.
assert.commandWorked(coll.insert([{item: "c", price: 3}, {item: "b", price: 1}]));
// Verifies that SBE group pushdown with $avg works in a sharded environment.
assertShardedGroupResultsMatch(coll, [{$group: {_id: "$item", a: {$avg: "$price"}}}]);
// Verifies that SBE group pushdown with sharded $avg works for missing data.
assertShardedGroupResultsMatch(coll, [{$group: {_id: "$item", a: {$avg: "$missing"}}}]);
st.stop();
}());
|
