1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
|
/**
* Test that sort queries work properly on dates ouside the 32 bit epoch range,
* [1970-01-01 00:00:00 UTC - 2038-01-29 03:13:07 UTC], when a collection scan is used.
*
* @tags: [
* # Explain of a resolved view must be executed by mongos.
* directly_against_shardsvrs_incompatible,
* # This complicates aggregation extraction.
* do_not_wrap_aggregations_in_facets,
* # Refusing to run a test that issues an aggregation command with explain because it may
* # return incomplete results if interrupted by a stepdown.
* does_not_support_stepdowns,
* # We need a timeseries collection.
* requires_timeseries,
* # Patched in 6.2
* requires_fcv_62
* ]
*/
(function() {
"use strict";
load("jstests/aggregation/extras/utils.js"); // For getExplainedPipelineFromAggregation.
load("jstests/core/timeseries/libs/timeseries.js");
load('jstests/libs/analyze_plan.js');
if (!TimeseriesTest.bucketUnpackWithSortEnabled(db.getMongo())) {
jsTestLog("Skipping test because 'BucketUnpackWithSort' is disabled.");
return;
}
const timeFieldName = "t";
// Create unindexed collection
const coll = db.timeseries_internal_bounded_sort;
const buckets = db['system.buckets.' + coll.getName()];
coll.drop();
assert.commandWorked(db.createCollection(coll.getName(), {timeseries: {timeField: 't'}}));
// Create collection indexed on time
const collIndexed = db.timeseries_internal_bounded_sort_with_index;
const bucketsIndexed = db['system.buckets.' + collIndexed.getName()];
collIndexed.drop();
assert.commandWorked(db.createCollection(collIndexed.getName(), {timeseries: {timeField: 't'}}));
assert.commandWorked(collIndexed.createIndex({'t': 1}));
jsTestLog(collIndexed.getIndexes());
jsTestLog(bucketsIndexed.getIndexes());
const intervalMillis = 60000;
function insertBucket(start) {
jsTestLog("Inserting bucket starting with " + Date(start).toString());
const batchSize = 1000;
const batch =
Array.from({length: batchSize}, (_, j) => ({t: new Date(start + j * intervalMillis)}));
assert.commandWorked(coll.insert(batch));
assert.commandWorked(collIndexed.insert(batch));
}
// Insert some data. We'll insert 5 buckets in each range, with values < 0,
// values between 0 and FFFFFFFF(unsigned), and values > FFFFFFFF. It turns out, however,
// that Javascript's Date doesn't handle dates beyond 2039 either, so we rely on lower dates
// to test for unexpected behavior.
function insertDocuments() {
// We want to choose the underflow and overflow lower bits in such a way that we
// encourage wrong results when the upper bytes are removed.
const underflowMin = new Date("1969-01-01").getTime(); // Day before the 32 bit epoch
const normalMin = new Date("2002-01-01").getTime(); // Middle of the 32 bit epoch
insertBucket(underflowMin);
var numBatches = 5;
const batchOffset = Math.floor(intervalMillis / (numBatches + 1));
for (let i = 0; i < numBatches; ++i) {
const start = normalMin + i * batchOffset;
insertBucket(start);
}
assert.gt(buckets.aggregate([{$count: 'n'}]).next().n, 1, 'Expected more than one bucket');
}
insertDocuments();
const unpackStage = getAggPlanStages(coll.explain().aggregate(), '$_internalUnpackBucket')[0];
function assertSorted(result, ascending) {
let prev = ascending ? {t: -Infinity} : {t: Infinity};
for (const doc of result) {
if (ascending) {
assert.lt(+prev.t,
+doc.t,
'Found two docs not in ascending time order: ' + tojson({prev, doc}));
} else {
assert.gt(+prev.t,
+doc.t,
'Found two docs not in descending time order: ' + tojson({prev, doc}));
}
prev = doc;
}
}
function checkAgainstReferenceBoundedSortUnexpected(
collection, reference, pipeline, hint, sortOrder) {
const options = hint ? {hint: hint} : {};
const opt = collection.aggregate(pipeline, options).toArray();
assertSorted(opt, sortOrder);
assert.eq(reference.length, opt.length);
for (var i = 0; i < opt.length; ++i) {
assert.docEq(reference[i], opt[i]);
}
const plan = collection.explain({}).aggregate(pipeline, options);
const stages = getAggPlanStages(plan, "$_internalBoundedSort");
assert.eq([], stages, plan);
}
function checkAgainstReferenceBoundedSortExpected(
collection, reference, pipeline, hint, sortOrder) {
const options = hint ? {hint: hint} : {};
const opt = collection.aggregate(pipeline, options).toArray();
assertSorted(opt, sortOrder);
assert.eq(reference.length, opt.length);
for (var i = 0; i < opt.length; ++i) {
assert.docEq(reference[i], opt[i]);
}
const plan = collection.explain({}).aggregate(pipeline, options);
const stages = getAggPlanStages(plan, "$_internalBoundedSort");
assert.neq([], stages, plan);
}
function runTest(ascending) {
const reference = buckets
.aggregate([
unpackStage,
{$_internalInhibitOptimization: {}},
{$sort: {t: ascending ? 1 : -1}},
])
.toArray();
assertSorted(reference, ascending);
// Check plan using collection scan
checkAgainstReferenceBoundedSortUnexpected(coll,
reference,
[
{$sort: {t: ascending ? 1 : -1}},
],
{},
ascending);
// Check plan using hinted collection scan
checkAgainstReferenceBoundedSortUnexpected(coll,
reference,
[
{$sort: {t: ascending ? 1 : -1}},
],
{"$natural": ascending ? 1 : -1},
ascending);
const referenceIndexed = bucketsIndexed
.aggregate([
unpackStage,
{$_internalInhibitOptimization: {}},
{$sort: {t: ascending ? 1 : -1}},
])
.toArray();
assertSorted(referenceIndexed, ascending);
// Check plan using index scan. If we've inserted a date before 1-1-1970, we round the min up
// towards 1970, rather then down, which has the effect of increasing the control.min.t. This
// means the minimum time in the bucket is likely to be lower than indicated and thus, actual
// dates may be out of order relative to what's indicated by the bucket bounds.
checkAgainstReferenceBoundedSortUnexpected(collIndexed,
referenceIndexed,
[
{$sort: {t: ascending ? 1 : -1}},
],
{},
ascending);
// Check plan using hinted index scan
checkAgainstReferenceBoundedSortUnexpected(collIndexed,
referenceIndexed,
[
{$sort: {t: ascending ? 1 : -1}},
],
{"t": 1},
ascending);
// Check plan using hinted collection scan
checkAgainstReferenceBoundedSortUnexpected(collIndexed,
referenceIndexed,
[
{$sort: {t: ascending ? 1 : -1}},
],
{"$natural": ascending ? 1 : -1},
ascending);
// The workaround in all cases is to create a reverse index on the time field, though
// it's necessary to force use of the reversed index.
const reverseIdxName = "reverseIdx";
collIndexed.createIndex({t: -1}, {name: reverseIdxName});
checkAgainstReferenceBoundedSortExpected(collIndexed,
referenceIndexed,
[
{$sort: {t: ascending ? 1 : -1}},
],
{"t": -1},
ascending);
collIndexed.dropIndex(reverseIdxName);
}
runTest(false); // descending
runTest(true); // ascending
})();
|
