/**
* Copyright (C) 2017 MongoDB Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License, version 3,
* as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see .
*
* As a special exception, the copyright holders give permission to link the
* code of portions of this program with the OpenSSL library under certain
* conditions as described in each individual source file and distribute
* linked combinations including the program with the OpenSSL library. You
* must comply with the GNU Affero General Public License in all respects for
* all of the code used other than as permitted herein. If you modify file(s)
* with this exception, you may extend this exception to your version of the
* file(s), but you are not obligated to do so. If you do not wish to do so,
* delete this exception statement from your version. If you delete this
* exception statement from all source files in the program, then also delete
* it in the license file.
*/
#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kSharding
#include "mongo/platform/basic.h"
#include "mongo/db/json.h"
#include "mongo/db/matcher/extensions_callback_noop.h"
#include "mongo/db/namespace_string.h"
#include "mongo/db/query/canonical_query.h"
#include "mongo/s/chunk_manager.h"
#include "mongo/s/shard_key_pattern.h"
#include "mongo/s/sharding_test_fixture.h"
#include "mongo/unittest/unittest.h"
#include "mongo/util/log.h"
namespace mongo {
namespace {
const double INF = std::numeric_limits::infinity();
/**
* Test the chunk manager index bounds for query functionality.
*/
class CMCollapseTreeTest : public ShardingTestFixture {
protected:
// Utility function to create a CanonicalQuery
std::unique_ptr canonicalize(const char* queryStr) {
BSONObj queryObj = fromjson(queryStr);
const NamespaceString nss("test.foo");
auto qr = stdx::make_unique(nss);
qr->setFilter(queryObj);
auto statusWithCQ = CanonicalQuery::canonicalize(
operationContext(), std::move(qr), ExtensionsCallbackNoop());
ASSERT_OK(statusWithCQ.getStatus());
return std::move(statusWithCQ.getValue());
}
void checkIndexBoundsWithKey(const char* keyStr,
const char* queryStr,
const IndexBounds& expectedBounds) {
auto query(canonicalize(queryStr));
ASSERT(query.get() != NULL);
BSONObj key = fromjson(keyStr);
IndexBounds indexBounds = ChunkManager::getIndexBoundsForQuery(key, *query.get());
ASSERT_EQUALS(indexBounds.size(), expectedBounds.size());
for (size_t i = 0; i < indexBounds.size(); i++) {
const OrderedIntervalList& oil = indexBounds.fields[i];
const OrderedIntervalList& expectedOil = expectedBounds.fields[i];
ASSERT_EQUALS(oil.intervals.size(), expectedOil.intervals.size());
for (size_t i = 0; i < oil.intervals.size(); i++) {
if (Interval::INTERVAL_EQUALS !=
oil.intervals[i].compare(expectedOil.intervals[i])) {
log() << oil.intervals[i] << " != " << expectedOil.intervals[i];
}
ASSERT_EQUALS(Interval::INTERVAL_EQUALS,
oil.intervals[i].compare(expectedOil.intervals[i]));
}
}
}
// Assume shard key is { a: 1 }
void checkIndexBounds(const char* queryStr, const OrderedIntervalList& expectedOil) {
auto query(canonicalize(queryStr));
ASSERT(query.get() != NULL);
BSONObj key = fromjson("{a: 1}");
IndexBounds indexBounds = ChunkManager::getIndexBoundsForQuery(key, *query.get());
ASSERT_EQUALS(indexBounds.size(), 1U);
const OrderedIntervalList& oil = indexBounds.fields.front();
if (oil.intervals.size() != expectedOil.intervals.size()) {
for (size_t i = 0; i < oil.intervals.size(); i++) {
log() << oil.intervals[i];
}
}
ASSERT_EQUALS(oil.intervals.size(), expectedOil.intervals.size());
for (size_t i = 0; i < oil.intervals.size(); i++) {
ASSERT_EQUALS(Interval::INTERVAL_EQUALS,
oil.intervals[i].compare(expectedOil.intervals[i]));
}
}
};
// { a: 2 } -> a: [2, 2]
TEST_F(CMCollapseTreeTest, Basic) {
OrderedIntervalList expected;
expected.intervals.push_back(Interval(BSON("" << 2 << "" << 2), true, true));
checkIndexBounds("{a: 2}", expected);
}
// { b: 2 } -> a: [MinKey, MaxKey]
TEST_F(CMCollapseTreeTest, AllValue) {
OrderedIntervalList expected;
BSONObjBuilder builder;
builder.appendMinKey("");
builder.appendMaxKey("");
expected.intervals.push_back(Interval(builder.obj(), true, true));
checkIndexBounds("{b: 2}", expected);
}
// { 'a' : { '$not' : { '$gt' : 1 } } } -> a: [MinKey, 1.0], (inf.0, MaxKey]
TEST_F(CMCollapseTreeTest, NegativeGT) {
OrderedIntervalList expected;
{
BSONObjBuilder builder;
builder.appendMinKey("");
builder.appendNumber("", 1.0);
expected.intervals.push_back(Interval(builder.obj(), true, true));
}
{
BSONObjBuilder builder;
builder.append("", std::numeric_limits::infinity());
builder.appendMaxKey("");
expected.intervals.push_back(Interval(builder.obj(), false, true));
}
checkIndexBounds("{ 'a' : { '$not' : { '$gt' : 1 } } }", expected);
}
// {$or: [{a: 20}, {$and: [{a:1}, {b:7}]}]} -> a: [1.0, 1.0], [20.0, 20.0]
TEST_F(CMCollapseTreeTest, OrWithAndChild) {
OrderedIntervalList expected;
expected.intervals.push_back(Interval(BSON("" << 1.0 << "" << 1.0), true, true));
expected.intervals.push_back(Interval(BSON("" << 20.0 << "" << 20.0), true, true));
checkIndexBounds("{$or: [{a: 20}, {$and: [{a:1}, {b:7}]}]}", expected);
}
// {a:20, $or: [{b:1}, {c:7}]} -> a: [20.0, 20.0]
TEST_F(CMCollapseTreeTest, AndWithUnindexedOrChild) {
// Logic rewrite could give a tree with root OR.
OrderedIntervalList expected;
expected.intervals.push_back(Interval(BSON("" << 20.0 << "" << 20.0), true, true));
checkIndexBounds("{a:20, $or: [{b:1}, {c:7}]}", expected);
}
// {$or: [{a:{$gt:2,$lt:10}}, {a:{$gt:0,$lt:5}}]} -> a: (0.0, 10.0)
TEST_F(CMCollapseTreeTest, OrOfAnd) {
OrderedIntervalList expected;
expected.intervals.push_back(Interval(BSON("" << 0.0 << "" << 10.0), false, false));
checkIndexBounds("{$or: [{a:{$gt:2,$lt:10}}, {a:{$gt:0,$lt:5}}]}", expected);
}
// {$or: [{a:{$gt:2,$lt:10}}, {a:{$gt:0,$lt:15}}, {a:{$gt:20}}]}
// -> a: (0.0, 15.0), (20.0, inf.0]
TEST_F(CMCollapseTreeTest, OrOfAnd2) {
OrderedIntervalList expected;
expected.intervals.push_back(Interval(BSON("" << 0.0 << "" << 15.0), false, false));
expected.intervals.push_back(Interval(BSON("" << 20.0 << "" << INF), false, true));
checkIndexBounds("{$or: [{a:{$gt:2,$lt:10}}, {a:{$gt:0,$lt:15}}, {a:{$gt:20}}]}", expected);
}
// "{$or: [{a:{$gt:1,$lt:5},b:6}, {a:3,b:{$gt:0,$lt:10}}]}" -> a: (1.0, 5.0)
TEST_F(CMCollapseTreeTest, OrOfAnd3) {
OrderedIntervalList expected;
expected.intervals.push_back(Interval(BSON("" << 1.0 << "" << 5.0), false, false));
checkIndexBounds("{$or: [{a:{$gt:1,$lt:5},b:6}, {a:3,b:{$gt:0,$lt:10}}]}", expected);
}
//
// Compound shard key
//
// "{$or: [{a:{$gt:1,$lt:5}, b:{$gt:0,$lt:3}, c:6}, "
// "{a:3, b:{$gt:1,$lt:2}, c:{$gt:0,$lt:10}}]}",
// -> a: (1.0, 5.0), b: (0.0, 3.0)
TEST_F(CMCollapseTreeTest, OrOfAnd4) {
IndexBounds expectedBounds;
expectedBounds.fields.push_back(OrderedIntervalList());
expectedBounds.fields.push_back(OrderedIntervalList());
expectedBounds.fields[0].intervals.push_back(
Interval(BSON("" << 1.0 << "" << 5.0), false, false));
expectedBounds.fields[1].intervals.push_back(
Interval(BSON("" << 0.0 << "" << 3.0), false, false));
checkIndexBoundsWithKey("{a: 1, b: 1}", // shard key
"{$or: [{a:{$gt:1,$lt:5}, b:{$gt:0,$lt:3}, c:6}, "
"{a:3, b:{$gt:1,$lt:2}, c:{$gt:0,$lt:10}}]}",
expectedBounds);
}
// "{$or: [{a:{$gt:1,$lt:5}, c:6}, "
// "{a:3, b:{$gt:1,$lt:2}, c:{$gt:0,$lt:10}}]}"));
// ->
TEST_F(CMCollapseTreeTest, OrOfAnd5) {
IndexBounds expectedBounds;
expectedBounds.fields.push_back(OrderedIntervalList());
expectedBounds.fields.push_back(OrderedIntervalList());
expectedBounds.fields[0].intervals.push_back(
Interval(BSON("" << 1.0 << "" << 5.0), false, false));
BSONObjBuilder builder;
builder.appendMinKey("");
builder.appendMaxKey("");
expectedBounds.fields[1].intervals.push_back(Interval(builder.obj(), true, true));
checkIndexBoundsWithKey("{a: 1, b: 1}", // shard key
"{$or: [{a:{$gt:1,$lt:5}, c:6}, "
"{a:3, b:{$gt:1,$lt:2}, c:{$gt:0,$lt:10}}]}",
expectedBounds);
}
// {$or: [{a:{$in:[1]},b:{$in:[1]}}, {a:{$in:[1,5]},b:{$in:[1,5]}}]}
// -> a: [1], [5]; b: [1], [5]
TEST_F(CMCollapseTreeTest, OrOfAnd6) {
IndexBounds expectedBounds;
expectedBounds.fields.push_back(OrderedIntervalList());
expectedBounds.fields.push_back(OrderedIntervalList());
// a: [1], [5]
expectedBounds.fields[0].intervals.push_back(
Interval(BSON("" << 1.0 << "" << 1.0), true, true));
expectedBounds.fields[0].intervals.push_back(
Interval(BSON("" << 5.0 << "" << 5.0), true, true));
// b: [1], [5]
expectedBounds.fields[1].intervals.push_back(
Interval(BSON("" << 1.0 << "" << 1.0), true, true));
expectedBounds.fields[1].intervals.push_back(
Interval(BSON("" << 5.0 << "" << 5.0), true, true));
checkIndexBoundsWithKey("{a: 1, b: 1}", // shard key
"{$or: [{a:{$in:[1]},b:{$in:[1]}}, {a:{$in:[1,5]},b:{$in:[1,5]}}]}",
expectedBounds);
}
//
// Array operators
//
// {a : {$elemMatch: {b:1}}} -> a.b: [MinKey, MaxKey]
// Shard key doesn't allow multikey, but query on array should succeed without error.
TEST_F(CMCollapseTreeTest, ElemMatchOneField) {
IndexBounds expectedBounds;
expectedBounds.fields.push_back(OrderedIntervalList());
OrderedIntervalList& oil = expectedBounds.fields.front();
oil.intervals.push_back(Interval(BSON("" << 1 << "" << 1), true, true));
checkIndexBoundsWithKey("{'a.b': 1}", "{a : {$elemMatch: {b:1}}}", expectedBounds);
}
// {foo: {$all: [ {$elemMatch: {a:1, b:1}}, {$elemMatch: {a:2, b:2}}]}}
// -> foo.a: [1, 1]
// Or -> foo.a: [2, 2]
TEST_F(CMCollapseTreeTest, BasicAllElemMatch) {
Interval expectedInterval(BSON("" << 1 << "" << 1), true, true);
const char* queryStr = "{foo: {$all: [ {$elemMatch: {a:1, b:1}} ]}}";
auto query(canonicalize(queryStr));
ASSERT(query.get() != NULL);
BSONObj key = fromjson("{'foo.a': 1}");
IndexBounds indexBounds = ChunkManager::getIndexBoundsForQuery(key, *query.get());
ASSERT_EQUALS(indexBounds.size(), 1U);
const OrderedIntervalList& oil = indexBounds.fields.front();
ASSERT_EQUALS(oil.intervals.size(), 1U);
const Interval& interval = oil.intervals.front();
// Choose one of the two possible solutions.
// Two solutions differ only by assignment of index tags.
ASSERT(Interval::INTERVAL_EQUALS == interval.compare(expectedInterval));
}
// {a : [1, 2, 3]} -> a: [1, 1], [[1, 2, 3], [1, 2, 3]]
TEST_F(CMCollapseTreeTest, ArrayEquality) {
OrderedIntervalList expected;
expected.intervals.push_back(Interval(BSON("" << 1 << "" << 1), true, true));
BSONArray array(BSON_ARRAY(1 << 2 << 3));
Interval interval(BSON("" << array << "" << array), true, true);
expected.intervals.push_back(interval);
checkIndexBounds("{a : [1, 2, 3]}", expected);
}
//
// Features: Regex, $where, $text, hashed key
//
// { a: /abc/ } -> a: ["", {}), [/abc/, /abc/]
TEST_F(CMCollapseTreeTest, Regex) {
OrderedIntervalList expected;
expected.intervals.push_back(Interval(BSON(""
<< ""
<< ""
<< BSONObj()),
true,
false));
BSONObjBuilder builder;
builder.appendRegex("", "abc");
builder.appendRegex("", "abc");
expected.intervals.push_back(Interval(builder.obj(), true, true));
checkIndexBounds("{ a: /abc/ }", expected);
}
// {$where: 'this.credits == this.debits' }
TEST_F(CMCollapseTreeTest, Where) {
OrderedIntervalList expected;
BSONObjBuilder builder;
builder.appendMinKey("");
builder.appendMaxKey("");
expected.intervals.push_back(Interval(builder.obj(), true, true));
checkIndexBounds("{$where: 'this.credits == this.debits' }", expected);
}
// { $text: { $search: "coffee -cake" } }
TEST_F(CMCollapseTreeTest, Text) {
OrderedIntervalList expected;
BSONObjBuilder builder;
builder.appendMinKey("");
builder.appendMaxKey("");
expected.intervals.push_back(Interval(builder.obj(), true, true));
checkIndexBounds("{ $text: { $search: 'coffee -cake' } }", expected);
}
// { a: 2, $text: { $search: "leche", $language: "es" } }
TEST_F(CMCollapseTreeTest, TextWithQuery) {
OrderedIntervalList expected;
BSONObjBuilder builder;
builder.appendMinKey("");
builder.appendMaxKey("");
expected.intervals.push_back(Interval(builder.obj(), true, true));
checkIndexBounds("{ a: 2, $text: { $search: 'leche', $language: 'es' } }", expected);
}
// { a: 0 } -> hashed a: [hash(0), hash(0)]
TEST_F(CMCollapseTreeTest, HashedSinglePoint) {
const char* queryStr = "{ a: 0 }";
auto query(canonicalize(queryStr));
ASSERT(query.get() != NULL);
BSONObj key = fromjson("{a: 'hashed'}");
IndexBounds indexBounds = ChunkManager::getIndexBoundsForQuery(key, *query.get());
ASSERT_EQUALS(indexBounds.size(), 1U);
const OrderedIntervalList& oil = indexBounds.fields.front();
ASSERT_EQUALS(oil.intervals.size(), 1U);
const Interval& interval = oil.intervals.front();
ASSERT(interval.isPoint());
}
// { a: { $lt: 2, $gt: 1} } -> hashed a: [Minkey, Maxkey]
TEST_F(CMCollapseTreeTest, HashedRange) {
IndexBounds expectedBounds;
expectedBounds.fields.push_back(OrderedIntervalList());
OrderedIntervalList& expectedOil = expectedBounds.fields.front();
BSONObjBuilder builder;
builder.appendMinKey("");
builder.appendMaxKey("");
expectedOil.intervals.push_back(Interval(builder.obj(), true, true));
checkIndexBoundsWithKey("{a: 'hashed'}", "{ a: { $lt: 2, $gt: 1} }", expectedBounds);
}
// { a: /abc/ } -> hashed a: [Minkey, Maxkey]
TEST_F(CMCollapseTreeTest, HashedRegex) {
IndexBounds expectedBounds;
expectedBounds.fields.push_back(OrderedIntervalList());
OrderedIntervalList& expectedOil = expectedBounds.fields.front();
BSONObjBuilder builder;
builder.appendMinKey("");
builder.appendMaxKey("");
expectedOil.intervals.push_back(Interval(builder.obj(), true, true));
checkIndexBoundsWithKey("{a: 'hashed'}", "{ a: /abc/ }", expectedBounds);
}
/**
* Tests the KeyPattern key bounds generation logic.
*/
class CMKeyBoundsTest : public mongo::unittest::Test {
protected:
void checkBoundList(const BoundList& list, const BoundList& expected) {
ASSERT_EQUALS(list.size(), expected.size());
for (size_t i = 0; i < list.size(); i++) {
ASSERT_EQUALS(list[i].first.woCompare(expected[i].first), 0);
ASSERT_EQUALS(list[i].second.woCompare(expected[i].second), 0);
}
}
};
// Key { a: 1 }, Bounds a: [0]
// => { a: 0 } -> { a: 0 }
TEST_F(CMKeyBoundsTest, Basic) {
IndexBounds indexBounds;
indexBounds.fields.push_back(OrderedIntervalList());
indexBounds.fields.front().intervals.push_back(Interval(BSON("" << 0 << "" << 0), true, true));
BoundList expectedList;
expectedList.emplace_back(fromjson("{a: 0}"), fromjson("{a: 0}"));
ShardKeyPattern skeyPattern(fromjson("{a: 1}"));
BoundList list = skeyPattern.flattenBounds(indexBounds);
checkBoundList(list, expectedList);
}
// Key { a: 1 }, Bounds a: [2, 3)
// => { a: 2 } -> { a: 3 } // bound inclusion is ignored.
TEST_F(CMKeyBoundsTest, SingleInterval) {
IndexBounds indexBounds;
indexBounds.fields.push_back(OrderedIntervalList());
indexBounds.fields.front().intervals.push_back(Interval(BSON("" << 2 << "" << 3), true, false));
BoundList expectedList;
expectedList.emplace_back(fromjson("{a: 2}"), fromjson("{a: 3}"));
ShardKeyPattern skeyPattern(fromjson("{a: 1}"));
BoundList list = skeyPattern.flattenBounds(indexBounds);
checkBoundList(list, expectedList);
}
// Key { a: 1, b: 1, c: 1 }, Bounds a: [2, 3), b: [2, 3), c: [2: 3)
// => { a: 2, b: 2, c: 2 } -> { a: 3, b: 3, c: 3 }
TEST_F(CMKeyBoundsTest, MultiIntervals) {
IndexBounds indexBounds;
indexBounds.fields.push_back(OrderedIntervalList());
indexBounds.fields.push_back(OrderedIntervalList());
indexBounds.fields.push_back(OrderedIntervalList());
indexBounds.fields[0].intervals.push_back(Interval(BSON("" << 2 << "" << 3), true, false));
indexBounds.fields[1].intervals.push_back(Interval(BSON("" << 2 << "" << 3), true, false));
indexBounds.fields[2].intervals.push_back(Interval(BSON("" << 2 << "" << 3), true, false));
BoundList expectedList;
expectedList.emplace_back(fromjson("{ a: 2, b: 2, c: 2 }"), fromjson("{ a: 3, b: 3, c: 3 }"));
ShardKeyPattern skeyPattern(fromjson("{a: 1, b: 1, c: 1}"));
BoundList list = skeyPattern.flattenBounds(indexBounds);
checkBoundList(list, expectedList);
}
// Key { a: 1, b: 1, c: 1 }, Bounds a: [0, 0], b: { $in: [4, 5, 6] }, c: [2: 3)
// => { a: 0, b: 4, c: 2 } -> { a: 0, b: 4, c: 3 }
// { a: 0, b: 5, c: 2 } -> { a: 0, b: 5, c: 3 }
// { a: 0, b: 6, c: 2 } -> { a: 0, b: 6, c: 3 }
TEST_F(CMKeyBoundsTest, IntervalExpansion) {
IndexBounds indexBounds;
indexBounds.fields.push_back(OrderedIntervalList());
indexBounds.fields.push_back(OrderedIntervalList());
indexBounds.fields.push_back(OrderedIntervalList());
indexBounds.fields[0].intervals.push_back(Interval(BSON("" << 0 << "" << 0), true, true));
indexBounds.fields[1].intervals.push_back(Interval(BSON("" << 4 << "" << 4), true, true));
indexBounds.fields[1].intervals.push_back(Interval(BSON("" << 5 << "" << 5), true, true));
indexBounds.fields[1].intervals.push_back(Interval(BSON("" << 6 << "" << 6), true, true));
indexBounds.fields[2].intervals.push_back(Interval(BSON("" << 2 << "" << 3), true, false));
BoundList expectedList;
expectedList.emplace_back(fromjson("{ a: 0, b: 4, c: 2 }"), fromjson("{ a: 0, b: 4, c: 3 }"));
expectedList.emplace_back(fromjson("{ a: 0, b: 5, c: 2 }"), fromjson("{ a: 0, b: 5, c: 3 }"));
expectedList.emplace_back(fromjson("{ a: 0, b: 6, c: 2 }"), fromjson("{ a: 0, b: 6, c: 3 }"));
ShardKeyPattern skeyPattern(fromjson("{a: 1, b: 1, c: 1}"));
BoundList list = skeyPattern.flattenBounds(indexBounds);
checkBoundList(list, expectedList);
}
// Key { a: 1, b: 1, c: 1 }, Bounds a: [0, 1], b: { $in: [4, 5, 6] }, c: [2: 3)
// => { a: 0, b: 4, c: 2 } -> { a: 1, b: 6, c: 3 }
// Since field "a" is not a point, expasion after "a" is not allowed.
TEST_F(CMKeyBoundsTest, NonPointIntervalExpasion) {
IndexBounds indexBounds;
indexBounds.fields.push_back(OrderedIntervalList());
indexBounds.fields.push_back(OrderedIntervalList());
indexBounds.fields.push_back(OrderedIntervalList());
indexBounds.fields[0].intervals.push_back(Interval(BSON("" << 0 << "" << 1), true, true));
indexBounds.fields[1].intervals.push_back(Interval(BSON("" << 4 << "" << 4), true, true));
indexBounds.fields[1].intervals.push_back(Interval(BSON("" << 5 << "" << 5), true, true));
indexBounds.fields[1].intervals.push_back(Interval(BSON("" << 6 << "" << 6), true, true));
indexBounds.fields[2].intervals.push_back(Interval(BSON("" << 2 << "" << 3), true, false));
BoundList expectedList;
expectedList.emplace_back(fromjson("{ a: 0, b: 4, c: 2 }"), fromjson("{ a: 1, b: 6, c: 3 }"));
ShardKeyPattern skeyPattern(fromjson("{a: 1, b: 1, c: 1}"));
BoundList list = skeyPattern.flattenBounds(indexBounds);
checkBoundList(list, expectedList);
}
} // namespace
} // namespace mongo