SERVER-27644 $unwind arrays when performing distinct on a view

(cherry picked from commit 3ed956d8813b7b56c2319456c28838dfa6c5c20b)

Conflicts:
    src/mongo/db/query/parsed_distinct_test.cpp

3 files changed, 75 insertions, 11 deletions

diff --git a/jstests/views/views_distinct.js b/jstests/views/views_distinct.js
index 9e99b2552e7..b53a8e9e871 100644
--- a/jstests/views/views_distinct.js
+++ b/jstests/views/views_distinct.js
@@ -31,26 +31,33 @@
     let identityView = viewsDB.getCollection("identityView");
     let largePopView = viewsDB.getCollection("largePopView");
 
+    function assertIdentityViewDistinctMatchesCollection(key, query) {
+        query = (query === undefined) ? {} : query;
+        const collDistinct = coll.distinct(key, query);
+        const viewDistinct = identityView.distinct(key, query);
+        assert(arrayEq(collDistinct, viewDistinct),
+               "Distinct on a collection did not match distinct on its identity view; got " +
+                   tojson(viewDistinct) + " but expected " + tojson(collDistinct));
+    }
+
     // Test basic distinct requests on known fields without a query.
-    assert(arrayEq(coll.distinct("pop"), identityView.distinct("pop")));
-    assert(arrayEq(coll.distinct("_id"), identityView.distinct("_id")));
+    assertIdentityViewDistinctMatchesCollection("pop");
+    assertIdentityViewDistinctMatchesCollection("_id");
     assert(arrayEq([7, 10], largePopView.distinct("pop")));
     assert(arrayEq(["New York", "Palo Alto"], largePopView.distinct("_id")));
 
     // Test distinct with the presence of a query.
-    assert(arrayEq(coll.distinct("state", {}), identityView.distinct("state", {})));
-    assert(arrayEq(coll.distinct("pop", {pop: {$exists: true}}),
-                   identityView.distinct("pop", {pop: {$exists: true}})));
-    assert(
-        arrayEq(coll.distinct("_id", {state: "CA"}), identityView.distinct("_id", {state: "CA"})));
+    assertIdentityViewDistinctMatchesCollection("state", {});
+    assertIdentityViewDistinctMatchesCollection("pop", {pop: {$exists: true}});
+    assertIdentityViewDistinctMatchesCollection("state", {pop: {$gt: 3}});
+    assertIdentityViewDistinctMatchesCollection("_id", {state: "CA"});
     assert(arrayEq(["CA"], largePopView.distinct("state", {pop: {$gte: 8}})));
     assert(arrayEq([7], largePopView.distinct("pop", {state: "NY"})));
 
     // Test distinct where we expect an empty set response.
-    assert.eq(coll.distinct("nonexistent"), identityView.distinct("nonexistent"));
+    assertIdentityViewDistinctMatchesCollection("nonexistent");
+    assertIdentityViewDistinctMatchesCollection("pop", {pop: {$gt: 1000}});
     assert.eq([], largePopView.distinct("nonexistent"));
-    assert.eq(coll.distinct("pop", {pop: {$gt: 1000}}),
-              identityView.distinct("pop", {pop: {$gt: 1000}}));
     assert.eq([], largePopView.distinct("_id", {state: "FL"}));
 
     // Explain works with distinct.
@@ -61,7 +68,30 @@
     assert.eq(explainPlan["stages"][0]["$cursor"]["queryPlanner"]["namespace"],
               "views_distinct.coll");
 
+    // Distinct commands fail when they try to change the collation of a view.
     assert.commandFailedWithCode(
         viewsDB.runCommand({distinct: "identityView", key: "state", collation: {locale: "en_US"}}),
         ErrorCodes.OptionNotSupportedOnView);
+
+    // Test distinct on nested objects, nested arrays and nullish values.
+    coll.drop();
+    allDocuments = [];
+    allDocuments.push({a: 1, b: [2, 3, [4, 5], {c: 6}], d: {e: [1, 2]}});
+    allDocuments.push({a: [1], b: [2, 3, 4, [5]], c: 6, d: {e: 1}});
+    allDocuments.push({a: [1, 2], b: 3, c: [6], d: [{e: 1}, {e: [1, 2]}]});
+    allDocuments.push({a: [1, 2], b: [4, 5], c: [undefined], d: [1]});
+    allDocuments.push({a: null, b: [4, 5, null, undefined], c: [], d: {e: null}});
+    allDocuments.push({a: undefined, b: null, c: [null], d: {e: undefined}});
+
+    bulk = coll.initializeUnorderedBulkOp();
+    allDocuments.forEach(function(doc) {
+        bulk.insert(doc);
+    });
+    assert.writeOK(bulk.execute());
+
+    assertIdentityViewDistinctMatchesCollection("a");
+    assertIdentityViewDistinctMatchesCollection("b");
+    assertIdentityViewDistinctMatchesCollection("c");
+    assertIdentityViewDistinctMatchesCollection("d");
+    assertIdentityViewDistinctMatchesCollection("e");
 }());
diff --git a/src/mongo/db/query/parsed_distinct.cpp b/src/mongo/db/query/parsed_distinct.cpp
index 19a092ebb39..02de55ed4e8 100644
--- a/src/mongo/db/query/parsed_distinct.cpp
+++ b/src/mongo/db/query/parsed_distinct.cpp
@@ -35,6 +35,7 @@
 #include "mongo/db/query/canonical_query.h"
 #include "mongo/db/query/query_request.h"
 #include "mongo/stdx/memory.h"
+#include "mongo/util/mongoutils/str.h"
 
 namespace mongo {
 
@@ -53,7 +54,8 @@ StatusWith<BSONObj> ParsedDistinct::asAggregationCommand() const {
     // pipeline that looks like this:
     //
     //      [
-    //          { $match: { ... },
+    //          { $match: { ... } },
+    //          { $unwind: { path: "$<key>", preserveNullAndEmptyArrays: true } },
     //          { $group: { _id: null, distinct: { $addToSet: "$<key>" } } }
     //      ]
     BSONArrayBuilder pipelineBuilder(aggregationBuilder.subarrayStart("pipeline"));
@@ -62,6 +64,14 @@ StatusWith<BSONObj> ParsedDistinct::asAggregationCommand() const {
         matchStageBuilder.append("$match", qr.getFilter());
         matchStageBuilder.doneFast();
     }
+    BSONObjBuilder unwindStageBuilder(pipelineBuilder.subobjStart());
+    {
+        BSONObjBuilder unwindBuilder(unwindStageBuilder.subobjStart("$unwind"));
+        unwindBuilder.append("path", str::stream() << "$" << _key);
+        unwindBuilder.append("preserveNullAndEmptyArrays", true);
+        unwindBuilder.doneFast();
+    }
+    unwindStageBuilder.doneFast();
     BSONObjBuilder groupStageBuilder(pipelineBuilder.subobjStart());
     {
         BSONObjBuilder groupBuilder(groupStageBuilder.subobjStart("$group"));
diff --git a/src/mongo/db/query/parsed_distinct_test.cpp b/src/mongo/db/query/parsed_distinct_test.cpp
index 60c993756a7..38dec86cc63 100644
--- a/src/mongo/db/query/parsed_distinct_test.cpp
+++ b/src/mongo/db/query/parsed_distinct_test.cpp
@@ -66,12 +66,20 @@ TEST(ParsedDistinctTest, ConvertToAggregationNoQuery) {
     ASSERT_BSONOBJ_EQ(ar.getValue().getCollation(), BSONObj());
 
     std::vector<BSONObj> expectedPipeline{
+        BSON("$unwind" << BSON("path"
+                               << "$x"
+                               << "preserveNullAndEmptyArrays"
+                               << true)),
         BSON("$group" << BSON("_id" << BSONNULL << "distinct" << BSON("$addToSet"
                                                                       << "$x")))};
     ASSERT(std::equal(expectedPipeline.begin(),
                       expectedPipeline.end(),
                       ar.getValue().getPipeline().begin(),
                       SimpleBSONObjComparator::kInstance.makeEqualTo()));
+    ASSERT(std::equal(ar.getValue().getPipeline().begin(),
+                      ar.getValue().getPipeline().end(),
+                      expectedPipeline.begin(),
+                      SimpleBSONObjComparator::kInstance.makeEqualTo()));
 }
 
 TEST(ParsedDistinctTest, ConvertToAggregationWithQuery) {
@@ -98,12 +106,20 @@ TEST(ParsedDistinctTest, ConvertToAggregationWithQuery) {
 
     std::vector<BSONObj> expectedPipeline{
         BSON("$match" << BSON("z" << 7)),
+        BSON("$unwind" << BSON("path"
+                               << "$y"
+                               << "preserveNullAndEmptyArrays"
+                               << true)),
         BSON("$group" << BSON("_id" << BSONNULL << "distinct" << BSON("$addToSet"
                                                                       << "$y")))};
     ASSERT(std::equal(expectedPipeline.begin(),
                       expectedPipeline.end(),
                       ar.getValue().getPipeline().begin(),
                       SimpleBSONObjComparator::kInstance.makeEqualTo()));
+    ASSERT(std::equal(ar.getValue().getPipeline().begin(),
+                      ar.getValue().getPipeline().end(),
+                      expectedPipeline.begin(),
+                      SimpleBSONObjComparator::kInstance.makeEqualTo()));
 }
 
 TEST(ParsedDistinctTest, ConvertToAggregationWithExplain) {
@@ -129,12 +145,20 @@ TEST(ParsedDistinctTest, ConvertToAggregationWithExplain) {
     ASSERT_BSONOBJ_EQ(ar.getValue().getCollation(), BSONObj());
 
     std::vector<BSONObj> expectedPipeline{
+        BSON("$unwind" << BSON("path"
+                               << "$x"
+                               << "preserveNullAndEmptyArrays"
+                               << true)),
         BSON("$group" << BSON("_id" << BSONNULL << "distinct" << BSON("$addToSet"
                                                                       << "$x")))};
     ASSERT(std::equal(expectedPipeline.begin(),
                       expectedPipeline.end(),
                       ar.getValue().getPipeline().begin(),
                       SimpleBSONObjComparator::kInstance.makeEqualTo()));
+    ASSERT(std::equal(ar.getValue().getPipeline().begin(),
+                      ar.getValue().getPipeline().end(),
+                      expectedPipeline.begin(),
+                      SimpleBSONObjComparator::kInstance.makeEqualTo()));
 }
 
 }  // namespace