SERVER-30705 Add $v field for update semantics in oplog updates.

With the new UpdateNodes class hierarchy, there are two code paths for
applying an update to a document that have slightly different
semantics. The order of fields in the resulting document can vary
depending on which code path is used to apply an update. A difference
in ordering between documents in a replica set is considered a
"mismatch," so we need to ensure that secondaries always apply updates
using the same update system that the primary uses.

When an update executes as part of the application of an oplog entry,
the update is now allowed to have a $v field, which allows it to
specify which semantics were used by the operation that we are
replicating by applying the entry.  When the primary uses the new
semantics (because it is a 3.6 mongod with featureCompatibilityVersion
set to 3.6), it includes {$v: 1} in the oplog's update document to
indicate that the secondary should apply with the newer 'UpdateNode'
semantics.

There are two other places where we need this behavior:
  1) In role_graph_update.cpp, where the handleOplogUpdate observer
  needs to update its in-memory BSON representation of a role to
  reflect an update in the admin database and
  2) in the applyOps command, which is used for testing how oplog
  entries get applied.

Both these code paths set the fromOplogApplication flag, which
replaces the old fromReplication flag, and they also gain behavior
that used to be exclusive to oplog applications from
replication. (Specifically, they skip update validation checks, which
should have already passed before the oplog entry was created.)

1 files changed, 99 insertions, 21 deletions

diff --git a/src/mongo/db/update/update_driver.cpp b/src/mongo/db/update/update_driver.cpp
index e8b33765444..3c6124e4d8e 100644
--- a/src/mongo/db/update/update_driver.cpp
+++ b/src/mongo/db/update/update_driver.cpp
@@ -59,6 +59,23 @@ using pathsupport::EqualityMatches;
 
 namespace {
 
+StatusWith<UpdateSemantics> updateSemanticsFromElement(BSONElement element) {
+    if (element.type() != BSONType::NumberInt && element.type() != BSONType::NumberLong) {
+        return {ErrorCodes::BadValue, "'$v' (UpdateSemantics) field must be an integer."};
+    }
+
+    auto updateSemantics = element.numberLong();
+
+    if (updateSemantics < 0 ||
+        updateSemantics >= static_cast<int>(UpdateSemantics::kNumUpdateSemantics)) {
+        return {ErrorCodes::BadValue,
+                str::stream() << "Unrecognized value for '$v' (UpdateSemantics) field: "
+                              << updateSemantics};
+    }
+
+    return static_cast<UpdateSemantics>(updateSemantics);
+}
+
 modifiertable::ModifierType validateMod(BSONElement mod) {
     auto modType = modifiertable::getType(mod.fieldName());
 
@@ -94,7 +111,19 @@ bool parseUpdateExpression(
     const std::map<StringData, std::unique_ptr<ExpressionWithPlaceholder>>& arrayFilters) {
     bool positional = false;
     std::set<std::string> foundIdentifiers;
+    bool foundUpdateSemanticsField = false;
     for (auto&& mod : updateExpr) {
+        // If there is a "$v" field among the modifiers, it should have already been used by the
+        // caller to determine that this is the correct parsing function.
+        if (mod.fieldNameStringData() == LogBuilder::kUpdateSemanticsFieldName) {
+            uassert(ErrorCodes::BadValue,
+                    "Duplicate $v in oplog update document",
+                    !foundUpdateSemanticsField);
+            foundUpdateSemanticsField = true;
+            invariant(mod.numberLong() == static_cast<long long>(UpdateSemantics::kUpdateNode));
+            continue;
+        }
+
         auto modType = validateMod(mod);
         for (auto&& field : mod.Obj()) {
             auto statusWithPositional = UpdateObjectNode::parseAndMerge(
@@ -152,29 +181,70 @@ Status UpdateDriver::parse(
     // Register the fact that this driver is not doing a full object replacement.
     _replacementMode = false;
 
-    // If the featureCompatibilityVersion is 3.4, parse using the ModifierInterfaces.
-    if (serverGlobalParams.featureCompatibility.version.load() ==
-        ServerGlobalParams::FeatureCompatibility::Version::k34) {
-        uassert(
-            ErrorCodes::InvalidOptions,
-            str::stream() << "The featureCompatibilityVersion must be 3.6 to use arrayFilters. See "
-                          << feature_compatibility_version::kDochubLink
-                          << ".",
-            arrayFilters.empty());
-        for (auto&& mod : updateExpr) {
-            auto modType = validateMod(mod);
-            for (auto&& field : mod.Obj()) {
-                auto status = addAndParse(modType, field);
-                if (!status.isOK()) {
-                    return status;
+    // Decide which update semantics to used, using the criteria outlined in the comment above this
+    // function's declaration.
+    BSONElement updateSemanticsElement = updateExpr[LogBuilder::kUpdateSemanticsFieldName];
+    UpdateSemantics updateSemantics;
+    if (updateSemanticsElement) {
+        if (!_modOptions.fromOplogApplication) {
+            return {ErrorCodes::FailedToParse, "The $v update field is only recognized internally"};
+        }
+        auto statusWithUpdateSemantics = updateSemanticsFromElement(updateSemanticsElement);
+        if (!statusWithUpdateSemantics.isOK()) {
+            return statusWithUpdateSemantics.getStatus();
+        }
+
+        updateSemantics = statusWithUpdateSemantics.getValue();
+    } else if (_modOptions.fromOplogApplication) {
+        updateSemantics = UpdateSemantics::kModifierInterface;
+    } else {
+        updateSemantics = (serverGlobalParams.featureCompatibility.version.load() ==
+                           ServerGlobalParams::FeatureCompatibility::Version::k34)
+            ? UpdateSemantics::kModifierInterface
+            : UpdateSemantics::kUpdateNode;
+    }
+
+    switch (updateSemantics) {
+        case UpdateSemantics::kModifierInterface: {
+            uassert(ErrorCodes::InvalidOptions,
+                    str::stream()
+                        << "The featureCompatibilityVersion must be 3.6 to use arrayFilters. See "
+                        << feature_compatibility_version::kDochubLink
+                        << ".",
+                    arrayFilters.empty());
+            bool foundUpdateSemanticsField = false;
+            for (auto&& mod : updateExpr) {
+                // If there is a "$v" field among the modifiers, we have already used it to
+                // determine that this is the correct parsing code.
+                if (mod.fieldNameStringData() == LogBuilder::kUpdateSemanticsFieldName) {
+                    uassert(ErrorCodes::BadValue,
+                            "Duplicate $v in oplog update document",
+                            !foundUpdateSemanticsField);
+                    foundUpdateSemanticsField = true;
+                    invariant(mod.numberLong() ==
+                              static_cast<long long>(UpdateSemantics::kModifierInterface));
+                    continue;
+                }
+
+                auto modType = validateMod(mod);
+                for (auto&& field : mod.Obj()) {
+                    auto status = addAndParse(modType, field);
+                    if (!status.isOK()) {
+                        return status;
+                    }
                 }
             }
+            break;
         }
-    } else {
-        auto root = stdx::make_unique<UpdateObjectNode>();
-        _positional =
-            parseUpdateExpression(updateExpr, root.get(), _modOptions.collator, arrayFilters);
-        _root = std::move(root);
+        case UpdateSemantics::kUpdateNode: {
+            auto root = stdx::make_unique<UpdateObjectNode>();
+            _positional =
+                parseUpdateExpression(updateExpr, root.get(), _modOptions.collator, arrayFilters);
+            _root = std::move(root);
+            break;
+        }
+        default:
+            MONGO_UNREACHABLE;
     }
 
     return Status::OK();
@@ -276,7 +346,7 @@ Status UpdateDriver::update(StringData matchedField,
         UpdateNode::ApplyParams applyParams(doc->root(), immutablePaths);
         applyParams.matchedField = matchedField;
         applyParams.insert = _insert;
-        applyParams.fromReplication = _modOptions.fromReplication;
+        applyParams.fromOplogApplication = _modOptions.fromOplogApplication;
         applyParams.validateForStorage = validateForStorage;
         applyParams.indexData = _indexedFields;
         if (_logOp && logOpRec) {
@@ -290,6 +360,14 @@ Status UpdateDriver::update(StringData matchedField,
         if (docWasModified) {
             *docWasModified = !applyResult.noop;
         }
+        if (!_replacementMode && _logOp && logOpRec) {
+            // When using kUpdateNode update semantics on the primary, we must include a "$v" field
+            // in the update document so that the secondary knows to apply the update with
+            // kUpdateNode semantics. If this is a full document replacement, we don't need to
+            // specify the semantics (and there would be no place to put a "$v" field in the update
+            // document).
+            invariantOK(logBuilder.setUpdateSemantics(UpdateSemantics::kUpdateNode));
+        }
 
     } else {