SERVER-52818 Fix SBE's implementation of strings to handle null bytes correctly

4 files changed, 26 insertions, 190 deletions

diff --git a/jstests/aggregation/bugs/server6195.js b/jstests/aggregation/bugs/server6195.js
index 13dccd10877..9bca99f2921 100644
--- a/jstests/aggregation/bugs/server6195.js
+++ b/jstests/aggregation/bugs/server6195.js
@@ -4,7 +4,7 @@ load('jstests/aggregation/extras/utils.js');
 
 c = db.s6570;
 c.drop();
-c.save({v: "$", w: ".", x: "foo", y: "bar"});
+c.save({v: "$", w: ".", x: "foo", y: "bar", z: "z\0z"});
 
 assert.eq(c.aggregate({$project: {str: {$concat: ["X", "$x", "Y", "$y"]}}}).toArray()[0].str,
           "XfooYbar");
@@ -12,6 +12,8 @@ assert.eq(c.aggregate({$project: {str: {$concat: ["$v", "X", "$w", "Y"]}}}).toAr
           "$X.Y");
 assert.eq(c.aggregate({$project: {str: {$concat: ["$w", "X", "$v", "Y"]}}}).toArray()[0].str,
           ".X$Y");
+assert.eq(c.aggregate({$project: {str: {$concat: ["X", "$z", "a\0a", "Y"]}}}).toArray()[0].str,
+          "Xz\0za\0aY");
 
 // Nullish (both with and without other strings)
 assert.isnull(c.aggregate({$project: {str: {$concat: ["$missing"]}}}).toArray()[0].str);
diff --git a/jstests/aggregation/bugs/upperlower.js b/jstests/aggregation/bugs/upperlower.js
index 22b30fbe66b..9f02d9ade27 100644
--- a/jstests/aggregation/bugs/upperlower.js
+++ b/jstests/aggregation/bugs/upperlower.js
@@ -30,6 +30,8 @@ assertResult('', '', ['']);
 assertResult('AB', 'ab', 'aB');
 assertResult('AB', 'ab', ['Ab']);
 assertResult('ABZ', 'abz', 'aBz');
+assertResult('A\0B\0\0Z', 'a\0b\0\0z', 'a\0B\0\0z');
+assertResult('A\0\0B\0Z', 'a\0\0b\0z', ['A\0\0b\0z']);
 
 // With non alphabet characters.
 assertResult('1', '1', '1');
@@ -69,7 +71,9 @@ assert.commandWorked(coll.insert({
     numberInt: NumberInt(42),
     numberDecimal: NumberDecimal(42.213),
     nested: {str: "hello world"},
-    unicode: "\u1ebd"
+    unicode: "\u1ebd",
+    string_with_nulls_1: "a\0B\0\0z",
+    string_with_nulls_2: "A\0\0b\0z"
 }));
 assertResult('-_AB', '-_ab', '$string');
 assertResult(
@@ -78,4 +82,6 @@ assertResult('2090845886852', '2090845886852', '$numberLong');
 assertResult('42', '42', '$numberInt');
 assertResult('HELLO WORLD', 'hello world', '$nested.str');
 assertResult('\u1ebd', '\u1ebd', '$unicode');
+assertResult('A\0B\0\0Z', 'a\0b\0\0z', '$string_with_nulls_1');
+assertResult('A\0\0B\0Z', 'a\0\0b\0z', '$string_with_nulls_2');
 }());
diff --git a/jstests/aggregation/expressions/split.js b/jstests/aggregation/expressions/split.js
index 0ef87b99741..9e58d581a24 100644
--- a/jstests/aggregation/expressions/split.js
+++ b/jstests/aggregation/expressions/split.js
@@ -1,17 +1,22 @@
 // In SERVER-6773, the $split expression was introduced. In this file, we test the functionality and
 // error cases of the expression.
-// @tags: [
-//   sbe_incompatible,
-// ]
-load("jstests/aggregation/extras/utils.js");  // For assertErrorCode and testExpression.
 
 (function() {
 "use strict";
 
-var coll = db.split;
+load("jstests/aggregation/extras/utils.js");  // For assertErrorCode and testExpression.
+load("jstests/libs/sbe_assert_error_override.js");
+
+const coll = db.split;
+
 coll.drop();
 assert.commandWorked(coll.insert({}));
 
+//
+// Tests with constant-folding optimization.
+//
+
+// Basic tests.
 testExpression(coll, {$split: ["abc", "b"]}, ["a", "c"]);
 testExpression(coll, {$split: ["aaa", "b"]}, ["aaa"]);
 testExpression(coll, {$split: ["a b a", "b"]}, ["a ", " a"]);
@@ -27,8 +32,6 @@ testExpression(coll, {$split: ["\0a\0", "a"]}, ["\0", "\0"]);
 testExpression(coll, {$split: ["\0\0\0", "a"]}, ["\0\0\0"]);
 
 // Ensure that $split operates correctly when the string has multi-byte tokens or input strings.
-// Note that this expression is not unicode-aware; splitting is based wholly off of the byte
-// sequence of the input and token.
 testExpression(coll, {$split: ["∫a∫", "a"]}, ["∫", "∫"]);
 testExpression(coll, {$split: ["a∫∫a", "∫"]}, ["a", "", "a"]);
 
@@ -40,8 +43,12 @@ testExpression(coll, {$split: [null, "abc"]}, null);
 testExpression(coll, {$split: ["$a", "a"]}, null);
 testExpression(coll, {$split: ["a", "$a"]}, null);
 
+//
+// Error Code tests with constant-folding optimization.
+//
+
 // Ensure that $split errors when given more or less than two arguments.
-var pipeline = {$project: {split: {$split: []}}};
+let pipeline = {$project: {split: {$split: []}}};
 assertErrorCode(coll, pipeline, 16020);
 
 pipeline = {
@@ -70,4 +77,4 @@ pipeline = {
     $project: {split: {$split: ["abc", ""]}}
 };
 assertErrorCode(coll, pipeline, 40087);
-}());
+})();
diff --git a/jstests/aggregation/expressions/split_sbe.js b/jstests/aggregation/expressions/split_sbe.js
deleted file mode 100644
index b6757dc3998..00000000000
--- a/jstests/aggregation/expressions/split_sbe.js
+++ /dev/null
@@ -1,179 +0,0 @@
-// Tests for the $split expression, for the SBE and classic engines. This file was created for
-// SERVER-51554 because a couple tests in jstests/aggregation/expressions/split.js didn't work in
-// SBE due to bugs in value::TypeTags::StringSmall and value::TypeTags::StringBig (SERVER-52818).
-// When resolved, this file can be combined with split.js.
-
-(function() {
-"use strict";
-
-load("jstests/aggregation/extras/utils.js");  // For assertErrorCode and testExpression.
-load("jstests/libs/sbe_assert_error_override.js");
-
-const coll = db.split;
-
-coll.drop();
-assert.commandWorked(coll.insert({}));
-
-//
-// Tests with constant-folding optimization.
-//
-
-// Basic tests.
-testExpression(coll, {$split: ["abc", "b"]}, ["a", "c"]);
-testExpression(coll, {$split: ["aaa", "b"]}, ["aaa"]);
-testExpression(coll, {$split: ["a b a", "b"]}, ["a ", " a"]);
-testExpression(coll, {$split: ["a", "a"]}, ["", ""]);
-testExpression(coll, {$split: ["aa", "a"]}, ["", "", ""]);
-testExpression(coll, {$split: ["aaa", "a"]}, ["", "", "", ""]);
-testExpression(coll, {$split: ["", "a"]}, [""]);
-testExpression(coll, {$split: ["abc abc cba abc", "abc"]}, ["", " ", " cba ", ""]);
-
-// Ensure that $split operates correctly when the string has multi-byte tokens or input strings.
-testExpression(coll, {$split: ["∫a∫", "a"]}, ["∫", "∫"]);
-testExpression(coll, {$split: ["a∫∫a", "∫"]}, ["a", "", "a"]);
-
-// Ensure that $split produces null when given null as input.
-testExpression(coll, {$split: ["abc", null]}, null);
-testExpression(coll, {$split: [null, "abc"]}, null);
-
-// Ensure that $split produces null when given missing fields as input.
-testExpression(coll, {$split: ["$a", "a"]}, null);
-testExpression(coll, {$split: ["a", "$a"]}, null);
-
-//
-// Tests without constant-folding optimization.
-//
-
-// Basic tests.
-assert(coll.drop());
-assert.commandWorked(coll.insert([
-    {string: "abc"},
-    {string: "a"},
-    {string: "aa"},
-    {string: "aaa"},
-    {string: "bbb"},
-    {string: "b a b"},
-    {string: ""},
-    {string: "a a b a"},
-]));
-
-let res = coll.aggregate({$project: {_id: 0, output: {$split: ["$string", "a"]}}}).toArray();
-assert(arrayEq(res, [
-    {output: ["", "bc"]},
-    {output: ["", ""]},
-    {output: ["", "", ""]},
-    {output: ["", "", "", ""]},
-    {output: ["bbb"]},
-    {output: ["b ", " b"]},
-    {output: [""]},
-    {output: ["", " ", " b ", ""]},
-]));
-
-// Ensure that $split operates correctly when the string has multi-byte tokens or input strings.
-assert(coll.drop());
-assert.commandWorked(coll.insert([
-    {string: "∫a∫"},
-    {string: "a∫∫a"},
-]));
-
-res = coll.aggregate({$project: {_id: 0, output: {$split: ["$string", "a"]}}}).toArray();
-
-assert(arrayEq(res, [
-    {output: ["∫", "∫"]},
-    {output: ["", "∫∫", ""]},
-]));
-
-res = coll.aggregate({$project: {_id: 0, output: {$split: ["$string", "∫"]}}}).toArray();
-
-assert(arrayEq(res, [
-    {output: ["", "a", ""]},
-    {output: ["a", "", "a"]},
-]));
-
-// Ensure that $split produces null when given null as input.
-assert(coll.drop());
-assert.commandWorked(coll.insert({string: "abc"}));
-res = coll.aggregate({$project: {_id: 0, output: {$split: ["$string", null]}}}).toArray();
-assert(arrayEq(res, [{output: null}]));
-
-assert(coll.drop());
-assert.commandWorked(coll.insert({string: null}));
-res = coll.aggregate({$project: {_id: 0, output: {$split: ["$string", "abc"]}}}).toArray();
-assert(arrayEq(res, [{output: null}]));
-
-// Ensure that $split produces null when given missing fields as input.
-res = coll.aggregate({$project: {_id: 0, output: {$split: ["$a", "a"]}}}).toArray();
-assert(arrayEq(res, [{output: null}]));
-res = coll.aggregate({$project: {_id: 0, output: {$split: ["a", "$a"]}}}).toArray();
-assert(arrayEq(res, [{output: null}]));
-
-//
-// Error Code tests with constant-folding optimization.
-//
-
-// Ensure that $split errors when given more or less than two arguments.
-var pipeline = {$project: {split: {$split: []}}};
-assertErrorCode(coll, pipeline, 16020);
-
-pipeline = {
-    $project: {split: {$split: ["a"]}}
-};
-assertErrorCode(coll, pipeline, 16020);
-
-pipeline = {
-    $project: {split: {$split: ["a", "b", "c"]}}
-};
-assertErrorCode(coll, pipeline, 16020);
-
-// Ensure that $split errors when given non-string input.
-pipeline = {
-    $project: {split: {$split: [1, "abc"]}}
-};
-assertErrorCode(coll, pipeline, 40085);
-
-pipeline = {
-    $project: {split: {$split: ["abc", 1]}}
-};
-assertErrorCode(coll, pipeline, 40086);
-
-// Ensure that $split errors when given an empty separator.
-pipeline = {
-    $project: {split: {$split: ["abc", ""]}}
-};
-assertErrorCode(coll, pipeline, 40087);
-
-//
-// Error Code tests without constant-folding optimization.
-//
-
-// Ensure that $split errors when given more or less than two arguments.
-pipeline = {
-    $project: {split: {$split: ["$string"]}}
-};
-assertErrorCode(coll, pipeline, 16020);
-
-pipeline = {
-    $project: {split: {$split: ["$string", "b", "c"]}}
-};
-assertErrorCode(coll, pipeline, 16020);
-
-// Ensure that $split errors when given non-string input.
-assert(coll.drop());
-assert.commandWorked(coll.insert({string: "abc"}));
-
-pipeline = {
-    $project: {split: {$split: [1, "$string"]}}
-};
-assertErrorCode(coll, pipeline, 40085);
-
-pipeline = {
-    $project: {split: {$split: ["$string", 1]}}
-};
-assertErrorCode(coll, pipeline, 40086);
-
-// Ensure that $split errors when given an empty separator.
-pipeline = {
-    $project: {split: {$split: ["$string", ""]}}
-};
-assertErrorCode(coll, pipeline, 40087);
-})();