SERVER-41771 Use `clang-format-7.0.1` in our clang-format helper script

Update our clang-format config file to reflect the newer syntax.

3 files changed, 159 insertions, 56 deletions

diff --git a/.clang-format b/.clang-format
index 40241c181e1..0261864a922 100644
--- a/.clang-format
+++ b/.clang-format
@@ -1,11 +1,10 @@
+---
 Language:        Cpp
-# BasedOnStyle:  Google
-# ---
 AccessModifierOffset: -4
 AlignAfterOpenBracket: Align
 AlignConsecutiveAssignments: false
 AlignConsecutiveDeclarations: false
-AlignEscapedNewlinesLeft: true
+AlignEscapedNewlines: Left
 AlignOperands:   false
 AlignTrailingComments: true
 AllowAllParametersOfDeclarationOnNextLine: true
@@ -17,10 +16,10 @@ AllowShortLoopsOnASingleLine: false
 AlwaysBreakAfterDefinitionReturnType: None
 AlwaysBreakAfterReturnType: None
 AlwaysBreakBeforeMultilineStrings: true
-AlwaysBreakTemplateDeclarations: true
+AlwaysBreakTemplateDeclarations: Yes
 BinPackArguments: false
 BinPackParameters: false
-BraceWrapping:
+BraceWrapping:   
   AfterClass:      false
   AfterControlStatement: false
   AfterEnum:       false
@@ -29,15 +28,25 @@ BraceWrapping:
   AfterObjCDeclaration: false
   AfterStruct:     false
   AfterUnion:      false
+  AfterExternBlock: false
   BeforeCatch:     false
   BeforeElse:      false
   IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
 BreakBeforeBinaryOperators: None
 BreakBeforeBraces: Attach
+BreakBeforeInheritanceComma: false
+BreakInheritanceList: BeforeColon
 BreakBeforeTernaryOperators: true
 BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
 ColumnLimit:     100
 CommentPragmas:  ''
+CompactNamespaces: false
 ConstructorInitializerAllOnOneLineOrOnePerLine: true
 ConstructorInitializerIndentWidth: 4
 ContinuationIndentWidth: 4
@@ -45,33 +54,55 @@ Cpp11BracedListStyle: true
 DerivePointerAlignment: false
 DisableFormat:   false
 ExperimentalAutoDetectBinPacking: false
-ForEachMacros:   [ foreach, Q_FOREACH, BOOST_FOREACH ]
-IncludeCategories:
+FixNamespaceComments: true
+ForEachMacros:   
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IncludeBlocks:   Preserve
+IncludeCategories: 
+  - Regex:           '.*'
+    Priority:        1
+  - Regex:           '^(<|"(gtest|gmock|isl|json)/)'
+    Priority:        3
   - Regex:           '.*'
     Priority:        1
+IncludeIsMainRegex: '(Test)?$'
 IndentCaseLabels: true
+IndentPPDirectives: None
 IndentWidth:     4
 IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
 KeepEmptyLinesAtTheStartOfBlocks: true
 MacroBlockBegin: ''
 MacroBlockEnd:   ''
 MaxEmptyLinesToKeep: 2
 NamespaceIndentation: None
+ObjCBinPackProtocolList: Auto
 ObjCBlockIndentWidth: 2
 ObjCSpaceAfterProperty: false
 ObjCSpaceBeforeProtocolList: false
+PenaltyBreakAssignment: 2
 PenaltyBreakBeforeFirstCallParameter: 19
 PenaltyBreakComment: 300
 PenaltyBreakFirstLessLess: 120
 PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
 PenaltyExcessCharacter: 1000000
 PenaltyReturnTypeOnItsOwnLine: 200
 PointerAlignment: Left
 ReflowComments:  true
 SortIncludes:    true
+SortUsingDeclarations: true
 SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
 SpaceBeforeAssignmentOperators: true
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
 SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: true
 SpaceInEmptyParentheses: false
 SpacesBeforeTrailingComments: 2
 SpacesInAngles:  false
@@ -82,16 +113,15 @@ SpacesInSquareBrackets: false
 Standard:        Cpp11
 TabWidth:        4
 UseTab:          Never
----
+...
 
+---
 Language:        JavaScript
-# BasedOnStyle:  Google
-# ---
 AccessModifierOffset: -1
 AlignAfterOpenBracket: Align
 AlignConsecutiveAssignments: false
 AlignConsecutiveDeclarations: false
-AlignEscapedNewlinesLeft: true
+AlignEscapedNewlines: Left
 AlignOperands:   false
 AlignTrailingComments: true
 AllowAllParametersOfDeclarationOnNextLine: true
@@ -103,10 +133,10 @@ AllowShortLoopsOnASingleLine: false
 AlwaysBreakAfterDefinitionReturnType: None
 AlwaysBreakAfterReturnType: None
 AlwaysBreakBeforeMultilineStrings: true
-AlwaysBreakTemplateDeclarations: true
+AlwaysBreakTemplateDeclarations: Yes
 BinPackArguments: false
 BinPackParameters: false
-BraceWrapping:
+BraceWrapping:   
   AfterClass:      false
   AfterControlStatement: false
   AfterEnum:       false
@@ -115,15 +145,25 @@ BraceWrapping:
   AfterObjCDeclaration: false
   AfterStruct:     false
   AfterUnion:      false
+  AfterExternBlock: false
   BeforeCatch:     false
   BeforeElse:      false
   IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
 BreakBeforeBinaryOperators: None
 BreakBeforeBraces: Attach
+BreakBeforeInheritanceComma: false
+BreakInheritanceList: BeforeColon
 BreakBeforeTernaryOperators: true
 BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
 ColumnLimit:     100
 CommentPragmas:  ''
+CompactNamespaces: false
 ConstructorInitializerAllOnOneLineOrOnePerLine: true
 ConstructorInitializerIndentWidth: 4
 ContinuationIndentWidth: 4
@@ -131,33 +171,55 @@ Cpp11BracedListStyle: true
 DerivePointerAlignment: true
 DisableFormat:   false
 ExperimentalAutoDetectBinPacking: false
-ForEachMacros:   [ foreach, Q_FOREACH, BOOST_FOREACH ]
-IncludeCategories:
+FixNamespaceComments: true
+ForEachMacros:   
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IncludeBlocks:   Preserve
+IncludeCategories: 
+  - Regex:           '.*'
+    Priority:        1
+  - Regex:           '^(<|"(gtest|gmock|isl|json)/)'
+    Priority:        3
   - Regex:           '.*'
     Priority:        1
+IncludeIsMainRegex: '(Test)?$'
 IndentCaseLabels: true
+IndentPPDirectives: None
 IndentWidth:     4
 IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
 KeepEmptyLinesAtTheStartOfBlocks: false
 MacroBlockBegin: ''
 MacroBlockEnd:   ''
 MaxEmptyLinesToKeep: 1
 NamespaceIndentation: None
+ObjCBinPackProtocolList: Auto
 ObjCBlockIndentWidth: 2
 ObjCSpaceAfterProperty: false
 ObjCSpaceBeforeProtocolList: false
+PenaltyBreakAssignment: 2
 PenaltyBreakBeforeFirstCallParameter: 1
 PenaltyBreakComment: 300
 PenaltyBreakFirstLessLess: 120
 PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
 PenaltyExcessCharacter: 1000000
 PenaltyReturnTypeOnItsOwnLine: 200
 PointerAlignment: Left
 ReflowComments:  true
 SortIncludes:    true
+SortUsingDeclarations: true
 SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
 SpaceBeforeAssignmentOperators: true
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
 SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: true
 SpaceInEmptyParentheses: false
 SpacesBeforeTrailingComments: 2
 SpacesInAngles:  false
@@ -168,3 +230,5 @@ SpacesInSquareBrackets: false
 Standard:        Auto
 TabWidth:        4
 UseTab:          Never
+...
+
diff --git a/buildscripts/clang_format.py b/buildscripts/clang_format.py
index 4fe639b3c10..6acdcaff192 100755
--- a/buildscripts/clang_format.py
+++ b/buildscripts/clang_format.py
@@ -10,6 +10,7 @@
 
 import difflib
 import glob
+from io import StringIO
 import os
 import re
 import shutil
@@ -41,17 +42,20 @@ from buildscripts.linter import parallel  # pylint: disable=wrong-import-positio
 #
 
 # Expected version of clang-format
-CLANG_FORMAT_VERSION = "3.8.0"
-CLANG_FORMAT_SHORT_VERSION = "3.8"
+CLANG_FORMAT_VERSION = "7.0.1"
+CLANG_FORMAT_SHORT_VERSION = "7.0"
+CLANG_FORMAT_SHORTER_VERSION = "70"
 
 # Name of clang-format as a binary
 CLANG_FORMAT_PROGNAME = "clang-format"
 
 # URL location of the "cached" copy of clang-format to download
 # for users which do not have clang-format installed
-CLANG_FORMAT_HTTP_LINUX_CACHE = "https://s3.amazonaws.com/boxes.10gen.com/build/clang-format-3.8-rhel55.tar.gz"
+CLANG_FORMAT_HTTP_LINUX_CACHE = "https://s3.amazonaws.com/boxes.10gen.com/build/clang-format-7.0.1-rhel70.tar.gz"
 
-CLANG_FORMAT_HTTP_DARWIN_CACHE = "https://s3.amazonaws.com/boxes.10gen.com/build/clang%2Bllvm-3.8.0-x86_64-apple-darwin.tar.xz"
+CLANG_FORMAT_HTTP_DARWIN_CACHE = "https://s3.amazonaws.com/boxes.10gen.com/build/clang-format-7.0.1-x86_64-apple-darwin.tar.gz"
+
+CLANG_FORMAT_TOOLCHAIN_PATH = "/opt/mongodbtoolchain/v3/bin/clang-format"
 
 # Path in the tarball to the clang-format binary
 CLANG_FORMAT_SOURCE_TAR_BASE = string.Template("clang+llvm-$version-$tar_path/bin/" +
@@ -59,9 +63,9 @@ CLANG_FORMAT_SOURCE_TAR_BASE = string.Template("clang+llvm-$version-$tar_path/bi
 
 
 ##############################################################################
-def callo(args):
+def callo(args, **kwargs):
     """Call a program, and capture its output."""
-    return subprocess.check_output(args).decode('utf-8')
+    return subprocess.check_output(args, **kwargs).decode('utf-8')
 
 
 def get_tar_path(version, tar_path):
@@ -129,7 +133,8 @@ def get_clang_format_from_linux_cache(dest_file):
 class ClangFormat(object):
     """ClangFormat class."""
 
-    def __init__(self, path, cache_dir):  # pylint: disable=too-many-branches
+    def __init__(self, path, cache_dir):
+        # pylint: disable=too-many-branches,too-many-statements
         """Initialize ClangFormat."""
         self.path = None
         clang_format_progname_ext = ""
@@ -158,6 +163,7 @@ class ClangFormat(object):
             programs = [
                 CLANG_FORMAT_PROGNAME + "-" + CLANG_FORMAT_VERSION,
                 CLANG_FORMAT_PROGNAME + "-" + CLANG_FORMAT_SHORT_VERSION,
+                CLANG_FORMAT_PROGNAME + CLANG_FORMAT_SHORTER_VERSION,
                 CLANG_FORMAT_PROGNAME,
             ]
 
@@ -165,18 +171,26 @@ class ClangFormat(object):
                 for i, _ in enumerate(programs):
                     programs[i] += '.exe'
 
-            for program in programs:
-                self.path = spawn.find_executable(program)
-
-                if self.path:
+            # Check for the binary in the expected toolchain directory on non-windows systems.
+            if sys.platform != "win32":
+                if os.path.exists(CLANG_FORMAT_TOOLCHAIN_PATH):
+                    self.path = CLANG_FORMAT_TOOLCHAIN_PATH
                     if not self._validate_version():
                         self.path = None
-                    else:
-                        break
+
+            if self.path is None:
+                for program in programs:
+                    self.path = spawn.find_executable(program)
+
+                    if self.path:
+                        if not self._validate_version():
+                            self.path = None
+                        else:
+                            break
 
         # If Windows, try to grab it from Program Files
         # Check both native Program Files and WOW64 version
-        if sys.platform == "win32":
+        if self.path is None and sys.platform == "win32":
             programfiles = [
                 os.environ["ProgramFiles"],
                 os.environ["ProgramFiles(x86)"],
@@ -233,8 +247,13 @@ class ClangFormat(object):
         with open(file_name, 'rb') as original_text:
             original_file = original_text.read().decode('utf-8')
 
-        # Get formatted file as clang-format would format the file
-        formatted_file = callo([self.path, "--style=file", file_name])
+            original_text.seek(0)
+
+            # Get formatted file as clang-format would format the file
+            formatted_file = callo([
+                self.path, "--assume-filename=" +
+                (file_name if not file_name.endswith(".h") else file_name + "pp"), "--style=file"
+            ], stdin=original_text)
 
         if original_file != formatted_file:
             if print_diff:
@@ -245,8 +264,7 @@ class ClangFormat(object):
                 # Take a lock to ensure diffs do not get mixed when printed to the screen
                 with self.print_lock:
                     print("ERROR: Found diff for " + file_name)
-                    print("To fix formatting errors, run %s --style=file -i %s" % (self.path,
-                                                                                   file_name))
+                    print("To fix formatting errors, run `buildscripts/clang-format.py format`")
                     for line in result:
                         print(line.rstrip())
 
@@ -264,7 +282,25 @@ class ClangFormat(object):
             return True
 
         # Update the file with clang-format
-        formatted = not subprocess.call([self.path, "--style=file", "-i", file_name])
+        # We have to tell `clang-format` to format on standard input due to its file type
+        # determiner.  `--assume-filename` doesn't work directly on files, but only on standard
+        # input.  Thus we have to open the file as the subprocess's standard input. Then we record
+        # that formatted standard output back into the file.  We can't use the `-i` option, due to
+        # the fact that `clang-format` believes that many of our C++ headers are Objective-C code.
+        formatted = True
+        with open(file_name, 'rb') as source_stream:
+            try:
+                reformatted_text = subprocess.check_output([
+                    self.path, "--assume-filename=" +
+                    (file_name if not file_name.endswith(".h") else file_name + "pp"),
+                    "--style=file"
+                ], stdin=source_stream)
+            except subprocess.CalledProcessError:
+                formatted = False
+
+        if formatted:
+            with open(file_name, "wb") as output_stream:
+                output_stream.write(reformatted_text)
 
         # Version 3.8 generates files like foo.cpp~RF83372177.TMP when it formats foo.cpp
         # on Windows, we must clean these up
@@ -281,8 +317,8 @@ FILES_RE = re.compile('\\.(h|hpp|ipp|cpp|js)$')
 
 def is_interesting_file(file_name):
     """Return true if this file should be checked."""
-    return ((file_name.startswith("jstests") or file_name.startswith("src"))
-            and not file_name.startswith("src/third_party/")
+    return (file_name.startswith("jstests")
+            or file_name.startswith("src") and not file_name.startswith("src/third_party/")
             and not file_name.startswith("src/mongo/gotools/")) and FILES_RE.search(file_name)
 
 
@@ -303,7 +339,7 @@ def _lint_files(clang_format, files):
     lint_clean = parallel.parallel_process([os.path.abspath(f) for f in files], clang_format.lint)
 
     if not lint_clean:
-        print("ERROR: Code Style does not match coding style")
+        print("ERROR: Source code does not match required source formatting style")
         sys.exit(1)
 
 
@@ -421,22 +457,22 @@ def reformat_branch(  # pylint: disable=too-many-branches,too-many-locals,too-ma
             % (new_branch, new_branch))
 
     commits = get_list_from_lines(
-        repo.log(["--reverse", "--pretty=format:%H",
-                  "%s..HEAD" % commit_prior_to_reformat]))
+        repo.git_log([
+            "--reverse", "--no-show-signature", "--pretty=format:%H",
+            "%s..HEAD" % commit_prior_to_reformat
+        ]))
 
     previous_commit_base = commit_after_reformat
 
-    files_match = re.compile('\\.(h|cpp|js)$')
-
     # Go through all the commits the user made on the local branch and migrate to a new branch
     # that is based on post_reformat commits instead
     for commit_hash in commits:
-        repo.checkout(["--quiet", commit_hash])
+        repo.git_checkout(["--quiet", commit_hash])
 
         deleted_files = []
 
         # Format each of the files by checking out just a single commit from the user's branch
-        commit_files = get_list_from_lines(repo.diff(["HEAD~", "--name-only"]))
+        commit_files = get_list_from_lines(repo.git_diff(["HEAD~", "--name-only"]))
 
         for commit_file in commit_files:
 
@@ -447,7 +483,7 @@ def reformat_branch(  # pylint: disable=too-many-branches,too-many-locals,too-ma
                 deleted_files.append(commit_file)
                 continue
 
-            if files_match.search(commit_file):
+            if is_interesting_file(commit_file):
                 clang_format.format(commit_file)
             else:
                 print("Skipping file '%s' since it is not a file clang_format should format" %
@@ -457,44 +493,44 @@ def reformat_branch(  # pylint: disable=too-many-branches,too-many-locals,too-ma
         if not repo.is_working_tree_dirty():
             print("Commit %s needed no reformatting" % commit_hash)
         else:
-            repo.commit(["--all", "--amend", "--no-edit"])
+            repo.git_commit(["--all", "--amend", "--no-edit"])
 
         # Rebase our new commit on top the post-reformat commit
-        previous_commit = repo.rev_parse(["HEAD"])
+        previous_commit = repo.git_rev_parse(["HEAD"])
 
         # Checkout the new branch with the reformatted commits
         # Note: we will not name as a branch until we are done with all commits on the local branch
-        repo.checkout(["--quiet", previous_commit_base])
+        repo.git_checkout(["--quiet", previous_commit_base])
 
         # Copy each file from the reformatted commit on top of the post reformat
         diff_files = get_list_from_lines(
-            repo.diff(["%s~..%s" % (previous_commit, previous_commit), "--name-only"]))
+            repo.git_diff(["%s~..%s" % (previous_commit, previous_commit), "--name-only"]))
 
         for diff_file in diff_files:
             # If the file was deleted in the commit we are reformatting, we need to delete it again
             if diff_file in deleted_files:
-                repo.rm([diff_file])
+                repo.git_rm(["--ignore-unmatch", diff_file])
                 continue
 
             # The file has been added or modified, continue as normal
-            file_contents = repo.show(["%s:%s" % (previous_commit, diff_file)])
+            file_contents = repo.git_show(["%s:%s" % (previous_commit, diff_file)])
 
             root_dir = os.path.dirname(diff_file)
             if root_dir and not os.path.exists(root_dir):
                 os.makedirs(root_dir)
 
-            with open(diff_file, "w+") as new_file:
+            with open(diff_file, "w+", encoding="utf-8") as new_file:
                 new_file.write(file_contents)
 
-            repo.add([diff_file])
+            repo.git_add([diff_file])
 
         # Create a new commit onto clang-formatted branch
-        repo.commit(["--reuse-message=%s" % previous_commit])
+        repo.git_commit(["--reuse-message=%s" % previous_commit, "--no-gpg-sign", "--allow-empty"])
 
-        previous_commit_base = repo.rev_parse(["HEAD"])
+        previous_commit_base = repo.git_rev_parse(["HEAD"])
 
     # Create a new branch to mark the hashes we have been using
-    repo.checkout(["-b", new_branch])
+    repo.git_checkout(["-b", new_branch])
 
     print("reformat-branch is done running.\n")
     print("A copy of your branch has been made named '%s', and formatted with clang-format.\n" %
@@ -531,7 +567,6 @@ def main():
         elif command == "format":
             format_func(options.clang_format)
         elif command == "format-my":
-
             format_my_func(options.clang_format, args[2] if len(args) > 2 else "origin/master")
         elif command == "reformat-branch":
 
diff --git a/buildscripts/linter/git_base.py b/buildscripts/linter/git_base.py
index 6af56ff29db..f7eaecae95c 100644
--- a/buildscripts/linter/git_base.py
+++ b/buildscripts/linter/git_base.py
@@ -25,6 +25,10 @@ class Repository(object):  # pylint: disable=too-many-public-methods
         """Run a git commit command."""
         return self._callgito("commit", args)
 
+    def git_checkout(self, args):
+        """Run a git checkout command."""
+        return self._callgito("checkout", args)
+
     def git_diff(self, args):
         """Run a git diff command."""
         return self._callgito("diff", args)