Port erlfmt formatting to 3.x

From PR: #3568

A few changes to the formatting logic compared to `main`:

  * rebar3 and `erlfmt` don't work with Erlang 20, so added an OTP version
    check to skip formatting for Erlang 20

  * `make erlfmt-*` commands are faster. Instead of spawning an `erlfmt`
    process per-file, spawn one per `directory/*.erl` pattern.

  * Remove the non-0 return code work-around. `erlfmt` was returning an error
    on `main` because the line width option used for formatting was different
    than the one used during format checking. Making both the default removed
    all the spurious non-0 exits with the current version of `erlfmt`. Thanks
    to Adam Kocoloski for pointing out the issue.

10 files changed, 236 insertions, 6 deletions

diff --git a/.gitignore b/.gitignore
index 3d49a336e..fe3e5acaa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -80,6 +80,8 @@ src/ssl_verify_fun/
 src/triq/
 src/unicode_util_compat/
 src/file_system/
+src/rebar3/
+src/erlfmt/
 tmp/
 
 src/couch/*.o
diff --git a/Makefile b/Makefile
index 7e369a084..e3916fc70 100644
--- a/Makefile
+++ b/Makefile
@@ -17,6 +17,7 @@
 include version.mk
 
 REBAR?=$(shell echo `pwd`/bin/rebar)
+ERLFMT?=$(shell echo `pwd`/bin/erlfmt)
 
 # Handle the following scenarios:
 #   1. When building from a tarball, use version.mk.
@@ -202,6 +203,12 @@ soak-eunit: couch
 	@$(REBAR) setup_eunit 2> /dev/null
 	while [ $$? -eq 0 ] ; do $(REBAR) -r eunit $(EUNIT_OPTS) ; done
 
+erlfmt-check:
+	ERLFMT_PATH=$(ERLFMT) python3 dev/format_check.py
+
+erlfmt-format:
+	ERLFMT_PATH=$(ERLFMT) python3 dev/format_all.py
+
 .venv/bin/black:
 	@python3 -m venv .venv
 	@.venv/bin/pip3 install black || touch .venv/bin/black
@@ -212,8 +219,8 @@ python-black: .venv/bin/black
 	       echo "Python formatter not supported on Python < 3.6; check results on a newer platform"
 	@python3 -c "import sys; exit(1 if sys.version_info >= (3,6) else 0)" || \
 		LC_ALL=C.UTF-8 LANG=C.UTF-8 .venv/bin/black --check \
-		--exclude="build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|src/rebar/pr2relnotes.py|src/fauxton" \
-		build-aux/*.py dev/run src/mango/test/*.py src/docs/src/conf.py src/docs/ext/*.py .
+		--exclude="build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|src/erlfmt|src/rebar/pr2relnotes.py|src/fauxton" \
+		build-aux/*.py dev/run dev/format_*.py src/mango/test/*.py src/docs/src/conf.py src/docs/ext/*.py .
 
 python-black-update: .venv/bin/black
 	@python3 -c "import sys; exit(1 if sys.version_info < (3,6) else 0)" || \
diff --git a/Makefile.win b/Makefile.win
index 25c4d9fa7..62a0a8dec 100644
--- a/Makefile.win
+++ b/Makefile.win
@@ -18,6 +18,7 @@ include version.mk
 
 SHELL=cmd.exe
 REBAR=bin\rebar.cmd
+ERLFMT=bin\erlfmt
 MAKE=make -f Makefile.win
 # REBAR?=$(shell where rebar.cmd)
 
@@ -171,6 +172,12 @@ just-eunit: export ERL_AFLAGS = "-config $(shell echo %cd%)/rel/files/eunit.conf
 just-eunit:
 	@$(REBAR) -r eunit $(EUNIT_OPTS)
 
+erlfmt-check:
+	ERLFMT_PATH=bin\erlfmt python3 dev\format_check.py
+
+erlfmt-format:
+	ERLFMT_PATH=bin\erlfmt python3 dev\format_all.py
+
 .venv/bin/black:
 	@python.exe -m venv .venv
 	@.venv\Scripts\pip3.exe install black || copy /b .venv\Scripts\black.exe +,,
@@ -181,16 +188,16 @@ python-black: .venv/bin/black
 		echo 'Python formatter not supported on Python < 3.6; check results on a newer platform'
 	@python.exe -c "import sys; exit(1 if sys.version_info >= (3,6) else 0)" || \
 		.venv\Scripts\black.exe --check \
-		--exclude="build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|src/rebar/pr2relnotes.py|src/fauxton" \
-		build-aux dev\run src\mango\test src\docs\src\conf.py src\docs\ext .
+		--exclude="build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|src/erlfmt|src/rebar/pr2relnotes.py|src/fauxton" \
+		build-aux dev\run dev\format_*.py src\mango\test src\docs\src\conf.py src\docs\ext .
 
 python-black-update: .venv/bin/black
 	@python.exe -c "import sys; exit(1 if sys.version_info < (3,6) else 0)" || \
 		echo 'Python formatter not supported on Python < 3.6; check results on a newer platform'
 	@python.exe -c "import sys; exit(1 if sys.version_info >= (3,6) else 0)" || \
 		.venv\Scripts\black.exe \
-		--exclude="build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|src/rebar/pr2relnotes.py|src/fauxton" \
-		build-aux dev\run src\mango\test src\docs\src\conf.py src\docs\ext .
+		--exclude="build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|src/erlfmt|src/rebar/pr2relnotes.py|src/fauxton" \
+		build-aux dev\run dev\format_*.py src\mango\test src\docs\src\conf.py src\docs\ext .
 
 .PHONY: elixir
 elixir: export MIX_ENV=integration
diff --git a/README-DEV.rst b/README-DEV.rst
index c8a52ffd4..863218de9 100644
--- a/README-DEV.rst
+++ b/README-DEV.rst
@@ -140,6 +140,20 @@ ignore their build and avoid any issues with their dependencies.
 
 See ``./configure --help`` for more information.
 
+Developing
+----------
+
+Formatting
+~~~~~~~~~~
+
+The ``erl`` files in ``src`` are formatted using erlfmt_. The checks are run
+for every PR in the CI. To run the checks locally, run ``make erlfmt-check``.
+To format the ``erl`` files in ``src``, run ``make erlfmt-format``.
+To use ``erlfmt`` for specific files only, use the executable ``bin/erlfmt``
+that is installed by ``configure``.
+
+.. _erlfmt: https://github.com/WhatsApp/erlfmt
+
 Testing
 -------
 
diff --git a/build-aux/Jenkinsfile.full b/build-aux/Jenkinsfile.full
index fdb2f4c6a..9fc131ff7 100644
--- a/build-aux/Jenkinsfile.full
+++ b/build-aux/Jenkinsfile.full
@@ -95,6 +95,7 @@ pipeline {
           set
           rm -rf apache-couchdb-*
           ./configure
+          make erlfmt-check
           make dist
           chmod -R a+w * .
         '''
diff --git a/build-aux/Jenkinsfile.pr b/build-aux/Jenkinsfile.pr
index 8f9d6f1d7..61ae43bba 100644
--- a/build-aux/Jenkinsfile.pr
+++ b/build-aux/Jenkinsfile.pr
@@ -82,6 +82,7 @@ pipeline {
           rm -rf apache-couchdb-*
           . /usr/local/kerl/${LOW_ERLANG_VER}/activate
           ./configure
+          make erlfmt-check
           make dist
           chmod -R a+w * .
         '''
diff --git a/configure b/configure
index a8d8bdb2d..6b0f1fe40 100755
--- a/configure
+++ b/configure
@@ -21,6 +21,8 @@ basename=`basename $0`
 
 PACKAGE_AUTHOR_NAME="The Apache Software Foundation"
 
+REBAR3_BRANCH="master"
+
 # TEST=0
 WITH_PROPER="true"
 WITH_FAUXTON=1
@@ -31,6 +33,7 @@ SKIP_DEPS=0
 COUCHDB_USER="$(whoami 2>/dev/null || echo couchdb)"
 SM_VSN=${SM_VSN:-"1.8.5"}
 ARCH="$(uname -m)"
+ERLANG_VER="$(erl -eval 'io:put_chars(erlang:system_info(otp_release)), halt().' -noshell)"
 
 . ${rootdir}/version.mk
 COUCHDB_VERSION=${vsn_major}.${vsn_minor}.${vsn_patch}
@@ -54,6 +57,8 @@ Options:
   --skip-deps                 do not update erlang dependencies
   --rebar=PATH                use rebar by specified path (version >=2.6.0 && <3.0 required)
   --generate-tls-dev-cert     generate a cert for TLS distribution (To enable TLS, change the vm.args file.)
+  --rebar3=PATH               use rebar3 by specified path
+  --erlfmt=PATH               use erlfmt by specified path
 EOF
 }
 
@@ -137,6 +142,28 @@ parse_opts() {
                 fi
                 ;;
 
+            --rebar3)
+                if [ -x "$2" ]; then
+                    eval REBAR3=$2
+                    shift 2
+                    continue
+                else
+                    printf 'ERROR: "--rebar3" requires valid path to executable.\n' >&2
+                    exit 1
+                fi
+                ;;
+
+            --erlfmt)
+                if [ -x "$2" ]; then
+                    eval ERLFMT=$2
+                    shift 2
+                    continue
+                else
+                    printf 'ERROR: "--erlfmt" requires valid path to executable.\n' >&2
+                    exit 1
+                fi
+                ;;
+
             --user|-u)
                 if [ -n "$2" ]; then
                     eval COUCHDB_USER=$2
@@ -275,12 +302,46 @@ install_local_rebar() {
     fi
 }
 
+install_local_rebar3() {
+    if [ ! -x "${rootdir}/bin/rebar3" ]; then
+        if [ ! -d "${rootdir}/src/rebar3" ]; then
+            git clone --depth 1 --branch ${REBAR3_BRANCH} https://github.com/erlang/rebar3.git ${rootdir}/src/rebar3
+        fi
+        cd src/rebar3
+        ./bootstrap
+        mv ${rootdir}/src/rebar3/rebar3 ${rootdir}/bin/rebar3
+        cd ../..
+    fi
+}
+
+install_local_erlfmt() {
+    if [ ! -x "${rootdir}/bin/erlfmt" ]; then
+        if [ ! -d "${rootdir}/src/erlfmt" ]; then
+            git clone --depth 1 https://github.com/WhatsApp/erlfmt.git ${rootdir}/src/erlfmt
+        fi
+        cd "${rootdir}"/src/erlfmt
+        ${REBAR3} as release escriptize
+        mv ${rootdir}/src/erlfmt/_build/release/bin/erlfmt ${rootdir}/bin/erlfmt
+        ${REBAR3} clean
+        cd ../..
+    fi
+}
 
 if [ -z "${REBAR}" ]; then
     install_local_rebar
     REBAR=${rootdir}/bin/rebar
 fi
 
+if [ -z "${REBAR3}" ] && [ "${ERLANG_VER}" != "20" ]; then
+    install_local_rebar3
+    REBAR3=${rootdir}/bin/rebar3
+fi
+
+if [ -z "${ERLFMT}" ] && [ "${ERLANG_VER}" != "20" ]; then
+    install_local_erlfmt
+    ERLFMT=${rootdir}/bin/erlfmt
+fi
+
 # only update dependencies, when we are not in a release tarball
 if [ -d .git  -a $SKIP_DEPS -ne 1 ]; then
     echo "==> updating dependencies"
diff --git a/dev/format_all.py b/dev/format_all.py
new file mode 100644
index 000000000..1927fb59e
--- /dev/null
+++ b/dev/format_all.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+"""Erlang formatter for CouchDB
+Warning: this file needs to run from the CouchDB repo root.
+USAGE: ERLFMT_PATH=<path_to_erlfmt> python3 dev/format_all.py
+"""
+
+import os
+import sys
+import subprocess
+
+from format_lib import get_source_paths, get_erlang_version
+
+if __name__ == "__main__":
+    if get_erlang_version() < 21:
+        print("Erlang version is < 21. Skipping format check")
+        sys.exit(0)
+
+    for path in get_source_paths():
+        subprocess.run(
+            [os.environ["ERLFMT_PATH"], "-w", path],
+            stdout=subprocess.PIPE,
+        )
diff --git a/dev/format_check.py b/dev/format_check.py
new file mode 100644
index 000000000..cbb0126d9
--- /dev/null
+++ b/dev/format_check.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+"""Erlang formatter for CouchDB
+Warning: this file needs to run from the CouchDB repo root.
+USAGE: ERLFMT_PATH=<path_to_erlfmt> python3 dev/format_check.py
+"""
+
+import os
+import subprocess
+import sys
+
+from format_lib import get_source_paths, get_erlang_version
+
+
+if __name__ == "__main__":
+    if get_erlang_version() < 21:
+        print("Erlang version is < 21. Skipping format check")
+        sys.exit(0)
+
+    exit_code = 0
+
+    for path in get_source_paths():
+        run_result = subprocess.run(
+            [os.environ["ERLFMT_PATH"], "-c", path],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        rc = run_result.returncode
+        if rc != 0:
+            print("\n %s error for %s" % (rc, path))
+            stderr_lines = run_result.stderr.decode("utf-8").split("\n")
+            for line in stderr_lines:
+                print("  > %s" % line, file=sys.stderr)
+            exit_code = 1
+
+    sys.exit(exit_code)
diff --git a/dev/format_lib.py b/dev/format_lib.py
new file mode 100644
index 000000000..3db0057fc
--- /dev/null
+++ b/dev/format_lib.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+"""Erlang formatter lib for CouchDB
+Warning: this file is not meant to be executed manually
+"""
+
+import pathlib
+import subprocess
+
+
+def get_erlang_version():
+    args = [
+        "erl",
+        "-eval",
+        "io:put_chars(erlang:system_info(otp_release)), halt().",
+        "-noshell",
+    ]
+    res = subprocess.run(args, stdout=subprocess.PIPE, check=True)
+    str_version = res.stdout.decode("utf-8").strip().strip('"')
+    return int(str_version)
+
+
+# Generate source paths as "directory/*.erl" wildcard patterns
+# those can be directly consumed by erlfmt and processed in parallel
+#
+def get_source_paths():
+    curdir = None
+    for item in (
+        subprocess.run(
+            ["git", "ls-files", "--", "*.erl"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        .stdout.decode("utf-8")
+        .split("\n")
+    ):
+        path = pathlib.Path(item)
+        if path.parent != curdir:
+            yield str(path.parent.joinpath("*.erl"))
+            curdir = path.parent
+    if curdir is not None:
+        yield str(curdir.joinpath("*.erl"))