diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | Makefile | 11 | ||||
| -rw-r--r-- | Makefile.win | 15 | ||||
| -rw-r--r-- | README-DEV.rst | 14 | ||||
| -rw-r--r-- | build-aux/Jenkinsfile.full | 1 | ||||
| -rw-r--r-- | build-aux/Jenkinsfile.pr | 1 | ||||
| -rwxr-xr-x | configure | 61 | ||||
| -rw-r--r-- | dev/format_all.py | 35 | ||||
| -rw-r--r-- | dev/format_check.py | 48 | ||||
| -rw-r--r-- | dev/format_lib.py | 54 |
10 files changed, 236 insertions, 6 deletions
diff --git a/.gitignore b/.gitignore index 3d49a336e..fe3e5acaa 100644 --- a/.gitignore +++ b/.gitignore @@ -80,6 +80,8 @@ src/ssl_verify_fun/ src/triq/ src/unicode_util_compat/ src/file_system/ +src/rebar3/ +src/erlfmt/ tmp/ src/couch/*.o @@ -17,6 +17,7 @@ include version.mk REBAR?=$(shell echo `pwd`/bin/rebar) +ERLFMT?=$(shell echo `pwd`/bin/erlfmt) # Handle the following scenarios: # 1. When building from a tarball, use version.mk. @@ -202,6 +203,12 @@ soak-eunit: couch @$(REBAR) setup_eunit 2> /dev/null while [ $$? -eq 0 ] ; do $(REBAR) -r eunit $(EUNIT_OPTS) ; done +erlfmt-check: + ERLFMT_PATH=$(ERLFMT) python3 dev/format_check.py + +erlfmt-format: + ERLFMT_PATH=$(ERLFMT) python3 dev/format_all.py + .venv/bin/black: @python3 -m venv .venv @.venv/bin/pip3 install black || touch .venv/bin/black @@ -212,8 +219,8 @@ python-black: .venv/bin/black echo "Python formatter not supported on Python < 3.6; check results on a newer platform" @python3 -c "import sys; exit(1 if sys.version_info >= (3,6) else 0)" || \ LC_ALL=C.UTF-8 LANG=C.UTF-8 .venv/bin/black --check \ - --exclude="build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|src/rebar/pr2relnotes.py|src/fauxton" \ - build-aux/*.py dev/run src/mango/test/*.py src/docs/src/conf.py src/docs/ext/*.py . + --exclude="build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|src/erlfmt|src/rebar/pr2relnotes.py|src/fauxton" \ + build-aux/*.py dev/run dev/format_*.py src/mango/test/*.py src/docs/src/conf.py src/docs/ext/*.py . python-black-update: .venv/bin/black @python3 -c "import sys; exit(1 if sys.version_info < (3,6) else 0)" || \ diff --git a/Makefile.win b/Makefile.win index 25c4d9fa7..62a0a8dec 100644 --- a/Makefile.win +++ b/Makefile.win @@ -18,6 +18,7 @@ include version.mk SHELL=cmd.exe REBAR=bin\rebar.cmd +ERLFMT=bin\erlfmt MAKE=make -f Makefile.win # REBAR?=$(shell where rebar.cmd) @@ -171,6 +172,12 @@ just-eunit: export ERL_AFLAGS = "-config $(shell echo %cd%)/rel/files/eunit.conf just-eunit: @$(REBAR) -r eunit $(EUNIT_OPTS) +erlfmt-check: + ERLFMT_PATH=bin\erlfmt python3 dev\format_check.py + +erlfmt-format: + ERLFMT_PATH=bin\erlfmt python3 dev\format_all.py + .venv/bin/black: @python.exe -m venv .venv @.venv\Scripts\pip3.exe install black || copy /b .venv\Scripts\black.exe +,, @@ -181,16 +188,16 @@ python-black: .venv/bin/black echo 'Python formatter not supported on Python < 3.6; check results on a newer platform' @python.exe -c "import sys; exit(1 if sys.version_info >= (3,6) else 0)" || \ .venv\Scripts\black.exe --check \ - --exclude="build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|src/rebar/pr2relnotes.py|src/fauxton" \ - build-aux dev\run src\mango\test src\docs\src\conf.py src\docs\ext . + --exclude="build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|src/erlfmt|src/rebar/pr2relnotes.py|src/fauxton" \ + build-aux dev\run dev\format_*.py src\mango\test src\docs\src\conf.py src\docs\ext . python-black-update: .venv/bin/black @python.exe -c "import sys; exit(1 if sys.version_info < (3,6) else 0)" || \ echo 'Python formatter not supported on Python < 3.6; check results on a newer platform' @python.exe -c "import sys; exit(1 if sys.version_info >= (3,6) else 0)" || \ .venv\Scripts\black.exe \ - --exclude="build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|src/rebar/pr2relnotes.py|src/fauxton" \ - build-aux dev\run src\mango\test src\docs\src\conf.py src\docs\ext . + --exclude="build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|src/erlfmt|src/rebar/pr2relnotes.py|src/fauxton" \ + build-aux dev\run dev\format_*.py src\mango\test src\docs\src\conf.py src\docs\ext . .PHONY: elixir elixir: export MIX_ENV=integration diff --git a/README-DEV.rst b/README-DEV.rst index c8a52ffd4..863218de9 100644 --- a/README-DEV.rst +++ b/README-DEV.rst @@ -140,6 +140,20 @@ ignore their build and avoid any issues with their dependencies. See ``./configure --help`` for more information. +Developing +---------- + +Formatting +~~~~~~~~~~ + +The ``erl`` files in ``src`` are formatted using erlfmt_. The checks are run +for every PR in the CI. To run the checks locally, run ``make erlfmt-check``. +To format the ``erl`` files in ``src``, run ``make erlfmt-format``. +To use ``erlfmt`` for specific files only, use the executable ``bin/erlfmt`` +that is installed by ``configure``. + +.. _erlfmt: https://github.com/WhatsApp/erlfmt + Testing ------- diff --git a/build-aux/Jenkinsfile.full b/build-aux/Jenkinsfile.full index fdb2f4c6a..9fc131ff7 100644 --- a/build-aux/Jenkinsfile.full +++ b/build-aux/Jenkinsfile.full @@ -95,6 +95,7 @@ pipeline { set rm -rf apache-couchdb-* ./configure + make erlfmt-check make dist chmod -R a+w * . ''' diff --git a/build-aux/Jenkinsfile.pr b/build-aux/Jenkinsfile.pr index 8f9d6f1d7..61ae43bba 100644 --- a/build-aux/Jenkinsfile.pr +++ b/build-aux/Jenkinsfile.pr @@ -82,6 +82,7 @@ pipeline { rm -rf apache-couchdb-* . /usr/local/kerl/${LOW_ERLANG_VER}/activate ./configure + make erlfmt-check make dist chmod -R a+w * . ''' @@ -21,6 +21,8 @@ basename=`basename $0` PACKAGE_AUTHOR_NAME="The Apache Software Foundation" +REBAR3_BRANCH="master" + # TEST=0 WITH_PROPER="true" WITH_FAUXTON=1 @@ -31,6 +33,7 @@ SKIP_DEPS=0 COUCHDB_USER="$(whoami 2>/dev/null || echo couchdb)" SM_VSN=${SM_VSN:-"1.8.5"} ARCH="$(uname -m)" +ERLANG_VER="$(erl -eval 'io:put_chars(erlang:system_info(otp_release)), halt().' -noshell)" . ${rootdir}/version.mk COUCHDB_VERSION=${vsn_major}.${vsn_minor}.${vsn_patch} @@ -54,6 +57,8 @@ Options: --skip-deps do not update erlang dependencies --rebar=PATH use rebar by specified path (version >=2.6.0 && <3.0 required) --generate-tls-dev-cert generate a cert for TLS distribution (To enable TLS, change the vm.args file.) + --rebar3=PATH use rebar3 by specified path + --erlfmt=PATH use erlfmt by specified path EOF } @@ -137,6 +142,28 @@ parse_opts() { fi ;; + --rebar3) + if [ -x "$2" ]; then + eval REBAR3=$2 + shift 2 + continue + else + printf 'ERROR: "--rebar3" requires valid path to executable.\n' >&2 + exit 1 + fi + ;; + + --erlfmt) + if [ -x "$2" ]; then + eval ERLFMT=$2 + shift 2 + continue + else + printf 'ERROR: "--erlfmt" requires valid path to executable.\n' >&2 + exit 1 + fi + ;; + --user|-u) if [ -n "$2" ]; then eval COUCHDB_USER=$2 @@ -275,12 +302,46 @@ install_local_rebar() { fi } +install_local_rebar3() { + if [ ! -x "${rootdir}/bin/rebar3" ]; then + if [ ! -d "${rootdir}/src/rebar3" ]; then + git clone --depth 1 --branch ${REBAR3_BRANCH} https://github.com/erlang/rebar3.git ${rootdir}/src/rebar3 + fi + cd src/rebar3 + ./bootstrap + mv ${rootdir}/src/rebar3/rebar3 ${rootdir}/bin/rebar3 + cd ../.. + fi +} + +install_local_erlfmt() { + if [ ! -x "${rootdir}/bin/erlfmt" ]; then + if [ ! -d "${rootdir}/src/erlfmt" ]; then + git clone --depth 1 https://github.com/WhatsApp/erlfmt.git ${rootdir}/src/erlfmt + fi + cd "${rootdir}"/src/erlfmt + ${REBAR3} as release escriptize + mv ${rootdir}/src/erlfmt/_build/release/bin/erlfmt ${rootdir}/bin/erlfmt + ${REBAR3} clean + cd ../.. + fi +} if [ -z "${REBAR}" ]; then install_local_rebar REBAR=${rootdir}/bin/rebar fi +if [ -z "${REBAR3}" ] && [ "${ERLANG_VER}" != "20" ]; then + install_local_rebar3 + REBAR3=${rootdir}/bin/rebar3 +fi + +if [ -z "${ERLFMT}" ] && [ "${ERLANG_VER}" != "20" ]; then + install_local_erlfmt + ERLFMT=${rootdir}/bin/erlfmt +fi + # only update dependencies, when we are not in a release tarball if [ -d .git -a $SKIP_DEPS -ne 1 ]; then echo "==> updating dependencies" diff --git a/dev/format_all.py b/dev/format_all.py new file mode 100644 index 000000000..1927fb59e --- /dev/null +++ b/dev/format_all.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +"""Erlang formatter for CouchDB +Warning: this file needs to run from the CouchDB repo root. +USAGE: ERLFMT_PATH=<path_to_erlfmt> python3 dev/format_all.py +""" + +import os +import sys +import subprocess + +from format_lib import get_source_paths, get_erlang_version + +if __name__ == "__main__": + if get_erlang_version() < 21: + print("Erlang version is < 21. Skipping format check") + sys.exit(0) + + for path in get_source_paths(): + subprocess.run( + [os.environ["ERLFMT_PATH"], "-w", path], + stdout=subprocess.PIPE, + ) diff --git a/dev/format_check.py b/dev/format_check.py new file mode 100644 index 000000000..cbb0126d9 --- /dev/null +++ b/dev/format_check.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +"""Erlang formatter for CouchDB +Warning: this file needs to run from the CouchDB repo root. +USAGE: ERLFMT_PATH=<path_to_erlfmt> python3 dev/format_check.py +""" + +import os +import subprocess +import sys + +from format_lib import get_source_paths, get_erlang_version + + +if __name__ == "__main__": + if get_erlang_version() < 21: + print("Erlang version is < 21. Skipping format check") + sys.exit(0) + + exit_code = 0 + + for path in get_source_paths(): + run_result = subprocess.run( + [os.environ["ERLFMT_PATH"], "-c", path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + rc = run_result.returncode + if rc != 0: + print("\n %s error for %s" % (rc, path)) + stderr_lines = run_result.stderr.decode("utf-8").split("\n") + for line in stderr_lines: + print(" > %s" % line, file=sys.stderr) + exit_code = 1 + + sys.exit(exit_code) diff --git a/dev/format_lib.py b/dev/format_lib.py new file mode 100644 index 000000000..3db0057fc --- /dev/null +++ b/dev/format_lib.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +"""Erlang formatter lib for CouchDB +Warning: this file is not meant to be executed manually +""" + +import pathlib +import subprocess + + +def get_erlang_version(): + args = [ + "erl", + "-eval", + "io:put_chars(erlang:system_info(otp_release)), halt().", + "-noshell", + ] + res = subprocess.run(args, stdout=subprocess.PIPE, check=True) + str_version = res.stdout.decode("utf-8").strip().strip('"') + return int(str_version) + + +# Generate source paths as "directory/*.erl" wildcard patterns +# those can be directly consumed by erlfmt and processed in parallel +# +def get_source_paths(): + curdir = None + for item in ( + subprocess.run( + ["git", "ls-files", "--", "*.erl"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + .stdout.decode("utf-8") + .split("\n") + ): + path = pathlib.Path(item) + if path.parent != curdir: + yield str(path.parent.joinpath("*.erl")) + curdir = path.parent + if curdir is not None: + yield str(curdir.joinpath("*.erl")) |