Merge pull request #1944 from cloudant/integrate-emilio

Integrate emilio

7 files changed, 183 insertions, 2 deletions

diff --git a/.gitignore b/.gitignore
index 60e6d145a..3cfa3721e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -45,6 +45,7 @@ src/couch/priv/couch_js/**/*.d
 src/couch/priv/icu_driver/couch_icu_driver.d
 src/mango/src/mango_cursor_text.nocompile
 src/docs/
+src/emilio/
 src/ets_lru/
 src/excoveralls/
 src/fauxton/
diff --git a/Makefile b/Makefile
index 97fc97c85..fff1df528 100644
--- a/Makefile
+++ b/Makefile
@@ -147,6 +147,7 @@ fauxton: share/www
 .PHONY: check
 # target: check - Test everything
 check: all python-black
+	@$(MAKE) emilio
 	@$(MAKE) eunit
 	@$(MAKE) javascript
 	@$(MAKE) mango-test
@@ -198,6 +199,9 @@ soak-eunit: couch
 	@$(REBAR) setup_eunit 2> /dev/null
 	while [ $$? -eq 0 ] ; do $(REBAR) -r eunit $(EUNIT_OPTS) ; done
 
+emilio:
+	@bin/emilio -c emilio.config src/ | bin/warnings_in_scope -s 3
+
 .venv/bin/black:
 	@python3 -m venv .venv
 	@.venv/bin/pip3 install black || touch .venv/bin/black
@@ -260,7 +264,7 @@ elixir-credo: elixir-init
 .PHONY: javascript
 # target: javascript - Run JavaScript test suites or specific ones defined by suites option
 javascript: export COUCHDB_TEST_ADMIN_PARTY_OVERRIDE=1
-javascript: 
+javascript:
 
 	@$(MAKE) devclean
 	@mkdir -p share/www/script/test
diff --git a/Makefile.win b/Makefile.win
index bdecc7315..0fc4d91c7 100644
--- a/Makefile.win
+++ b/Makefile.win
@@ -134,6 +134,7 @@ fauxton: share\www
 .PHONY: check
 # target: check - Test everything
 check: all python-black
+	@$(MAKE) emilio
 	@$(MAKE) eunit
 	@$(MAKE) javascript
 	@$(MAKE) mango-test
@@ -175,6 +176,9 @@ just-eunit: export ERL_AFLAGS = "-config $(shell echo %cd%)/rel/files/eunit.conf
 just-eunit:
 	@$(REBAR) -r eunit $(EUNIT_OPTS)
 
+emilio:
+	@bin\emilio -c emilio.config src\ | python.exe bin\warnings_in_scope -s 3
+
 .venv/bin/black:
 	@python.exe -m venv .venv
 	@.venv\Scripts\pip3.exe install black || copy /b .venv\Scripts\black.exe +,,
@@ -359,7 +363,7 @@ install: release
 	@echo .
 	@echo     To install CouchDB into your system, copy the rel\couchdb
 	@echo     to your desired installation location. For example:
-	@echo     xcopy /E rel\couchdb C:\CouchDB\ 
+	@echo     xcopy /E rel\couchdb C:\CouchDB\
 	@echo .
 
 ################################################################################
diff --git a/bin/warnings_in_scope b/bin/warnings_in_scope
new file mode 100755
index 000000000..2a854211a
--- /dev/null
+++ b/bin/warnings_in_scope
@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+import os
+import subprocess
+from pathlib import Path
+import optparse
+import sys
+import re
+
+def run(command, cwd=None):
+    try:
+        return subprocess.Popen(
+            command, shell=True, cwd=cwd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
+    except OSError as err:
+        raise OSError("Error in command '{0}': {1}".format(command, err))
+
+def parse_location(line):
+    # take substring between @@
+    # take second part of it
+    location = line.split(b'@@')[1].strip().split(b' ')[1]
+    tokens = location.split(b',')
+    if len(tokens) == 1:
+        return (int(tokens[0][1:]), 1)
+    elif len(tokens) == 2:
+        return (int(tokens[0][1:]), int(tokens[1]))
+
+def changed_files(directory, scope):
+    result = {}
+    proc = run('git diff --no-prefix --unified={0}'.format(scope), cwd=str(directory))
+    file_path = None
+    for line in iter(proc.stdout.readline, b''):
+        if line.startswith(b'diff --git '):
+            # this would be problematic if directory has space in the name
+            file_name = line.split(b' ')[3].strip()
+            file_path = str(directory.joinpath(str(file_name, 'utf-8')))
+            result[file_path] = set()
+            continue
+        if line.startswith(b'@@'):
+            start_pos, number_of_lines = parse_location(line)
+            for line_number in range(start_pos, start_pos + number_of_lines):
+                result[file_path].add(line_number)
+    return result
+
+def print_changed(file_name, line_number):
+    print('{0}:{1}'.format(str(file_name), str(line_number)))
+
+def changes(dirs, scope):
+    result = {}
+    for directory in dirs:
+        result.update(changed_files(directory, scope))
+    return result
+
+def repositories(root):
+    for directory in Path(root).rglob('.git'):
+        if not directory.is_dir():
+            continue
+        yield directory.parent
+
+def setup_argparse():
+    parser = optparse.OptionParser(description="Filter output to remove unrelated warning")
+    parser.add_option(
+        "-r",
+        "--regexp",
+        dest="regexp",
+        default='(?P<file_name>[^:]+):(?P<line>\d+).*',
+        help="Regexp used to extract file_name and line number",
+    )
+    parser.add_option(
+        "-s",
+        "--scope",
+        dest="scope",
+        default=0,
+        help="Number of lines surrounding the change we consider relevant",
+    )
+    parser.add_option(
+        "-p",
+        "--print-only",
+        action="store_true",
+        dest="print_only",
+        default=False,
+        help="Print changed lines only",
+    )
+    return parser.parse_args()
+
+def filter_stdin(regexp, changes):
+    any_matches = False
+    for line in iter(sys.stdin.readline, ''):
+        matches = re.match(regexp, line)
+        if matches:
+            file_name = matches.group('file_name')
+            line_number = int(matches.group('line'))
+            if file_name in changes and line_number in changes[file_name]:
+                print(line, end='')
+                any_matches = True
+    return any_matches
+
+def validate_regexp(regexp):
+    index = regexp.groupindex
+    if 'file_name' in index and 'line' in index:
+        return True
+    else:
+        raise TypeError("Regexp must define following groups:\n  - file_name\n  - line")
+
+def main():
+    opts, args = setup_argparse()
+    if opts.print_only:
+        for file_name, changed_lines in changes(repositories('.'), opts.scope).items():
+            for line_number in changed_lines:
+                print_changed(file_name, line_number)
+        return 0
+    else:
+        regexp = re.compile(opts.regexp)
+        validate_regexp(regexp)
+        if filter_stdin(regexp, changes(repositories('.'), opts.scope)):
+            return 1
+        else:
+            return 0
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except KeyboardInterrupt:
+        pass
+
diff --git a/configure b/configure
index 38e62e317..854366c8a 100755
--- a/configure
+++ b/configure
@@ -255,12 +255,25 @@ install_local_rebar() {
     fi
 }
 
+install_local_emilio() {
+    if [ ! -x "${rootdir}/bin/emilio" ]; then
+        if [ ! -d "${rootdir}/src/emilio" ]; then
+            git clone --depth 1 https://github.com/cloudant-labs/emilio ${rootdir}/src/emilio
+        fi
+        cd ${rootdir}/src/emilio && ${REBAR} compile escriptize; cd ${rootdir}
+        mv ${rootdir}/src/emilio/emilio ${rootdir}/bin/emilio
+        chmod +x ${rootdir}/bin/emilio
+        cd ${rootdir}/src/emilio && ${REBAR} clean; cd ${rootdir}
+    fi
+}
 
 if [ -z "${REBAR}" ]; then
     install_local_rebar
     REBAR=${rootdir}/bin/rebar
 fi
 
+install_local_emilio
+
 # only update dependencies, when we are not in a release tarball
 if [ -d .git  -a $SKIP_DEPS -ne 1 ]; then
     echo "==> updating dependencies"
diff --git a/configure.ps1 b/configure.ps1
index c74fbcf41..65f8517d6 100644
--- a/configure.ps1
+++ b/configure.ps1
@@ -205,6 +205,20 @@ if ((Get-Command "rebar.cmd" -ErrorAction SilentlyContinue) -eq $null)
    $env:Path += ";$rootdir\bin"
 }
 
+# check for emilio; if not found, get it and build it
+if ((Get-Command "emilio.cmd" -ErrorAction SilentlyContinue) -eq $null)
+{
+   Write-Verbose "==> emilio.cmd not found; bootstrapping..."
+   if (-Not (Test-Path "src\emilio"))
+   {
+      git clone --depth 1 https://github.com/wohali/emilio $rootdir\src\emilio
+   }
+   cmd /c "cd $rootdir\src\emilio && rebar compile escriptize; cd $rootdir"
+   cp $rootdir\src\emilio\emilio $rootdir\bin\emilio
+   cp $rootdir\src\emilio\bin\emilio.cmd $rootdir\bin\emilio.cmd
+   cmd /c "cd $rootdir\src\emilio && rebar clean; cd $rootdir"
+}
+
 # only update dependencies, when we are not in a release tarball
 if ( (Test-Path .git -PathType Container) -and (-not $SkipDeps) ) {
     Write-Verbose "==> updating dependencies"
diff --git a/emilio.config b/emilio.config
new file mode 100644
index 000000000..0dad93898
--- /dev/null
+++ b/emilio.config
@@ -0,0 +1,20 @@
+{ignore, [
+    "src[\/]bear[\/]*",
+    "src[\/]b64url[\/]*",
+    "src[\/]docs[\/]*",
+    "src[\/]*[\/].eunit[\/]*",
+    "src[\/]fauxton[\/]*",
+    "src[\/]rebar[\/]*",
+    "src[\/]emilio[\/]*",
+    "src[\/]folsom[\/]*",
+    "src[\/]mochiweb[\/]*",
+    "src[\/]snappy[\/]*",
+    "src[\/]ssl_verify_fun[\/]*",
+    "src[\/]ibrowse[\/]*",
+    "src[\/]jiffy[\/]*",
+    "src[\/]meck[\/]*",
+    "src[\/]proper[\/]*",
+    "src[\/]recon[\/]*",
+    "src[\/]hyper[\/]*",
+    "src[\/]triq[\/]*"
+]}.