Add auto log uploader for CI workflowupload-failed-logfiles

This commit is intended to improve failure analysis in Travis and
Jenkins runs by uploading couch log files after failed test runs.

A new script has been added that, upon invocation from the top-level
directory, tars up all EUnit logfiles and the dev/logs/node1.log file.
It then constructs a document based on the running environment (Travis,
Jenkins, manual). The document is posted to a CouchDB instance running
on couchdb-vm.apache.org with credentials passed in the COUCHAUTH
environment variable.

The .travis.yml file has been updated to install the script's
prerequisites (python-requests) and with the secret credentials, and to
instruct travis to run the log uploader after any test failure. Similar
steps will be taken to reconfigure Jenkins after these changes are
merged to master.

The test/javascript/run harness had to be modified to terminate after
the first failure. This is because each new test wipes the logfile of
the previous run. All tests can still be run, regardless of failure, by
specifing the -a/--all flag, however it is expected that the reduction
in test suite execution time when a test fails will be appreciated by
many.

4 files changed, 147 insertions, 5 deletions

diff --git a/.gitignore b/.gitignore
index e3601f7c7..f6e0c37b3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,8 @@ log
 apache-couchdb-*/
 bin/
 config.erl
+*.tar.gz
+*.tar.bz2
 dev/boot_node.beam
 dev/lib/
 dev/logs/
@@ -106,3 +108,13 @@ src/mango/ebin/
 src/mango/test/*.pyc
 src/mango/venv/
 src/mango/.eunit
+
+src/chttpd/.eunit/
+src/couch_event/.eunit/
+src/couch_mrview/.eunit/
+src/couch_plugins/.eunit/
+src/couch_stats/.eunit/
+src/ddoc_cache/.eunit/
+src/fabric/.eunit/
+src/mem3/.eunit/
+src/rexi/.eunit/
diff --git a/.travis.yml b/.travis.yml
index 580ac6c39..156856cc7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,19 +17,29 @@ addons:
     - libicu-dev
     - libmozjs185-dev
     - pkg-config
+    - python-requests
+    - python-sphinx
     - help2man
     - shunit2
 
+python:
+  - "2.7"
+
 git:
   depth: 10
 
+# logfile uploader uses requests
+cache:
+  - pip
+
+# logfile uploader credentials
+env:
+  global:
+    - secure: "UdA/gKIlyuXaW+hUgRx40t1TYjLCGxMqHvM5Uw7UbUH2dqEkgJiLfhZGchS1JVzl8M01VKZUUzS7v2nvRLiHZN1kvaw5kfq31VRoafUah8jfmvqNWZVdLovHl3aw5UX/HRt0RkbWbhdbdknTfh6+YinSZ+Nb54jCErMg9nabXtM="
+
 # Enable this block if you want to build docs & fauxton too
-#cache:
-#  - pip
 #node_js:
 #  - 6
-#install:
-#  - pip install sphinx
 #before_script:
 #  - ./configure -c
 
@@ -40,6 +50,9 @@ before_script:
 script:
    - make check
 
+after_failure:
+  - build-aux/logfile-uploader.py
+
 # Re-enable once test suite is reliable
 #notifications:
 #  email: false
diff --git a/build-aux/logfile-uploader.py b/build-aux/logfile-uploader.py
new file mode 100755
index 000000000..9db4142dc
--- /dev/null
+++ b/build-aux/logfile-uploader.py
@@ -0,0 +1,113 @@
+#!/usr/bin/env python
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+from __future__ import print_function
+
+import datetime
+import glob
+import json
+import os
+import tarfile
+import time
+
+import requests
+
+COUCH_URL = "http://couchdb-vm.apache.org:5984/ci_errorlogs"
+TARFILE = "couchlog.tar.gz"
+
+def _tojson(req):
+    """Support requests v0.x as well as 1.x+"""
+    if requests.__version__[0] == '0':
+        return json.loads(req.content)
+    return req.json()
+
+def collect_logfiles():
+    """ Find and tarball all logfiles """
+    tb = tarfile.open(name=TARFILE, mode='w:gz')
+    # EUnit
+    for log in glob.glob('src/*/.eunit/couch.log'):
+        tb.add(log)
+    # JS harness
+    for log in glob.glob('dev/logs/node1.log'):
+        tb.add(log)
+    tb.close()
+
+def build_ci_doc():
+    """ Build a metadata document with relevant detail from CI env """
+    doc = {}
+    if 'TRAVIS' in os.environ:
+        doc['builder'] = 'travis'
+        doc['build_id'] = os.environ['TRAVIS_JOB_ID']
+        doc['erlang'] = os.environ['TRAVIS_OTP_RELEASE']
+        doc['url'] = 'https://travis-ci.org/apache/couchdb/jobs/' + \
+            os.environ['TRAVIS_JOB_ID']
+        doc['branch'] = os.environ['TRAVIS_BRANCH']
+        doc['commit'] = os.environ['TRAVIS_COMMIT']
+        doc['repo'] = 'https://github.com/' + os.environ['TRAVIS_REPO_SLUG']
+    elif 'JENKINS_URL' in os.environ:
+        doc['builder'] = 'jenkins'
+        doc['build_id'] = os.environ['BUILD_NUMBER']
+        doc['url'] = os.environ['BUILD_URL']
+        doc['branch'] = os.environ['GIT_BRANCH']
+        doc['commit'] = os.environ['GIT_COMMIT']
+        doc['repo'] = os.environ['GIT_URL']
+    else:
+        doc['builder'] = 'manual'
+        # TODO: shell out to get correct repo, commit, branch info?
+        doc['repo'] = 'https://github.com/apache/couchdb'
+        doc['build_id'] = str(time.time())
+
+    # shorten doc id
+    repo = doc['repo'].split('/')[-1]
+    repo = repo.replace('.git', '')
+
+    doc['_id'] = doc['builder'] + '-' + repo + '-' + \
+        doc['build_id'] + \
+        '-' + datetime.datetime.utcnow().isoformat()
+
+    return doc
+
+def upload_logs():
+    lp = os.environ['COUCHAUTH'].split(':')
+    creds = (lp[0], lp[1])
+    doc = build_ci_doc()
+    req = requests.post(COUCH_URL,
+        data=json.dumps(doc),
+        auth=creds,
+        headers={'Content-type': 'application/json'})
+    req.raise_for_status()
+    req = _tojson(req)
+    with open(TARFILE, 'rb') as f:
+        # ancient versions of requests break if data is iterable
+        fdata = f.read()
+        req2 = requests.put(COUCH_URL + '/' + doc['_id'] + '/' + TARFILE,
+            headers={'Content-type': 'application/x-gtar'},
+            auth=creds,
+            params={'rev': req['rev']},
+            data=fdata)
+    req2.raise_for_status()
+    return req2
+
+
+def main():
+    """ Find latest logfile and upload to Couch logfile db. """
+    print ("Uploading logfiles...")
+    collect_logfiles()
+    req = upload_logs()
+    print (req.url)
+    print (req.content)
+    print ("Upload complete!")
+
+if __name__ == '__main__':
+    main()
diff --git a/test/javascript/run b/test/javascript/run
index 10d00d83b..7f366ebed 100755
--- a/test/javascript/run
+++ b/test/javascript/run
@@ -99,7 +99,9 @@ def options():
     return [
         op.make_option("-s", "--start", metavar="FILENAME", default=None,
                        help="Start from the given filename if multiple files "
-                            "are passed")
+                            "are passed"),
+        op.make_option("-a", "--all", action="store_true", dest="all",
+                       help="Run all tests, even if one or more fail")
     ]
 
 
@@ -141,6 +143,8 @@ def main():
             passed += 1
         else:
             failed += 1
+            if not opts.all:
+                break
 
     sys.stderr.write("======================================================="
         + os.linesep)