1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
|
#!/usr/bin/env python
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
from __future__ import print_function
import datetime
import glob
import json
import os
import tarfile
import time
import requests
COUCH_URL = "https://couchdb-vm2.apache.org/ci_errorlogs"
TARFILE = "couchlog.tar.gz"
def _tojson(req):
"""Support requests v0.x as well as 1.x+"""
if requests.__version__[0] == '0':
return json.loads(req.content)
return req.json()
def collect_logfiles():
""" Find and tarball all logfiles """
tb = tarfile.open(name=TARFILE, mode='w:gz')
# EUnit
for log in glob.glob('src/*/.eunit/couch.log'):
tb.add(log)
# JS harness
for log in glob.glob('dev/logs/node1.log'):
tb.add(log)
tb.close()
def build_ci_doc():
""" Build a metadata document with relevant detail from CI env """
doc = {}
if 'TRAVIS' in os.environ:
doc['builder'] = 'travis'
doc['build_id'] = os.environ['TRAVIS_JOB_ID']
doc['erlang'] = os.environ['TRAVIS_OTP_RELEASE']
doc['url'] = 'https://travis-ci.org/apache/couchdb/jobs/' + \
os.environ['TRAVIS_JOB_ID']
doc['branch'] = os.environ['TRAVIS_BRANCH']
doc['commit'] = os.environ['TRAVIS_COMMIT']
doc['repo'] = 'https://github.com/' + os.environ['TRAVIS_REPO_SLUG']
elif 'JENKINS_URL' in os.environ:
doc['builder'] = 'jenkins'
doc['build_id'] = os.environ['BUILD_NUMBER']
doc['url'] = os.environ['BUILD_URL']
doc['branch'] = os.environ['BRANCH_NAME']
doc['repo'] = 'https://github.com/apache/couchdb'
else:
doc['builder'] = 'manual'
# TODO: shell out to get correct repo, commit, branch info?
doc['repo'] = 'https://github.com/apache/couchdb'
doc['build_id'] = str(time.time())
# shorten doc id
repo = doc['repo'].split('/')[-1]
repo = repo.replace('.git', '')
doc['_id'] = doc['builder'] + '-' + repo + '-' + \
doc['build_id'] + \
'-' + datetime.datetime.utcnow().isoformat()
return doc
def upload_logs():
try:
lp = os.environ['COUCHAUTH'].split(':')
except KeyError as e:
print ("ERROR: COUCHAUTH credentials unavailable! "
"Unable to upload logfiles.")
exit(1)
creds = (lp[0], lp[1])
doc = build_ci_doc()
req = requests.post(COUCH_URL,
data=json.dumps(doc),
auth=creds,
headers={'Content-type': 'application/json'})
req.raise_for_status()
req = _tojson(req)
with open(TARFILE, 'rb') as f:
# ancient versions of requests break if data is iterable
fdata = f.read()
req2 = requests.put(COUCH_URL + '/' + doc['_id'] + '/' + TARFILE,
headers={'Content-type': 'application/x-gtar'},
auth=creds,
params={'rev': req['rev']},
data=fdata)
req2.raise_for_status()
return req2
def main():
""" Find latest logfile and upload to Couch logfile db. """
print ("Uploading logfiles...")
collect_logfiles()
req = upload_logs()
print (req.url)
print (req.content)
print ("Upload complete!")
if __name__ == '__main__':
main()
|
