1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
|
#
# TAP.py - TAP parser
#
# A pyparsing parser to process the output of the Perl
# "Test Anything Protocol"
# (http://search.cpan.org/~petdance/TAP-1.00/TAP.pm)
#
# TAP output lines are preceded or followed by a test number range:
# 1..n
# with 'n' TAP output lines.
#
# The general format of a TAP output line is:
# ok/not ok (required)
# Test number (recommended)
# Description (recommended)
# Directive (only when necessary)
#
# A TAP output line may also indicate abort of the test suit with the line:
# Bail out!
# optionally followed by a reason for bailing
#
# Copyright 2008, by Paul McGuire
#
from pyparsing import ParserElement,LineEnd,Optional,Word,nums,Regex,\
Literal,CaselessLiteral,Group,OneOrMore,Suppress,restOfLine,\
FollowedBy,empty
__all__ = ['tapOutputParser', 'TAPTest', 'TAPSummary']
# newlines are significant whitespace, so set default skippable
# whitespace to just spaces and tabs
ParserElement.setDefaultWhitespaceChars(" \t")
NL = LineEnd().suppress()
integer = Word(nums)
plan = '1..' + integer("ubound")
OK,NOT_OK = map(Literal,['ok','not ok'])
testStatus = (OK | NOT_OK)
description = Regex("[^#\n]+")
description.setParseAction(lambda t:t[0].lstrip('- '))
TODO,SKIP = map(CaselessLiteral,'TODO SKIP'.split())
directive = Group(Suppress('#') + (TODO + restOfLine |
FollowedBy(SKIP) +
restOfLine.copy().setParseAction(lambda t:['SKIP',t[0]]) ))
commentLine = Suppress("#") + empty + restOfLine
testLine = Group(
Optional(OneOrMore(commentLine + NL))("comments") +
testStatus("passed") +
Optional(integer)("testNumber") +
Optional(description)("description") +
Optional(directive)("directive")
)
bailLine = Group(Literal("Bail out!")("BAIL") +
empty + Optional(restOfLine)("reason"))
tapOutputParser = Optional(Group(plan)("plan") + NL) & \
Group(OneOrMore((testLine|bailLine) + NL))("tests")
class TAPTest(object):
def __init__(self,results):
self.num = results.testNumber
self.passed = (results.passed=="ok")
self.skipped = self.todo = False
if results.directive:
self.skipped = (results.directive[0][0]=='SKIP')
self.todo = (results.directive[0][0]=='TODO')
@classmethod
def bailedTest(cls,num):
ret = TAPTest(empty.parseString(""))
ret.num = num
ret.skipped = True
return ret
class TAPSummary(object):
def __init__(self,results):
self.passedTests = []
self.failedTests = []
self.skippedTests = []
self.todoTests = []
self.bonusTests = []
self.bail = False
if results.plan:
expected = list(range(1, int(results.plan.ubound)+1))
else:
expected = list(range(1,len(results.tests)+1))
for i,res in enumerate(results.tests):
# test for bail out
if res.BAIL:
#~ print "Test suite aborted: " + res.reason
#~ self.failedTests += expected[i:]
self.bail = True
self.skippedTests += [ TAPTest.bailedTest(ii) for ii in expected[i:] ]
self.bailReason = res.reason
break
#~ print res.dump()
testnum = i+1
if res.testNumber != "":
if testnum != int(res.testNumber):
print("ERROR! test %(testNumber)s out of sequence" % res)
testnum = int(res.testNumber)
res["testNumber"] = testnum
test = TAPTest(res)
if test.passed:
self.passedTests.append(test)
else:
self.failedTests.append(test)
if test.skipped: self.skippedTests.append(test)
if test.todo: self.todoTests.append(test)
if test.todo and test.passed: self.bonusTests.append(test)
self.passedSuite = not self.bail and (set(self.failedTests)-set(self.todoTests) == set())
def summary(self, showPassed=False, showAll=False):
testListStr = lambda tl : "[" + ",".join(str(t.num) for t in tl) + "]"
summaryText = []
if showPassed or showAll:
summaryText.append( "PASSED: %s" % testListStr(self.passedTests) )
if self.failedTests or showAll:
summaryText.append( "FAILED: %s" % testListStr(self.failedTests) )
if self.skippedTests or showAll:
summaryText.append( "SKIPPED: %s" % testListStr(self.skippedTests) )
if self.todoTests or showAll:
summaryText.append( "TODO: %s" % testListStr(self.todoTests) )
if self.bonusTests or showAll:
summaryText.append( "BONUS: %s" % testListStr(self.bonusTests) )
if self.passedSuite:
summaryText.append( "PASSED" )
else:
summaryText.append( "FAILED" )
return "\n".join(summaryText)
# create TAPSummary objects from tapOutput parsed results, by setting
# class as parse action
tapOutputParser.setParseAction(TAPSummary)
if __name__ == "__main__":
test1 = """\
1..4
ok 1 - Input file opened
not ok 2 - First line of the input valid
ok 3 - Read the rest of the file
not ok 4 - Summarized correctly # TODO Not written yet
"""
test2 = """\
ok 1
not ok 2 some description # TODO with a directive
ok 3 a description only, no directive
ok 4 # TODO directive only
ok a description only, no directive
ok # Skipped only a directive, no description
ok
"""
test3 = """\
ok - created Board
ok
ok
not ok
ok
ok
ok
ok
# +------+------+------+------+
# | |16G | |05C |
# | |G N C | |C C G |
# | | G | | C +|
# +------+------+------+------+
# |10C |01G | |03C |
# |R N G |G A G | |C C C |
# | R | G | | C +|
# +------+------+------+------+
# | |01G |17C |00C |
# | |G A G |G N R |R N R |
# | | G | R | G |
# +------+------+------+------+
ok - board has 7 tiles + starter tile
1..9
"""
test4 = """\
1..4
ok 1 - Creating test program
ok 2 - Test program runs, no error
not ok 3 - infinite loop # TODO halting problem unsolved
not ok 4 - infinite loop 2 # TODO halting problem unsolved
"""
test5 = """\
1..20
ok - database handle
not ok - failed database login
Bail out! Couldn't connect to database.
"""
test6 = """\
ok 1 - retrieving servers from the database
# need to ping 6 servers
ok 2 - pinged diamond
ok 3 - pinged ruby
not ok 4 - pinged sapphire
ok 5 - pinged onyx
not ok 6 - pinged quartz
ok 7 - pinged gold
1..7
"""
for test in (test1,test2,test3,test4,test5,test6):
print(test)
tapResult = tapOutputParser.parseString(test)[0]
print(tapResult.summary(showAll=True))
print()
|
