diff options
Diffstat (limited to 'paste/debug/doctest_webapp.py')
| -rwxr-xr-x | paste/debug/doctest_webapp.py | 432 |
1 files changed, 432 insertions, 0 deletions
diff --git a/paste/debug/doctest_webapp.py b/paste/debug/doctest_webapp.py new file mode 100755 index 0000000..ffcfaa7 --- /dev/null +++ b/paste/debug/doctest_webapp.py @@ -0,0 +1,432 @@ +# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org) +# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php +#!/usr/bin/env python2.4 +# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org) +# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php + +""" +These are functions for use when doctest-testing a document. +""" + +import subprocess +import doctest +import os +import sys +import shutil +import re +import cgi +import rfc822 +from cStringIO import StringIO +from paste.util import PySourceColor + + +here = os.path.abspath(__file__) +paste_parent = os.path.dirname( + os.path.dirname(os.path.dirname(here))) + +def run(command): + data = run_raw(command) + if data: + print(data) + +def run_raw(command): + """ + Runs the string command, returns any output. + """ + proc = subprocess.Popen(command, shell=True, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, env=_make_env()) + data = proc.stdout.read() + proc.wait() + while data.endswith('\n') or data.endswith('\r'): + data = data[:-1] + if data: + data = '\n'.join( + [l for l in data.splitlines() if l]) + return data + else: + return '' + +def run_command(command, name, and_print=False): + output = run_raw(command) + data = '$ %s\n%s' % (command, output) + show_file('shell-command', name, description='shell transcript', + data=data) + if and_print and output: + print(output) + +def _make_env(): + env = os.environ.copy() + env['PATH'] = (env.get('PATH', '') + + ':' + + os.path.join(paste_parent, 'scripts') + + ':' + + os.path.join(paste_parent, 'paste', '3rd-party', + 'sqlobject-files', 'scripts')) + env['PYTHONPATH'] = (env.get('PYTHONPATH', '') + + ':' + + paste_parent) + return env + +def clear_dir(dir): + """ + Clears (deletes) the given directory + """ + shutil.rmtree(dir, True) + +def ls(dir=None, recurse=False, indent=0): + """ + Show a directory listing + """ + dir = dir or os.getcwd() + fns = os.listdir(dir) + fns.sort() + for fn in fns: + full = os.path.join(dir, fn) + if os.path.isdir(full): + fn = fn + '/' + print(' '*indent + fn) + if os.path.isdir(full) and recurse: + ls(dir=full, recurse=True, indent=indent+2) + +default_app = None +default_url = None + +def set_default_app(app, url): + global default_app + global default_url + default_app = app + default_url = url + +def resource_filename(fn): + """ + Returns the filename of the resource -- generally in the directory + resources/DocumentName/fn + """ + return os.path.join( + os.path.dirname(sys.testing_document_filename), + 'resources', + os.path.splitext(os.path.basename(sys.testing_document_filename))[0], + fn) + +def show(path_info, example_name): + fn = resource_filename(example_name + '.html') + out = StringIO() + assert default_app is not None, ( + "No default_app set") + url = default_url + path_info + out.write('<span class="doctest-url"><a href="%s">%s</a></span><br>\n' + % (url, url)) + out.write('<div class="doctest-example">\n') + proc = subprocess.Popen( + ['paster', 'serve' '--server=console', '--no-verbose', + '--url=' + path_info], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + env=_make_env()) + stdout, errors = proc.communicate() + stdout = StringIO(stdout) + headers = rfc822.Message(stdout) + content = stdout.read() + for header, value in headers.items(): + if header.lower() == 'status' and int(value.split()[0]) == 200: + continue + if header.lower() in ('content-type', 'content-length'): + continue + if (header.lower() == 'set-cookie' + and value.startswith('_SID_')): + continue + out.write('<span class="doctest-header">%s: %s</span><br>\n' + % (header, value)) + lines = [l for l in content.splitlines() if l.strip()] + for line in lines: + out.write(line + '\n') + if errors: + out.write('<pre class="doctest-errors">%s</pre>' + % errors) + out.write('</div>\n') + result = out.getvalue() + if not os.path.exists(fn): + f = open(fn, 'wb') + f.write(result) + f.close() + else: + f = open(fn, 'rb') + expected = f.read() + f.close() + if not html_matches(expected, result): + print('Pages did not match. Expected from %s:' % fn) + print('-'*60) + print(expected) + print('='*60) + print('Actual output:') + print('-'*60) + print(result) + +def html_matches(pattern, text): + regex = re.escape(pattern) + regex = regex.replace(r'\.\.\.', '.*') + regex = re.sub(r'0x[0-9a-f]+', '.*', regex) + regex = '^%s$' % regex + return re.search(regex, text) + +def convert_docstring_string(data): + if data.startswith('\n'): + data = data[1:] + lines = data.splitlines() + new_lines = [] + for line in lines: + if line.rstrip() == '.': + new_lines.append('') + else: + new_lines.append(line) + data = '\n'.join(new_lines) + '\n' + return data + +def create_file(path, version, data): + data = convert_docstring_string(data) + write_data(path, data) + show_file(path, version) + +def append_to_file(path, version, data): + data = convert_docstring_string(data) + f = open(path, 'a') + f.write(data) + f.close() + # I think these appends can happen so quickly (in less than a second) + # that the .pyc file doesn't appear to be expired, even though it + # is after we've made this change; so we have to get rid of the .pyc + # file: + if path.endswith('.py'): + pyc_file = path + 'c' + if os.path.exists(pyc_file): + os.unlink(pyc_file) + show_file(path, version, description='added to %s' % path, + data=data) + +def show_file(path, version, description=None, data=None): + ext = os.path.splitext(path)[1] + if data is None: + f = open(path, 'rb') + data = f.read() + f.close() + if ext == '.py': + html = ('<div class="source-code">%s</div>' + % PySourceColor.str2html(data, PySourceColor.dark)) + else: + html = '<pre class="source-code">%s</pre>' % cgi.escape(data, 1) + html = '<span class="source-filename">%s</span><br>%s' % ( + description or path, html) + write_data(resource_filename('%s.%s.gen.html' % (path, version)), + html) + +def call_source_highlight(input, format): + proc = subprocess.Popen(['source-highlight', '--out-format=html', + '--no-doc', '--css=none', + '--src-lang=%s' % format], shell=False, + stdout=subprocess.PIPE) + stdout, stderr = proc.communicate(input) + result = stdout + proc.wait() + return result + + +def write_data(path, data): + dir = os.path.dirname(os.path.abspath(path)) + if not os.path.exists(dir): + os.makedirs(dir) + f = open(path, 'wb') + f.write(data) + f.close() + + +def change_file(path, changes): + f = open(os.path.abspath(path), 'rb') + lines = f.readlines() + f.close() + for change_type, line, text in changes: + if change_type == 'insert': + lines[line:line] = [text] + elif change_type == 'delete': + lines[line:text] = [] + else: + assert 0, ( + "Unknown change_type: %r" % change_type) + f = open(path, 'wb') + f.write(''.join(lines)) + f.close() + +class LongFormDocTestParser(doctest.DocTestParser): + + """ + This parser recognizes some reST comments as commands, without + prompts or expected output, like: + + .. run: + + do_this(... + ...) + """ + + _EXAMPLE_RE = re.compile(r""" + # Source consists of a PS1 line followed by zero or more PS2 lines. + (?: (?P<source> + (?:^(?P<indent> [ ]*) >>> .*) # PS1 line + (?:\n [ ]* \.\.\. .*)*) # PS2 lines + \n? + # Want consists of any non-blank lines that do not start with PS1. + (?P<want> (?:(?![ ]*$) # Not a blank line + (?![ ]*>>>) # Not a line starting with PS1 + .*$\n? # But any other line + )*)) + | + (?: # This is for longer commands that are prefixed with a reST + # comment like '.. run:' (two colons makes that a directive). + # These commands cannot have any output. + + (?:^\.\.[ ]*(?P<run>run):[ ]*\n) # Leading command/command + (?:[ ]*\n)? # Blank line following + (?P<runsource> + (?:(?P<runindent> [ ]+)[^ ].*$) + (?:\n [ ]+ .*)*) + ) + | + (?: # This is for shell commands + + (?P<shellsource> + (?:^(P<shellindent> [ ]*) [$] .*) # Shell line + (?:\n [ ]* [>] .*)*) # Continuation + \n? + # Want consists of any non-blank lines that do not start with $ + (?P<shellwant> (?:(?![ ]*$) + (?![ ]*[$]$) + .*$\n? + )*)) + """, re.MULTILINE | re.VERBOSE) + + def _parse_example(self, m, name, lineno): + r""" + Given a regular expression match from `_EXAMPLE_RE` (`m`), + return a pair `(source, want)`, where `source` is the matched + example's source code (with prompts and indentation stripped); + and `want` is the example's expected output (with indentation + stripped). + + `name` is the string's name, and `lineno` is the line number + where the example starts; both are used for error messages. + + >>> def parseit(s): + ... p = LongFormDocTestParser() + ... return p._parse_example(p._EXAMPLE_RE.search(s), '<string>', 1) + >>> parseit('>>> 1\n1') + ('1', {}, '1', None) + >>> parseit('>>> (1\n... +1)\n2') + ('(1\n+1)', {}, '2', None) + >>> parseit('.. run:\n\n test1\n test2\n') + ('test1\ntest2', {}, '', None) + """ + # Get the example's indentation level. + runner = m.group('run') or '' + indent = len(m.group('%sindent' % runner)) + + # Divide source into lines; check that they're properly + # indented; and then strip their indentation & prompts. + source_lines = m.group('%ssource' % runner).split('\n') + if runner: + self._check_prefix(source_lines[1:], ' '*indent, name, lineno) + else: + self._check_prompt_blank(source_lines, indent, name, lineno) + self._check_prefix(source_lines[2:], ' '*indent + '.', name, lineno) + if runner: + source = '\n'.join([sl[indent:] for sl in source_lines]) + else: + source = '\n'.join([sl[indent+4:] for sl in source_lines]) + + if runner: + want = '' + exc_msg = None + else: + # Divide want into lines; check that it's properly indented; and + # then strip the indentation. Spaces before the last newline should + # be preserved, so plain rstrip() isn't good enough. + want = m.group('want') + want_lines = want.split('\n') + if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]): + del want_lines[-1] # forget final newline & spaces after it + self._check_prefix(want_lines, ' '*indent, name, + lineno + len(source_lines)) + want = '\n'.join([wl[indent:] for wl in want_lines]) + + # If `want` contains a traceback message, then extract it. + m = self._EXCEPTION_RE.match(want) + if m: + exc_msg = m.group('msg') + else: + exc_msg = None + + # Extract options from the source. + options = self._find_options(source, name, lineno) + + return source, options, want, exc_msg + + + def parse(self, string, name='<string>'): + """ + Divide the given string into examples and intervening text, + and return them as a list of alternating Examples and strings. + Line numbers for the Examples are 0-based. The optional + argument `name` is a name identifying this string, and is only + used for error messages. + """ + string = string.expandtabs() + # If all lines begin with the same indentation, then strip it. + min_indent = self._min_indent(string) + if min_indent > 0: + string = '\n'.join([l[min_indent:] for l in string.split('\n')]) + + output = [] + charno, lineno = 0, 0 + # Find all doctest examples in the string: + for m in self._EXAMPLE_RE.finditer(string): + # Add the pre-example text to `output`. + output.append(string[charno:m.start()]) + # Update lineno (lines before this example) + lineno += string.count('\n', charno, m.start()) + # Extract info from the regexp match. + (source, options, want, exc_msg) = \ + self._parse_example(m, name, lineno) + # Create an Example, and add it to the list. + if not self._IS_BLANK_OR_COMMENT(source): + # @@: Erg, this is the only line I need to change... + output.append(doctest.Example( + source, want, exc_msg, + lineno=lineno, + indent=min_indent+len(m.group('indent') or m.group('runindent')), + options=options)) + # Update lineno (lines inside this example) + lineno += string.count('\n', m.start(), m.end()) + # Update charno. + charno = m.end() + # Add any remaining post-example text to `output`. + output.append(string[charno:]) + return output + + + +if __name__ == '__main__': + if sys.argv[1:] and sys.argv[1] == 'doctest': + doctest.testmod() + sys.exit() + if not paste_parent in sys.path: + sys.path.append(paste_parent) + for fn in sys.argv[1:]: + fn = os.path.abspath(fn) + # @@: OK, ick; but this module gets loaded twice + sys.testing_document_filename = fn + doctest.testfile( + fn, module_relative=False, + optionflags=doctest.ELLIPSIS|doctest.REPORT_ONLY_FIRST_FAILURE, + parser=LongFormDocTestParser()) + new = os.path.splitext(fn)[0] + '.html' + assert new != fn + os.system('rst2html.py %s > %s' % (fn, new)) |