diff options
Diffstat (limited to 'paste')
73 files changed, 3495 insertions, 7745 deletions
diff --git a/paste/__init__.py b/paste/__init__.py index ba66606..4e2d638 100644 --- a/paste/__init__.py +++ b/paste/__init__.py @@ -6,7 +6,7 @@ try: except ImportError: # don't prevent use of paste if pkg_resources isn't installed from pkgutil import extend_path - __path__ = extend_path(__path__, __name__) + __path__ = extend_path(__path__, __name__) try: import modulefinder diff --git a/paste/auth/auth_tkt.py b/paste/auth/auth_tkt.py index 8dfce00..da8ddbd 100644 --- a/paste/auth/auth_tkt.py +++ b/paste/auth/auth_tkt.py @@ -39,14 +39,21 @@ non-Python code run under Apache. import time as time_mod try: - from hashlib import md5 + import hashlib except ImportError: - from md5 import md5 -import Cookie + # mimic hashlib (will work for md5, fail for secure hashes) + import md5 as hashlib +try: + from http.cookies import SimpleCookie +except ImportError: + # Python 2 + from Cookie import SimpleCookie from paste import request from urllib import quote as url_quote from urllib import unquote as url_unquote +DEFAULT_DIGEST = hashlib.md5 + class AuthTicket(object): @@ -55,8 +62,9 @@ class AuthTicket(object): the shared secret, the userid, and the IP address. Optionally you can include tokens (a list of strings, representing role names), 'user_data', which is arbitrary data available for your own use in - later scripts. Lastly, you can override the cookie name and - timestamp. + later scripts. Lastly, you can override the timestamp, cookie name, + whether to secure the cookie and the digest algorithm (for details + look at ``AuthTKTMiddleware``). Once you provide all the arguments, use .cookie_value() to generate the appropriate authentication ticket. .cookie() @@ -67,10 +75,10 @@ class AuthTicket(object): token = auth_tkt.AuthTick('sharedsecret', 'username', os.environ['REMOTE_ADDR'], tokens=['admin']) - print 'Status: 200 OK' - print 'Content-type: text/html' - print token.cookie() - print + print('Status: 200 OK') + print('Content-type: text/html') + print(token.cookie()) + print("") ... redirect HTML ... Webware usage:: @@ -86,11 +94,13 @@ class AuthTicket(object): def __init__(self, secret, userid, ip, tokens=(), user_data='', time=None, cookie_name='auth_tkt', - secure=False): + secure=False, digest_algo=DEFAULT_DIGEST): self.secret = secret self.userid = userid self.ip = ip - self.tokens = ','.join(tokens) + if not isinstance(tokens, basestring): + tokens = ','.join(tokens) + self.tokens = tokens self.user_data = user_data if time is None: self.time = time_mod.time() @@ -98,11 +108,16 @@ class AuthTicket(object): self.time = time self.cookie_name = cookie_name self.secure = secure + if isinstance(digest_algo, str): + # correct specification of digest from hashlib or fail + self.digest_algo = getattr(hashlib, digest_algo) + else: + self.digest_algo = digest_algo def digest(self): return calculate_digest( self.ip, self.time, self.secret, self.userid, self.tokens, - self.user_data) + self.user_data, self.digest_algo) def cookie_value(self): v = '%s%08x%s!' % (self.digest(), int(self.time), url_quote(self.userid)) @@ -112,7 +127,7 @@ class AuthTicket(object): return v def cookie(self): - c = Cookie.SimpleCookie() + c = SimpleCookie() c[self.cookie_name] = self.cookie_value().encode('base64').strip().replace('\n', '') c[self.cookie_name]['path'] = '/' if self.secure: @@ -132,21 +147,25 @@ class BadTicket(Exception): Exception.__init__(self, msg) -def parse_ticket(secret, ticket, ip): +def parse_ticket(secret, ticket, ip, digest_algo=DEFAULT_DIGEST): """ Parse the ticket, returning (timestamp, userid, tokens, user_data). If the ticket cannot be parsed, ``BadTicket`` will be raised with an explanation. """ + if isinstance(digest_algo, str): + # correct specification of digest from hashlib or fail + digest_algo = getattr(hashlib, digest_algo) + digest_hexa_size = digest_algo().digest_size * 2 ticket = ticket.strip('"') - digest = ticket[:32] + digest = ticket[:digest_hexa_size] try: - timestamp = int(ticket[32:40], 16) - except ValueError, e: + timestamp = int(ticket[digest_hexa_size:digest_hexa_size + 8], 16) + except ValueError as e: raise BadTicket('Timestamp is not a hex integer: %s' % e) try: - userid, data = ticket[40:].split('!', 1) + userid, data = ticket[digest_hexa_size + 8:].split('!', 1) except ValueError: raise BadTicket('userid is not followed by !') userid = url_unquote(userid) @@ -158,7 +177,8 @@ def parse_ticket(secret, ticket, ip): user_data = data expected = calculate_digest(ip, timestamp, secret, - userid, tokens, user_data) + userid, tokens, user_data, + digest_algo) if expected != digest: raise BadTicket('Digest signature is not correct', @@ -169,15 +189,17 @@ def parse_ticket(secret, ticket, ip): return (timestamp, userid, tokens, user_data) -def calculate_digest(ip, timestamp, secret, userid, tokens, user_data): +# @@: Digest object constructor compatible with named ones in hashlib only +def calculate_digest(ip, timestamp, secret, userid, tokens, user_data, + digest_algo): secret = maybe_encode(secret) userid = maybe_encode(userid) tokens = maybe_encode(tokens) user_data = maybe_encode(user_data) - digest0 = md5( + digest0 = digest_algo( encode_ip_timestamp(ip, timestamp) + secret + userid + '\0' + tokens + '\0' + user_data).hexdigest() - digest = md5(digest0 + secret).hexdigest() + digest = digest_algo(digest0 + secret).hexdigest() return digest @@ -234,6 +256,12 @@ class AuthTKTMiddleware(object): page will be shown as usual, but the user will also be logged out when they visit this page. + ``digest_algo``: + Digest algorithm specified as a name of the algorithm provided by + ``hashlib`` or as a compatible digest object constructor. + Defaults to ``md5``, as in mod_auth_tkt. The others currently + compatible with mod_auth_tkt are ``sha256`` and ``sha512``. + If used with mod_auth_tkt, then these settings (except logout_path) should match the analogous Apache configuration settings. @@ -253,7 +281,7 @@ class AuthTKTMiddleware(object): def __init__(self, app, secret, cookie_name='auth_tkt', secure=False, include_ip=True, logout_path=None, httponly=False, no_domain_cookie=True, current_domain_cookie=True, - wildcard_cookie=True): + wildcard_cookie=True, digest_algo=DEFAULT_DIGEST): self.app = app self.secret = secret self.cookie_name = cookie_name @@ -264,6 +292,11 @@ class AuthTKTMiddleware(object): self.no_domain_cookie = no_domain_cookie self.current_domain_cookie = current_domain_cookie self.wildcard_cookie = wildcard_cookie + if isinstance(digest_algo, str): + # correct specification of digest from hashlib or fail + self.digest_algo = getattr(hashlib, digest_algo) + else: + self.digest_algo = digest_algo def __call__(self, environ, start_response): cookies = request.get_cookies(environ) @@ -282,7 +315,7 @@ class AuthTKTMiddleware(object): # Also, timeouts should cause cookie refresh try: timestamp, userid, tokens, user_data = parse_ticket( - self.secret, cookie_value, remote_addr) + self.secret, cookie_value, remote_addr, self.digest_algo) tokens = ','.join(tokens) environ['REMOTE_USER'] = userid if environ.get('REMOTE_USER_TOKENS'): diff --git a/paste/auth/basic.py b/paste/auth/basic.py index 69db128..24d1731 100644 --- a/paste/auth/basic.py +++ b/paste/auth/basic.py @@ -108,14 +108,14 @@ def make_basic(app, global_conf, realm, authfunc, **kw): use = egg:Paste#auth_basic realm=myrealm authfunc=somepackage.somemodule:somefunction - + """ from paste.util.import_string import eval_import import types authfunc = eval_import(authfunc) assert isinstance(authfunc, types.FunctionType), "authfunc must resolve to a function" return AuthBasicHandler(app, realm, authfunc) - + if "__main__" == __name__: import doctest diff --git a/paste/auth/cas.py b/paste/auth/cas.py index c3521a0..44e4e98 100644 --- a/paste/auth/cas.py +++ b/paste/auth/cas.py @@ -18,7 +18,7 @@ passed to the system so that it can be used as middleware at any stage of processing. It has the secondary goal of allowing for other authentication methods to be used concurrently. """ -import urllib +from six.moves.urllib.parse import urlencode from paste.request import construct_url from paste.httpexceptions import HTTPSeeOther, HTTPForbidden @@ -69,10 +69,10 @@ def AuthCASHandler(application, authority): ticket = qs.pop().split("=", 1)[1] environ['QUERY_STRING'] = "&".join(qs) service = construct_url(environ) - args = urllib.urlencode( + args = urlencode( {'service': service,'ticket': ticket}) requrl = authority + "validate?" + args - result = urllib.urlopen(requrl).read().split("\n") + result = urlopen(requrl).read().split("\n") if 'yes' == result[0]: environ['REMOTE_USER'] = result[1] environ['AUTH_TYPE'] = 'cas' @@ -80,7 +80,7 @@ def AuthCASHandler(application, authority): exce = CASLoginFailure() else: service = construct_url(environ) - args = urllib.urlencode({'service': service}) + args = urlencode({'service': service}) location = authority + "login?" + args exce = CASAuthenticate(location) return exce.wsgi_application(environ, start_response) diff --git a/paste/auth/cookie.py b/paste/auth/cookie.py index 2601a90..8f11d1b 100644 --- a/paste/auth/cookie.py +++ b/paste/auth/cookie.py @@ -41,7 +41,7 @@ corresponding to a database session id) is stored in the cookie. """ -import hmac, base64, random, time, warnings +import hmac, base64, random, six, time, warnings try: from hashlib import sha1 except ImportError: @@ -52,7 +52,7 @@ from paste.request import get_cookies def make_time(value): return time.strftime("%Y%m%d%H%M", time.gmtime(value)) -_signature_size = len(hmac.new('x', 'x', sha1).digest()) +_signature_size = len(hmac.new(b'x', b'x', sha1).digest()) _header_size = _signature_size + len(make_time(time.time())) # @@: Should this be using urllib.quote? @@ -62,7 +62,7 @@ _encode = [('\\', '\\x5c'), ('"', '\\x22'), _decode = [(v, k) for (k, v) in _encode] _decode.reverse() def encode(s, sublist = _encode): - return reduce((lambda a, (b, c): a.replace(b, c)), sublist, str(s)) + return six.moves.reduce((lambda a, b: a.replace(b[0], b[1])), sublist, str(s)) decode = lambda s: encode(s, _decode) class CookieTooLarge(RuntimeError): @@ -74,7 +74,10 @@ class CookieTooLarge(RuntimeError): _all_chars = ''.join([chr(x) for x in range(0, 255)]) def new_secret(): """ returns a 64 byte secret """ - return ''.join(random.sample(_all_chars, 64)) + secret = ''.join(random.sample(_all_chars, 64)) + if six.PY3: + secret = secret.encode('utf8') + return secret class AuthCookieSigner(object): """ @@ -124,7 +127,7 @@ class AuthCookieSigner(object): """ def __init__(self, secret = None, timeout = None, maxlen = None): self.timeout = timeout or 30 - if isinstance(timeout, basestring): + if isinstance(timeout, six.string_types): raise ValueError( "Timeout must be a number (minutes), not a string (%r)" % timeout) @@ -137,12 +140,16 @@ class AuthCookieSigner(object): need to be escaped and quoted). The expiration of this cookie is handled server-side in the auth() function. """ + timestamp = make_time(time.time() + 60*self.timeout) + if six.PY3: + content = content.encode('utf8') + timestamp = timestamp.encode('utf8') cookie = base64.encodestring( hmac.new(self.secret, content, sha1).digest() + - make_time(time.time() + 60*self.timeout) + + timestamp + content) - cookie = cookie.replace("/", "_").replace("=", "~") - cookie = cookie.replace('\n', '').replace('\r', '') + cookie = cookie.replace(b"/", b"_").replace(b"=", b"~") + cookie = cookie.replace(b'\n', b'').replace(b'\r', b'') if len(cookie) > self.maxlen: raise CookieTooLarge(content, cookie) return cookie @@ -258,7 +265,7 @@ class AuthCookieHandler(object): raise AssertionError("AuthCookie already installed!") scanlist = self.environ_class(self, self.scanlist) jar = get_cookies(environ) - if jar.has_key(self.cookie_name): + if self.cookie_name in jar: content = self.signer.auth(jar[self.cookie_name].value) if content: for pair in content.split(";"): @@ -298,6 +305,8 @@ class AuthCookieHandler(object): if content: content = ";".join(content) content = self.signer.sign(content) + if six.PY3: + content = content.decode('utf8') cookie = '%s=%s; Path=/;' % (self.cookie_name, content) if 'https' == environ['wsgi.url_scheme']: cookie += ' secure;' @@ -368,9 +377,9 @@ def make_auth_cookie( The maximum length of the cookie that is sent (default 4k, which is a typical browser maximum) - + """ - if isinstance(scanlist, basestring): + if isinstance(scanlist, six.string_types): scanlist = scanlist.split() if secret is None and global_conf.get('secret'): secret = global_conf['secret'] diff --git a/paste/auth/digest.py b/paste/auth/digest.py index 483c38d..85e0362 100644 --- a/paste/auth/digest.py +++ b/paste/auth/digest.py @@ -36,11 +36,43 @@ try: except ImportError: from md5 import md5 import time, random -from urllib import quote as url_quote +from six.moves.urllib.parse import quote as url_quote +import six + +def _split_auth_string(auth_string): + """ split a digest auth string into individual key=value strings """ + prev = None + for item in auth_string.split(","): + try: + if prev.count('"') == 1: + prev = "%s,%s" % (prev, item) + continue + except AttributeError: + if prev == None: + prev = item + continue + else: + raise StopIteration + yield prev.strip() + prev = item + + yield prev.strip() + raise StopIteration + +def _auth_to_kv_pairs(auth_string): + """ split a digest auth string into key, value pairs """ + for item in _split_auth_string(auth_string): + (k, v) = item.split("=", 1) + if v.startswith('"') and len(v) > 1 and v.endswith('"'): + v = v[1:-1] + yield (k, v) def digest_password(realm, username, password): """ construct the appropriate hashcode needed for HTTP digest """ - return md5("%s:%s:%s" % (username, realm, password)).hexdigest() + content = "%s:%s:%s" % (username, realm, password) + if six.PY3: + content = content.encode('utf8') + return md5(content).hexdigest() class AuthDigestAuthenticator(object): """ implementation of RFC 2617 - HTTP Digest Authentication """ @@ -51,10 +83,16 @@ class AuthDigestAuthenticator(object): def build_authentication(self, stale = ''): """ builds the authentication error """ - nonce = md5( - "%s:%s" % (time.time(), random.random())).hexdigest() - opaque = md5( - "%s:%s" % (time.time(), random.random())).hexdigest() + content = "%s:%s" % (time.time(), random.random()) + if six.PY3: + content = content.encode('utf-8') + nonce = md5(content).hexdigest() + + content = "%s:%s" % (time.time(), random.random()) + if six.PY3: + content = content.encode('utf-8') + opaque = md5(content).hexdigest() + self.nonce[nonce] = None parts = {'realm': self.realm, 'qop': 'auth', 'nonce': nonce, 'opaque': opaque } @@ -69,17 +107,22 @@ class AuthDigestAuthenticator(object): """ computes the authentication, raises error if unsuccessful """ if not ha1: return self.build_authentication() - ha2 = md5('%s:%s' % (method, path)).hexdigest() + content = '%s:%s' % (method, path) + if six.PY3: + content = content.encode('utf8') + ha2 = md5(content).hexdigest() if qop: chk = "%s:%s:%s:%s:%s:%s" % (ha1, nonce, nc, cnonce, qop, ha2) else: chk = "%s:%s:%s" % (ha1, nonce, ha2) + if six.PY3: + chk = chk.encode('utf8') if response != md5(chk).hexdigest(): if nonce in self.nonce: del self.nonce[nonce] return self.build_authentication() pnc = self.nonce.get(nonce,'00000000') - if nc <= pnc: + if pnc is not None and nc <= pnc: if nonce in self.nonce: del self.nonce[nonce] return self.build_authentication(stale = True) @@ -98,10 +141,7 @@ class AuthDigestAuthenticator(object): (authmeth, auth) = authorization.split(" ", 1) if 'digest' != authmeth.lower(): return self.build_authentication() - amap = {} - for itm in auth.split(","): - (k,v) = [s.strip() for s in itm.strip().split("=", 1)] - amap[k] = v.replace('"', '') + amap = dict(_auth_to_kv_pairs(auth)) try: username = amap['username'] authpath = amap['uri'] diff --git a/paste/auth/form.py b/paste/auth/form.py index 4e6aa49..9be82a2 100644 --- a/paste/auth/form.py +++ b/paste/auth/form.py @@ -131,7 +131,7 @@ def make_form(app, global_conf, realm, authfunc, **kw): use = egg:Paste#auth_form realm=myrealm authfunc=somepackage.somemodule:somefunction - + """ from paste.util.import_string import eval_import import types diff --git a/paste/auth/grantip.py b/paste/auth/grantip.py index 94b3900..3fe6e1c 100644 --- a/paste/auth/grantip.py +++ b/paste/auth/grantip.py @@ -3,6 +3,7 @@ """ Grant roles and logins based on IP address. """ +import six from paste.util import ip4 class GrantIPMiddleware(object): @@ -34,10 +35,10 @@ class GrantIPMiddleware(object): self.clobber_username = clobber_username def _convert_user_role(self, username, roles): - if roles and isinstance(roles, basestring): + if roles and isinstance(roles, six.string_types): roles = roles.split(',') return (username, roles) - + def __call__(self, environ, start_response): addr = ip4.ip2int(environ['REMOTE_ADDR'], False) remove_user = False @@ -61,7 +62,7 @@ class GrantIPMiddleware(object): def _set_roles(self, environ, roles): cur_roles = environ.get('REMOTE_USER_TOKENS', '').split(',') # Get rid of empty roles: - cur_roles = filter(None, cur_roles) + cur_roles = list(filter(None, cur_roles)) remove_roles = [] for role in roles: if role.startswith('-'): @@ -73,8 +74,8 @@ class GrantIPMiddleware(object): if role in cur_roles: cur_roles.remove(role) environ['REMOTE_USER_TOKENS'] = ','.join(cur_roles) - - + + def make_grantip(app, global_conf, clobber_username=False, **kw): """ Grant roles or usernames based on IP addresses. @@ -92,7 +93,7 @@ def make_grantip(app, global_conf, clobber_username=False, **kw): 192.168.0.7 = joe # And one IP is should not be logged in: 192.168.0.10 = __remove__:-editor - + """ from paste.deploy.converters import asbool clobber_username = asbool(clobber_username) @@ -109,5 +110,5 @@ def make_grantip(app, global_conf, clobber_username=False, **kw): role = '' ip_map[key] = value return GrantIPMiddleware(app, ip_map, clobber_username) - - + + diff --git a/paste/auth/open_id.py b/paste/auth/open_id.py index f6efe61..f79f7f8 100644 --- a/paste/auth/open_id.py +++ b/paste/auth/open_id.py @@ -58,6 +58,7 @@ __all__ = ['AuthOpenIDHandler'] import cgi import urlparse import re +import six import paste.request from paste import httpexceptions @@ -90,20 +91,20 @@ class AuthOpenIDHandler(object): ``app`` Your WSGI app to call - + ``data_store_path`` Directory to store crypto data in for use with OpenID servers. - + ``auth_prefix`` Location for authentication process/verification - + ``login_redirect`` Location to load after successful process of login - + ``catch_401`` If true, then any 401 responses will turn into open ID login requirements. - + ``url_to_username`` A function called like ``url_to_username(environ, url)``, which should return a string username. If not given, the URL will be the username. @@ -398,7 +399,7 @@ def make_open_id_middleware( from paste.deploy.converters import asbool from paste.util import import_string catch_401 = asbool(catch_401) - if url_to_username and isinstance(url_to_username, basestring): + if url_to_username and isinstance(url_to_username, six.string_types): url_to_username = import_string.eval_import(url_to_username) apply_auth_tkt = asbool(apply_auth_tkt) new_app = AuthOpenIDHandler( diff --git a/paste/cascade.py b/paste/cascade.py index 1c4acfb..8207ae3 100644 --- a/paste/cascade.py +++ b/paste/cascade.py @@ -15,7 +15,7 @@ __all__ = ['Cascade'] def make_cascade(loader, global_conf, catch='404', **local_conf): """ Entry point for Paste Deploy configuration - + Expects configuration like:: [composit:cascade] @@ -39,7 +39,7 @@ def make_cascade(loader, global_conf, catch='404', **local_conf): apps.sort() apps = [app for name, app in apps] return Cascade(apps, catch=catch) - + class Cascade(object): """ @@ -70,7 +70,7 @@ class Cascade(object): self.catch_codes[code] = exc self.catch_exceptions.append(exc) self.catch_exceptions = tuple(self.catch_exceptions) - + def __call__(self, environ, start_response): """ WSGI application interface @@ -123,7 +123,7 @@ class Cascade(object): list(v) # then close: v.close() - except self.catch_exceptions, e: + except self.catch_exceptions: pass if copy_wsgi_input: environ['wsgi.input'].seek(0) diff --git a/paste/cgiapp.py b/paste/cgiapp.py index 8801750..e5a62f4 100644 --- a/paste/cgiapp.py +++ b/paste/cgiapp.py @@ -7,11 +7,12 @@ Application that runs a CGI script. import os import sys import subprocess -import urllib +from six.moves.urllib.parse import quote try: import select except ImportError: select = None +import six from paste.util import converters @@ -69,8 +70,8 @@ class CGIApplication(object): def __call__(self, environ, start_response): if 'REQUEST_URI' not in environ: environ['REQUEST_URI'] = ( - urllib.quote(environ.get('SCRIPT_NAME', '')) - + urllib.quote(environ.get('PATH_INFO', ''))) + quote(environ.get('SCRIPT_NAME', '')) + + quote(environ.get('PATH_INFO', ''))) if self.include_os_environ: cgi_environ = os.environ.copy() else: @@ -119,18 +120,18 @@ class CGIWriter(object): self.headers = [] self.headers_finished = False self.writer = None - self.buffer = '' + self.buffer = b'' def write(self, data): if self.headers_finished: self.writer(data) return self.buffer += data - while '\n' in self.buffer: - if '\r\n' in self.buffer and self.buffer.find('\r\n') < self.buffer.find('\n'): - line1, self.buffer = self.buffer.split('\r\n', 1) + while b'\n' in self.buffer: + if b'\r\n' in self.buffer and self.buffer.find(b'\r\n') < self.buffer.find(b'\n'): + line1, self.buffer = self.buffer.split(b'\r\n', 1) else: - line1, self.buffer = self.buffer.split('\n', 1) + line1, self.buffer = self.buffer.split(b'\n', 1) if not line1: self.headers_finished = True self.writer = self.start_response( @@ -140,13 +141,16 @@ class CGIWriter(object): del self.headers del self.status break - elif ':' not in line1: + elif b':' not in line1: raise CGIError( "Bad header line: %r" % line1) else: - name, value = line1.split(':', 1) + name, value = line1.split(b':', 1) value = value.lstrip() name = name.strip() + if six.PY3: + name = name.decode('utf8') + value = value.decode('utf8') if name.lower() == 'status': if ' ' not in value: # WSGI requires this space, sometimes CGI scripts don't set it: @@ -161,6 +165,7 @@ class StdinReader(object): self.stdin = stdin self.content_length = content_length + @classmethod def from_environ(cls, environ): length = environ.get('CONTENT_LENGTH') if length: @@ -169,11 +174,9 @@ class StdinReader(object): length = 0 return cls(environ['wsgi.input'], length) - from_environ = classmethod(from_environ) - def read(self, size=None): if not self.content_length: - return '' + return b'' if size is None: text = self.stdin.read(self.content_length) else: @@ -193,7 +196,7 @@ def proc_communicate(proc, stdin=None, stdout=None, stderr=None): """ read_set = [] write_set = [] - input_buffer = '' + input_buffer = b'' trans_nl = proc.universal_newlines and hasattr(open, 'newlines') if proc.stdin: @@ -222,7 +225,7 @@ def proc_communicate(proc, stdin=None, stdout=None, stderr=None): # When select has indicated that the file is writable, # we can write up to PIPE_BUF bytes without risk # blocking. POSIX defines PIPE_BUF >= 512 - next, input_buffer = input_buffer, '' + next, input_buffer = input_buffer, b'' next_len = 512-len(next) if next_len: next += stdin.read(next_len) @@ -236,7 +239,7 @@ def proc_communicate(proc, stdin=None, stdout=None, stderr=None): if proc.stdout in rlist: data = os.read(proc.stdout.fileno(), 1024) - if data == "": + if data == b"": proc.stdout.close() read_set.remove(proc.stdout) if trans_nl: @@ -245,7 +248,7 @@ def proc_communicate(proc, stdin=None, stdout=None, stderr=None): if proc.stderr in rlist: data = os.read(proc.stderr.fileno(), 1024) - if data == "": + if data == b"": proc.stderr.close() read_set.remove(proc.stderr) if trans_nl: @@ -254,7 +257,7 @@ def proc_communicate(proc, stdin=None, stdout=None, stderr=None): try: proc.wait() - except OSError, e: + except OSError as e: if e.errno != 10: raise diff --git a/paste/cgitb_catcher.py b/paste/cgitb_catcher.py index 1c815b7..f88ffb8 100644 --- a/paste/cgitb_catcher.py +++ b/paste/cgitb_catcher.py @@ -10,7 +10,8 @@ for more. """ import cgitb -from cStringIO import StringIO +import six +from six.moves import cStringIO as StringIO import sys from paste.util import converters @@ -31,7 +32,7 @@ class CgitbMiddleware(object): global_conf = {} if display is NoDefault: display = global_conf.get('debug') - if isinstance(display, basestring): + if isinstance(display, six.string_types): display = converters.asbool(display) self.display = display self.logdir = logdir @@ -48,6 +49,8 @@ class CgitbMiddleware(object): [('content-type', 'text/html')], exc_info) response = self.exception_handler(exc_info, environ) + if six.PY3: + response = response.encode('utf8') return [response] def catching_iter(self, app_iter, environ): @@ -71,6 +74,8 @@ class CgitbMiddleware(object): response += ( '<hr noshade>Error in .close():<br>%s' % close_response) + if six.PY3: + response = response.encode('utf8') yield response def exception_handler(self, exc_info, environ): @@ -82,7 +87,7 @@ class CgitbMiddleware(object): format=self.format) hook(*exc_info) return dummy_file.getvalue() - + def make_cgitb_middleware(app, global_conf, display=NoDefault, logdir=None, @@ -91,7 +96,7 @@ def make_cgitb_middleware(app, global_conf, """ Wraps the application in the ``cgitb`` (standard library) error catcher. - + display: If true (or debug is set in the global configuration) then the traceback will be displayed in the browser diff --git a/paste/cowbell/__init__.py b/paste/cowbell/__init__.py index 43b7097..5a0d22d 100644 --- a/paste/cowbell/__init__.py +++ b/paste/cowbell/__init__.py @@ -45,7 +45,7 @@ function showSomewhere() { var sec, el; if (cowbellState == 'hidden') { el = document.getElementById('cowbell-ascending'); - lastCowbellPosition = [parseInt(Math.random()*(window.innerWidth-200)), + lastCowbellPosition = [parseInt(Math.random()*(window.innerWidth-200)), parseInt(Math.random()*(window.innerHeight-200))]; el.style.left = lastCowbellPosition[0] + 'px'; el.style.top = lastCowbellPosition[1] + 'px'; diff --git a/paste/debug/debugapp.py b/paste/debug/debugapp.py index 190cbdd..f752c36 100755 --- a/paste/debug/debugapp.py +++ b/paste/debug/debugapp.py @@ -17,7 +17,7 @@ class SimpleApplication(object): Produces a simple web page """ def __call__(self, environ, start_response): - body = "<html><body>simple</body></html>" + body = b"<html><body>simple</body></html>" start_response("200 OK", [('Content-Type', 'text/html'), ('Content-Length', str(len(body)))]) return [body] @@ -41,7 +41,7 @@ class SlowConsumer(object): remaining = int(total) while remaining > 0: if self.progress: - print "%s of %s remaining" % (remaining, total) + print("%s of %s remaining" % (remaining, total)) if remaining > 4096: chunk = environ['wsgi.input'].read(4096) else: @@ -59,7 +59,7 @@ class SlowConsumer(object): '<input type="file" name="file">\n' '<input type="submit" >\n' '</form></body></html>\n') - print "bingles" + print("bingles") start_response("200 OK", [('Content-Type', 'text/html'), ('Content-Length', len(body))]) return [body] diff --git a/paste/debug/doctest_webapp.py b/paste/debug/doctest_webapp.py index 935b291..ffcfaa7 100755 --- a/paste/debug/doctest_webapp.py +++ b/paste/debug/doctest_webapp.py @@ -8,10 +8,7 @@ These are functions for use when doctest-testing a document. """ -try: - import subprocess -except ImportError: - from paste.util import subprocess24 as subprocess +import subprocess import doctest import os import sys @@ -30,7 +27,7 @@ paste_parent = os.path.dirname( def run(command): data = run_raw(command) if data: - print data + print(data) def run_raw(command): """ @@ -56,7 +53,7 @@ def run_command(command, name, and_print=False): show_file('shell-command', name, description='shell transcript', data=data) if and_print and output: - print output + print(output) def _make_env(): env = os.environ.copy() @@ -88,7 +85,7 @@ def ls(dir=None, recurse=False, indent=0): full = os.path.join(dir, fn) if os.path.isdir(full): fn = fn + '/' - print ' '*indent + fn + print(' '*indent + fn) if os.path.isdir(full) and recurse: ls(dir=full, recurse=True, indent=indent+2) @@ -158,13 +155,13 @@ def show(path_info, example_name): expected = f.read() f.close() if not html_matches(expected, result): - print 'Pages did not match. Expected from %s:' % fn - print '-'*60 - print expected - print '='*60 - print 'Actual output:' - print '-'*60 - print result + print('Pages did not match. Expected from %s:' % fn) + print('-'*60) + print(expected) + print('='*60) + print('Actual output:') + print('-'*60) + print(result) def html_matches(pattern, text): regex = re.escape(pattern) @@ -214,7 +211,7 @@ def show_file(path, version, description=None, data=None): data = f.read() f.close() if ext == '.py': - html = ('<div class="source-code">%s</div>' + html = ('<div class="source-code">%s</div>' % PySourceColor.str2html(data, PySourceColor.dark)) else: html = '<pre class="source-code">%s</pre>' % cgi.escape(data, 1) @@ -241,7 +238,7 @@ def write_data(path, data): f = open(path, 'wb') f.write(data) f.close() - + def change_file(path, changes): f = open(os.path.abspath(path), 'rb') @@ -282,7 +279,7 @@ class LongFormDocTestParser(doctest.DocTestParser): (?![ ]*>>>) # Not a line starting with PS1 .*$\n? # But any other line )*)) - | + | (?: # This is for longer commands that are prefixed with a reST # comment like '.. run:' (two colons makes that a directive). # These commands cannot have any output. @@ -331,7 +328,7 @@ class LongFormDocTestParser(doctest.DocTestParser): # Get the example's indentation level. runner = m.group('run') or '' indent = len(m.group('%sindent' % runner)) - + # Divide source into lines; check that they're properly # indented; and then strip their indentation & prompts. source_lines = m.group('%ssource' % runner).split('\n') diff --git a/paste/debug/fsdiff.py b/paste/debug/fsdiff.py index 2849ea8..6f9ec2d 100644 --- a/paste/debug/fsdiff.py +++ b/paste/debug/fsdiff.py @@ -12,7 +12,13 @@ the file was. import os from fnmatch import fnmatch from datetime import datetime -from paste.util.UserDict24 import IterableUserDict + +try: + # Python 3 + import collections.UserDict as IterableUserDict +except ImportError: + # Python 2.5-2.7 + from UserDict import IterableUserDict import operator import re @@ -121,13 +127,6 @@ class Snapshot(IterableUserDict): return True return False - def _ignore_file(self, fn): - if fn in self.ignore_paths: - return True - if self.ignore_hidden and os.path.basename(fn).startswith('.'): - return True - return False - def _find_traverse(self, path, result): full = os.path.join(self.base_path, path) if os.path.isdir(full): @@ -253,8 +252,8 @@ class File(object): __tracebackhide__ = True bytes = self.bytes if s not in bytes: - print 'Could not find %r in:' % s - print bytes + print('Could not find %r in:' % s) + print(bytes) assert s in bytes def __repr__(self): @@ -288,7 +287,7 @@ class Dir(File): "Directory %r doesn't have content" % self) bytes = property(bytes__get) - + def _space_prefix(pref, full, sep=None, indent=None, include_sep=True): """ diff --git a/paste/debug/prints.py b/paste/debug/prints.py index d30fc8f..b660bfa 100644 --- a/paste/debug/prints.py +++ b/paste/debug/prints.py @@ -23,6 +23,7 @@ import cgi from paste.util import threadedprint from paste import wsgilib from paste import response +import six import sys _threadedprint_installed = False @@ -71,7 +72,7 @@ class PrintDebugMiddleware(object): # the entry point self.app = app self.force_content_type = force_content_type - if isinstance(print_wsgi_errors, basestring): + if isinstance(print_wsgi_errors, six.string_types): from paste.deploy.converters import asbool print_wsgi_errors = asbool(print_wsgi_errors) self.print_wsgi_errors = print_wsgi_errors @@ -131,7 +132,7 @@ class PrintDebugMiddleware(object): _body_re = re.compile(r'<body[^>]*>', re.I) _explicit_re = re.compile(r'<pre\s*[^>]*id="paste-debug-prints".*?>', re.I+re.S) - + def add_log(self, html, log): if not log: return html diff --git a/paste/debug/profile.py b/paste/debug/profile.py index 8e2d40a..470a54a 100644 --- a/paste/debug/profile.py +++ b/paste/debug/profile.py @@ -12,6 +12,7 @@ import hotshot import hotshot.stats import threading import cgi +import six import time from cStringIO import StringIO from paste import response @@ -99,7 +100,7 @@ def profile_decorator(**options): """ Profile a single function call. - + Used around a function, like:: @profile_decorator(options...) @@ -200,16 +201,16 @@ class DecoratedProfile(object): f.close() if exc_info: # We captured an exception earlier, now we re-raise it - raise exc_info[0], exc_info[1], exc_info[2] + six.reraise(exc_info[0], exc_info[1], exc_info[2]) return result - + def format_function(self, func, *args, **kw): args = map(repr, args) args.extend( ['%s=%r' % (k, v) for k, v in kw.items()]) return '%s(%s)' % (func.__name__, ', '.join(args)) - - + + def make_profile_middleware( app, global_conf, log_filename='profile.log.tmp', diff --git a/paste/debug/testserver.py b/paste/debug/testserver.py index 26c477a..8044c7c 100755 --- a/paste/debug/testserver.py +++ b/paste/debug/testserver.py @@ -28,7 +28,7 @@ class WSGIRegressionServer(WSGIServer): self.pending = [] self.timeout = self.defaulttimeout # this is a local connection, be quick - self.socket.settimeout(2) + self.socket.settimeout(2) def serve_forever(self): from threading import Thread thread = Thread(target=self.serve_pending) @@ -47,7 +47,7 @@ class WSGIRegressionServer(WSGIServer): if now > self.expires and self.timeout: # note regression test doesn't handle exceptions in # threads very well; so we just print and exit - print "\nWARNING: WSGIRegressionServer timeout exceeded\n" + print("\nWARNING: WSGIRegressionServer timeout exceeded\n") break if self.pending: self.handle_request() @@ -62,12 +62,12 @@ class WSGIRegressionServer(WSGIServer): def serve(application, host=None, port=None, handler=None): server = WSGIRegressionServer(application, host, port, handler) - print "serving on %s:%s" % server.server_address + print("serving on %s:%s" % server.server_address) server.serve_forever() return server if __name__ == '__main__': - import urllib + from six.moves.urllib.request import urlopen from paste.wsgilib import dump_environ server = serve(dump_environ) baseuri = ("http://%s:%s" % server.server_address) @@ -75,13 +75,13 @@ if __name__ == '__main__': def fetch(path): # tell the server to humor exactly one more request server.accept(1) - # not needed; but this is what you do if the server + # not needed; but this is what you do if the server # may not respond in a resonable time period import socket socket.setdefaulttimeout(5) # build a uri, fetch and return - return urllib.urlopen(baseuri + path).read() - + return urlopen(baseuri + path).read() + assert "PATH_INFO: /foo" in fetch("/foo") assert "PATH_INFO: /womble" in fetch("/womble") @@ -90,4 +90,4 @@ if __name__ == '__main__': # and then schedule a stop() server.stop() # and then... fetch it... - urllib.urlopen(baseuri) + urlopen(baseuri) diff --git a/paste/debug/watchthreads.py b/paste/debug/watchthreads.py index 1048213..b06ccea 100644 --- a/paste/debug/watchthreads.py +++ b/paste/debug/watchthreads.py @@ -123,7 +123,7 @@ page_template = HTMLTemplate(''' } return false ">▸ Show environ</a> - + <div id="environ-{{thread.thread_id}}" style="display: none"> {{if thread.environ:}} <table class="environ"> @@ -221,7 +221,7 @@ class WatchThreads(object): thread.uri_short = shorten(thread.uri) thread.environ = worker_environ thread.traceback = traceback_thread(thread_id) - + page = page_template.substitute( title="Thread Pool Worker Tracker", nworkers=nworkers, @@ -255,7 +255,7 @@ class WatchThreads(object): exc = httpexceptions.HTTPFound( headers=[('Location', script_name+'?kill=%s' % thread_id)]) return exc(environ, start_response) - + def traceback_thread(thread_id): """ Returns a plain-text traceback of the given thread, or None if it @@ -290,13 +290,13 @@ def format_environ(environ): environ_template.substitute( key=cgi.escape(str(key)), value=cgi.escape(str(value)))) - except Exception, e: + except Exception as e: environ_rows.append( environ_template.substitute( key=cgi.escape(str(key)), value='Error in <code>repr()</code>: %s' % e)) return ''.join(environ_rows) - + def format_time(time_length): if time_length >= 60*60: # More than an hour @@ -339,7 +339,7 @@ def make_bad_app(global_conf, pause=0): else: count = 0 while 1: - print "I'm alive %s (%s)" % (count, thread.get_ident()) + print("I'm alive %s (%s)" % (count, thread.get_ident())) time.sleep(10) count += 1 start_response('200 OK', [('content-type', 'text/plain')]) diff --git a/paste/debug/wdg_validate.py b/paste/debug/wdg_validate.py index d3678fb..225baf9 100644 --- a/paste/debug/wdg_validate.py +++ b/paste/debug/wdg_validate.py @@ -6,10 +6,7 @@ Middleware that tests the validity of all generated HTML using the """ from cStringIO import StringIO -try: - import subprocess -except ImportError: - from paste.util import subprocess24 as subprocess +import subprocess from paste.response import header_value import re import cgi diff --git a/paste/errordocument.py b/paste/errordocument.py index e57c162..34f2d4a 100644 --- a/paste/errordocument.py +++ b/paste/errordocument.py @@ -11,10 +11,11 @@ URL where the content can be displayed to the user as an error document. import warnings import sys -from urlparse import urlparse +from six.moves.urllib import parse as urlparse from paste.recursive import ForwardRequestException, RecursiveMiddleware, RecursionLoop from paste.util import converters from paste.response import replace_header +import six def forward(app, codes): """ @@ -46,7 +47,7 @@ def forward(app, codes): '%s is not valid'%repr(code)) def error_codes_mapper(code, message, environ, global_conf, codes): - if codes.has_key(code): + if code in codes: return codes[code] else: return None @@ -84,11 +85,17 @@ class StatusKeeper(object): #raise Exception(self.url, self.status) try: return self.app(environ, keep_status_start_response) - except RecursionLoop, e: - environ['wsgi.errors'].write('Recursion error getting error page: %s\n' % e) + except RecursionLoop as e: + line = 'Recursion error getting error page: %s\n' % e + if six.PY3: + line = line.encode('utf8') + environ['wsgi.errors'].write(line) keep_status_start_response('500 Server Error', [('Content-type', 'text/plain')], sys.exc_info()) - return ['Error: %s. (Error page could not be fetched)' - % self.status] + body = ('Error: %s. (Error page could not be fetched)' + % self.status) + if six.PY3: + body = body.encode('utf8') + return [body] class StatusBasedForward(object): @@ -161,7 +168,6 @@ class StatusBasedForward(object): def __call__(self, environ, start_response): url = [] - writer = [] def change_response(status, headers, exc_info=None): status_code = status.split(' ') @@ -342,7 +348,7 @@ class _StatusBasedRedirect(object): new_environ = {} for k, v in environ.items(): if k != 'QUERY_STRING': - new_environ['QUERY_STRING'] = urlparse(url_)[4] + new_environ['QUERY_STRING'] = urlparse.urlparse(url_)[4] else: new_environ[k] = v class InvalidForward(Exception): @@ -363,7 +369,7 @@ class _StatusBasedRedirect(object): forward.start_response = eat_start_response try: app_iter = forward(url_, new_environ) - except InvalidForward, e: + except InvalidForward: code, message = code_message[0] environ['wsgi.errors'].write( 'Error occurred in ' diff --git a/paste/evalexception/evalcontext.py b/paste/evalexception/evalcontext.py index dca2a97..42f2efa 100644 --- a/paste/evalexception/evalcontext.py +++ b/paste/evalexception/evalcontext.py @@ -1,9 +1,10 @@ # (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org) # Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php -from cStringIO import StringIO +from six.moves import cStringIO as StringIO import traceback import threading import pdb +import six import sys exec_lock = threading.Lock() @@ -34,7 +35,7 @@ class EvalContext(object): sys.stdout = out try: code = compile(s, '<web>', "single", 0, 1) - exec code in self.namespace, self.globs + six.exec_(code, self.globs, self.namespace) debugger.set_continue() except KeyboardInterrupt: raise diff --git a/paste/evalexception/middleware.py b/paste/evalexception/middleware.py index 4349b88..da7876d 100644 --- a/paste/evalexception/middleware.py +++ b/paste/evalexception/middleware.py @@ -24,11 +24,15 @@ can look for the header X-Debug-URL in your 500 responses if you want to see the full debuggable traceback. Also, this URL is printed to ``wsgi.errors``, so you can open it up in another browser window. """ + +from __future__ import print_function + import sys import os import cgi import traceback -from cStringIO import StringIO +import six +from six.moves import cStringIO as StringIO import pprint import itertools import time @@ -40,7 +44,7 @@ from paste import httpexceptions from paste import registry from paste import request from paste import response -import evalcontext +from paste.evalexception import evalcontext limit = 200 @@ -146,7 +150,7 @@ def get_debug_info(func): % debugcount) debug_info = self.debug_infos[debugcount] return func(self, debug_info=debug_info, **form) - except ValueError, e: + except ValueError as e: form['headers']['status'] = '500 Server Error' return '<html>There was an error: %s</html>' % html_quote(e) return debug_info_replacement @@ -159,7 +163,7 @@ def get_debug_count(environ): if 'paste.evalexception.debug_count' in environ: return environ['paste.evalexception.debug_count'] else: - environ['paste.evalexception.debug_count'] = next = debug_counter.next() + environ['paste.evalexception.debug_count'] = next = six.next(debug_counter) return next class EvalException(object): @@ -328,7 +332,10 @@ class EvalException(object): start_response('500 Internal Server Error', headers, exc_info) - environ['wsgi.errors'].write('Debug at: %s\n' % view_uri) + msg = 'Debug at: %s\n' % view_uri + if six.PY3: + msg = msg.encode('utf8') + environ['wsgi.errors'].write(msg) exc_data = collector.collect_exception(*exc_info) debug_info = DebugInfo(count, exc_info, exc_data, base_path, @@ -337,7 +344,7 @@ class EvalException(object): self.debug_infos[count] = debug_info if self.xmlhttp_key: - get_vars = wsgilib.parse_querystring(environ) + get_vars = request.parse_querystring(environ) if dict(get_vars).get(self.xmlhttp_key): exc_data = collector.collect_exception(*exc_info) html = formatter.format_html( @@ -351,7 +358,7 @@ class EvalException(object): def exception_handler(self, exc_info, environ): simple_html_error = False if self.xmlhttp_key: - get_vars = wsgilib.parse_querystring(environ) + get_vars = request.parse_querystring(environ) if dict(get_vars).get(self.xmlhttp_key): simple_html_error = True return errormiddleware.handle_exception( @@ -398,8 +405,7 @@ class DebugInfo(object): if id(frame) == tbid: return frame else: - raise ValueError, ( - "No frame by id %s found from %r" % (tbid, self.frames)) + raise ValueError("No frame by id %s found from %r" % (tbid, self.frames)) def wsgi_application(self, environ, start_response): start_response('200 OK', [('content-type', 'text/html')]) @@ -414,6 +420,8 @@ class DebugInfo(object): 'repost_button': repost_button or '', 'head_html': head_html, 'body': html} + if six.PY3: + page = page.encode('utf8') return [page] def eval_javascript(self): @@ -457,8 +465,8 @@ def make_table(items): out = StringIO() try: pprint.pprint(value, out) - except Exception, e: - print >> out, 'Error: %s' % e + except Exception as e: + print('Error: %s' % e, file=out) value = html_quote(out.getvalue()) if len(value) > 100: # @@: This can actually break the HTML :( diff --git a/paste/exceptions/collector.py b/paste/exceptions/collector.py index 65f9ec0..8867bf7 100644 --- a/paste/exceptions/collector.py +++ b/paste/exceptions/collector.py @@ -23,10 +23,7 @@ supplements import sys import traceback import time -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from six.moves import cStringIO as StringIO import linecache from paste.exceptions import serial_number_generator import warnings @@ -95,7 +92,7 @@ class ExceptionCollector(object): The actually interpretation of these values is largely up to the reporters and formatters. - + ``collect_exception(*sys.exc_info())`` will return an object with several attributes: @@ -116,7 +113,7 @@ class ExceptionCollector(object): can refer to the exception later. (@@: should it include a portion that allows identification of the specific instance of the exception as well?) - + The list of frames goes innermost first. Each frame has these attributes; some values may be None if they could not be determined. @@ -143,7 +140,7 @@ class ExceptionCollector(object): the value of any ``__traceback_hide__`` variable ``traceback_log``: the value of any ``__traceback_log__`` variable - + ``__traceback_supplement__`` is thrown away, but a fixed set of attributes are captured; each of these attributes is @@ -192,7 +189,7 @@ class ExceptionCollector(object): hide frames that are part of the 'framework' or underlying system. There are a variety of rules about special values for this variables that formatters should be aware of. - + TODO: More attributes in __traceback_supplement__? Maybe an attribute @@ -284,10 +281,10 @@ class ExceptionCollector(object): data['tbid'] = id(tb) # Output a traceback supplement, if any. - if locals.has_key('__traceback_supplement__'): + if '__traceback_supplement__' in locals: # Use the supplement defined in the function. tbs = locals['__traceback_supplement__'] - elif globals.has_key('__traceback_supplement__'): + elif '__traceback_supplement__' in globals: # Use the supplement defined in the module. # This is used by Scripts (Python). tbs = globals['__traceback_supplement__'] @@ -506,7 +503,7 @@ class ExceptionFrame(Bunch): for lineno in range(self.lineno-context, self.lineno+context+1): lines.append(linecache.getline(self.filename, lineno)) return ''.join(lines) - + if hasattr(sys, 'tracebacklimit'): limit = min(limit, sys.tracebacklimit) @@ -515,7 +512,7 @@ col = ExceptionCollector() def collect_exception(t, v, tb, limit=None): """ Collection an exception from ``sys.exc_info()``. - + Use like:: try: diff --git a/paste/exceptions/errormiddleware.py b/paste/exceptions/errormiddleware.py index 784414f..95c1261 100644 --- a/paste/exceptions/errormiddleware.py +++ b/paste/exceptions/errormiddleware.py @@ -7,13 +7,11 @@ Error handler middleware import sys import traceback import cgi -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +from six.moves import cStringIO as StringIO from paste.exceptions import formatter, collector, reporter from paste import wsgilib from paste import request +import six __all__ = ['ErrorMiddleware', 'handle_exception'] @@ -26,7 +24,7 @@ class ErrorMiddleware(object): """ Error handling middleware - + Usage:: error_catching_wsgi_app = ErrorMiddleware(wsgi_app) @@ -37,14 +35,14 @@ class ErrorMiddleware(object): If true, then tracebacks will be shown in the browser. ``error_email``: - an email address (or list of addresses) to send exception + an email address (or list of addresses) to send exception reports to ``error_log``: a filename to append tracebacks to ``show_exceptions_in_wsgi_errors``: - If true, then errors will be printed to ``wsgi.errors`` + If true, then errors will be printed to ``wsgi.errors`` (frequently a server error log, or stderr). ``from_address``, ``smtp_server``, ``error_subject_prefix``, ``smtp_username``, ``smtp_password``, ``smtp_use_tls``: @@ -60,7 +58,7 @@ class ErrorMiddleware(object): HTML page. Environment Configuration: - + ``paste.throw_errors``: If this setting in the request environment is true, then this middleware is disabled. This can be useful in a testing situation @@ -68,10 +66,10 @@ class ErrorMiddleware(object): ``paste.expected_exceptions``: When this middleware encounters an exception listed in this - environment variable and when the ``start_response`` has not + environment variable and when the ``start_response`` has not yet occurred, the exception will be re-raised instead of being - caught. This should generally be set by middleware that may - (but probably shouldn't be) installed above this middleware, + caught. This should generally be set by middleware that may + (but probably shouldn't be) installed above this middleware, and wants to get certain exceptions. Exceptions raised after ``start_response`` have been called are always caught since by definition they are no longer expected. @@ -126,7 +124,7 @@ class ErrorMiddleware(object): if xmlhttp_key is None: xmlhttp_key = global_conf.get('xmlhttp_key', '_') self.xmlhttp_key = xmlhttp_key - + def __call__(self, environ, start_response): """ The WSGI application interface. @@ -154,6 +152,8 @@ class ErrorMiddleware(object): exc_info) # @@: it would be nice to deal with bad content types here response = self.exception_handler(exc_info, environ) + if six.PY3: + response = response.encode('utf8') return [response] finally: # clean up locals... @@ -168,7 +168,7 @@ class ErrorMiddleware(object): def exception_handler(self, exc_info, environ): simple_html_error = False if self.xmlhttp_key: - get_vars = wsgilib.parse_querystring(environ) + get_vars = request.parse_querystring(environ) if dict(get_vars).get(self.xmlhttp_key): simple_html_error = True return handle_exception( @@ -245,7 +245,10 @@ class CatchingIter(object): [('content-type', 'text/html')], exc_info) + if six.PY3: + response = response.encode('utf8') return response + __next__ = next def close(self): # This should at least print something to stderr if the @@ -315,7 +318,7 @@ class Supplement(object): (1, 0, 1): 'CGI', (1, 1, 1): 'Multi thread/process CGI (?)', } - + def handle_exception(exc_info, error_stream, html=True, debug_mode=False, error_email=None, @@ -323,8 +326,8 @@ def handle_exception(exc_info, error_stream, html=True, show_exceptions_in_wsgi_errors=False, error_email_from='errors@localhost', smtp_server='localhost', - smtp_username=None, - smtp_password=None, + smtp_username=None, + smtp_password=None, smtp_use_tls=False, error_subject_prefix='', error_message=None, @@ -381,8 +384,11 @@ def handle_exception(exc_info, error_stream, html=True, else: reported = True else: - error_stream.write('Error - %s: %s\n' % ( - exc_data.exception_type, exc_data.exception_value)) + line = ('Error - %s: %s\n' + % (exc_data.exception_type, exc_data.exception_value)) + if six.PY3: + line = line.encode('utf8') + error_stream.write(line) if html: if debug_mode and simple_html_error: return_error = formatter.format_html( diff --git a/paste/exceptions/formatter.py b/paste/exceptions/formatter.py index e1fadbe..c83ab50 100644 --- a/paste/exceptions/formatter.py +++ b/paste/exceptions/formatter.py @@ -8,6 +8,7 @@ Formatters for the exception data that comes from ExceptionCollector. # Use this: http://www.zope.org/Members/tino/VisualTraceback/VisualTracebackNews import cgi +import six import re from paste.util import PySourceColor @@ -76,7 +77,7 @@ class AbstractFormatter(object): lines.append(self.format_long_source( source, long_source)) etype = exc_data.exception_type - if not isinstance(etype, basestring): + if not isinstance(etype, six.string_types): etype = etype.__name__ exc_info = self.format_exception_info( etype, @@ -195,7 +196,7 @@ class TextFormatter(AbstractFormatter): '%s: %s' % (self.quote(etype), self.quote(evalue))) def format_traceback_info(self, info): return info - + def format_combine(self, data_by_importance, lines, exc_info): lines[:0] = [value for n, value in data_by_importance['important']] lines.append(exc_info) @@ -220,7 +221,7 @@ class TextFormatter(AbstractFormatter): for n, v in items: try: v = repr(v) - except Exception, e: + except Exception as e: v = 'Cannot display: %s' % e v = truncate(v) lines.append(' %s: %s' % (n, v)) @@ -310,7 +311,7 @@ class HTMLFormatter(TextFormatter): for name, value in rows: try: value = repr(value) - except Exception, e: + except Exception as e: value = 'Cannot print: %s' % e odd = not odd table.append( @@ -379,7 +380,7 @@ function switch_source(el, hide_type) { } </script>''' - + error_css = """ <style type="text/css"> @@ -463,7 +464,7 @@ def format_html(exc_data, include_hidden_frames=False, **ops): <textarea style="width: 100%%" rows=10 cols=60>%s</textarea> </div> """ % (short_er, long_er, cgi.escape(text_er)) - + def format_text(exc_data, **ops): return TextFormatter(**ops).format_collected_data(exc_data) diff --git a/paste/exceptions/reporter.py b/paste/exceptions/reporter.py index 95e31ba..7c0c266 100644 --- a/paste/exceptions/reporter.py +++ b/paste/exceptions/reporter.py @@ -1,8 +1,8 @@ # (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org) # Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php -from email.MIMEText import MIMEText -from email.MIMEMultipart import MIMEMultipart +from email.mime.text import MIMEText +from email.mime.multipart import MIMEMultipart import smtplib import time try: diff --git a/paste/exceptions/serial_number_generator.py b/paste/exceptions/serial_number_generator.py index 0289663..3f80107 100644 --- a/paste/exceptions/serial_number_generator.py +++ b/paste/exceptions/serial_number_generator.py @@ -13,6 +13,8 @@ try: except ImportError: from md5 import md5 +import six + good_characters = "23456789abcdefghjkmnpqrtuvwxyz" base = len(good_characters) @@ -21,7 +23,7 @@ def make_identifier(number): """ Encodes a number as an identifier. """ - if not isinstance(number, (int, long)): + if not isinstance(number, six.integer_types): raise ValueError( "You can only make identifiers out of integers (not %r)" % number) @@ -34,7 +36,7 @@ def make_identifier(number): next = number % base result.append(good_characters[next]) # Note, this depends on integer rounding of results: - number = number / base + number = number // base return ''.join(result) def hash_identifier(s, length, pad=True, hasher=md5, prefix='', @@ -56,17 +58,21 @@ def hash_identifier(s, length, pad=True, hasher=md5, prefix='', # Accept sha/md5 modules as well as callables hasher = hasher.new if length > 26 and hasher is md5: - raise ValueError, ( + raise ValueError( "md5 cannot create hashes longer than 26 characters in " "length (you gave %s)" % length) - if isinstance(s, unicode): + if isinstance(s, six.text_type): s = s.encode('utf-8') - h = hasher(str(s)) + elif not isinstance(s, six.binary_type): + s = str(s) + if six.PY3: + s = s.encode('utf-8') + h = hasher(s) bin_hash = h.digest() modulo = base ** length number = 0 for c in list(bin_hash): - number = (number * 256 + ord(c)) % modulo + number = (number * 256 + six.byte2int([c])) % modulo ident = make_identifier(number) if pad: ident = good_characters[0]*(length-len(ident)) + ident @@ -120,4 +126,4 @@ __test__ = { if __name__ == '__main__': import doctest doctest.testmod() - + diff --git a/paste/fileapp.py b/paste/fileapp.py index 8432511..e18281a 100644 --- a/paste/fileapp.py +++ b/paste/fileapp.py @@ -122,8 +122,8 @@ class DataApp(object): for head in list_headers(entity=True): head.delete(headers) start_response('304 Not Modified', headers) - return [''] - except HTTPBadRequest, exce: + return [b''] + except HTTPBadRequest as exce: return exce.wsgi_application(environ, start_response) # If we get If-None-Match and If-Modified-Since, and @@ -133,13 +133,14 @@ class DataApp(object): if not client_etags: try: client_clock = IF_MODIFIED_SINCE.parse(environ) - if client_clock >= int(self.last_modified): + if (client_clock is not None + and client_clock >= int(self.last_modified)): # horribly inefficient, n^2 performance, yuck! for head in list_headers(entity=True): head.delete(headers) start_response('304 Not Modified', headers) - return [''] # empty body - except HTTPBadRequest, exce: + return [b''] # empty body + except HTTPBadRequest as exce: return exce.wsgi_application(environ, start_response) (lower, upper) = (0, self.content_length - 1) @@ -158,10 +159,10 @@ class DataApp(object): CONTENT_RANGE.update(headers, first_byte=lower, last_byte=upper, total_length = self.content_length) CONTENT_LENGTH.update(headers, content_length) - if content_length == self.content_length: - start_response('200 OK', headers) - else: + if range or content_length != self.content_length: start_response('206 Partial Content', headers) + else: + start_response('200 OK', headers) if self.content is not None: return [self.content[lower:upper+1]] return (lower, content_length) @@ -215,7 +216,7 @@ class FileApp(DataApp): return exc(environ, start_response) try: file = open(self.filename, 'rb') - except (IOError, OSError), e: + except (IOError, OSError) as e: exc = HTTPForbidden( 'You are not permitted to view this file (%s)' % e) return exc.wsgi_application( @@ -224,11 +225,11 @@ class FileApp(DataApp): if isinstance(retval, list): # cached content, exception, or not-modified if is_head: - return [''] + return [b''] return retval (lower, content_length) = retval if is_head: - return [''] + return [b''] file.seek(lower) file_wrapper = environ.get('wsgi.file_wrapper', None) if file_wrapper: @@ -256,6 +257,7 @@ class _FileIter(object): if not data: raise StopIteration return data + __next__ = next def close(self): self.file.close() diff --git a/paste/fixture.py b/paste/fixture.py index 242a1de..df1c75d 100644 --- a/paste/fixture.py +++ b/paste/fixture.py @@ -9,27 +9,30 @@ for testing WSGI applications, and the `TestFileEnvironment effects of command-line scripts. """ +from __future__ import print_function + import sys import random -import urllib -import urlparse import mimetypes import time -import cgi import os import shutil import smtplib import shlex -from Cookie import BaseCookie -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO import re +import six +import subprocess +from six.moves import cStringIO as StringIO +from six.moves.urllib.parse import urlencode +from six.moves.urllib import parse as urlparse try: - import subprocess + # Python 3 + from http.cookies import BaseCookie + from urllib.parse import splittype, splithost except ImportError: - from paste.util import subprocess24 as subprocess + # Python 2 + from Cookie import BaseCookie + from urllib import splittype, splithost from paste import wsgilib from paste import lint @@ -125,7 +128,7 @@ class TestApp(object): ``post_request_hook`` is a function, similar to ``pre_request_hook``, to be called after requests are made. """ - if isinstance(app, (str, unicode)): + if isinstance(app, (six.binary_type, six.text_type)): from paste.deploy import loadapp # @@: Should pick up relative_to from calling module's # __file__ @@ -189,8 +192,8 @@ class TestApp(object): # Hide from py.test: __tracebackhide__ = True if params: - if not isinstance(params, (str, unicode)): - params = urllib.urlencode(params, doseq=True) + if not isinstance(params, (six.binary_type, six.text_type)): + params = urlencode(params, doseq=True) if '?' in url: url += '&' else: @@ -207,7 +210,7 @@ class TestApp(object): req = TestRequest(url, environ, expect_errors) return self.do_request(req, status=status) - def _gen_request(self, method, url, params='', headers=None, extra_environ=None, + def _gen_request(self, method, url, params=b'', headers=None, extra_environ=None, status=None, upload_files=None, expect_errors=False): """ Do a generic request. @@ -219,12 +222,14 @@ class TestApp(object): environ = self._make_environ() # @@: Should this be all non-strings? if isinstance(params, (list, tuple, dict)): - params = urllib.urlencode(params) + params = urlencode(params) if hasattr(params, 'items'): # Some other multi-dict like format - params = urllib.urlencode(params.items()) + params = urlencode(params.items()) + if six.PY3: + params = params.encode('utf8') if upload_files: - params = cgi.parse_qsl(params, keep_blank_values=True) + params = urlparse.parse_qsl(params, keep_blank_values=True) content_type, params = self.encode_multipart( params, upload_files) environ['CONTENT_TYPE'] = content_type @@ -236,13 +241,13 @@ class TestApp(object): environ['QUERY_STRING'] = '' environ['CONTENT_LENGTH'] = str(len(params)) environ['REQUEST_METHOD'] = method - environ['wsgi.input'] = StringIO(params) + environ['wsgi.input'] = six.BytesIO(params) self._set_headers(headers, environ) environ.update(extra_environ) req = TestRequest(url, environ, expect_errors) return self.do_request(req, status=status) - def post(self, url, params='', headers=None, extra_environ=None, + def post(self, url, params=b'', headers=None, extra_environ=None, status=None, upload_files=None, expect_errors=False): """ Do a POST request. Very like the ``.get()`` method. @@ -261,7 +266,7 @@ class TestApp(object): upload_files=upload_files, expect_errors=expect_errors) - def put(self, url, params='', headers=None, extra_environ=None, + def put(self, url, params=b'', headers=None, extra_environ=None, status=None, upload_files=None, expect_errors=False): """ Do a PUT request. Very like the ``.get()`` method. @@ -280,7 +285,7 @@ class TestApp(object): upload_files=upload_files, expect_errors=expect_errors) - def delete(self, url, params='', headers=None, extra_environ=None, + def delete(self, url, params=b'', headers=None, extra_environ=None, status=None, expect_errors=False): """ Do a DELETE request. Very like the ``.get()`` method. @@ -318,26 +323,41 @@ class TestApp(object): typical POST body, returning the (content_type, body). """ boundary = '----------a_BoUnDaRy%s$' % random.random() + content_type = 'multipart/form-data; boundary=%s' % boundary + if six.PY3: + boundary = boundary.encode('ascii') + lines = [] for key, value in params: - lines.append('--'+boundary) - lines.append('Content-Disposition: form-data; name="%s"' % key) - lines.append('') - lines.append(value) + lines.append(b'--'+boundary) + line = 'Content-Disposition: form-data; name="%s"' % key + if six.PY3: + line = line.encode('utf8') + lines.append(line) + lines.append(b'') + line = value + if six.PY3 and isinstance(line, six.text_type): + line = line.encode('utf8') + lines.append(line) for file_info in files: key, filename, value = self._get_file_info(file_info) - lines.append('--'+boundary) - lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' + lines.append(b'--'+boundary) + line = ('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename)) + if six.PY3: + line = line.encode('utf8') + lines.append(line) fcontent = mimetypes.guess_type(filename)[0] - lines.append('Content-Type: %s' % - fcontent or 'application/octet-stream') - lines.append('') + line = ('Content-Type: %s' + % (fcontent or 'application/octet-stream')) + if six.PY3: + line = line.encode('utf8') + lines.append(line) + lines.append(b'') lines.append(value) - lines.append('--' + boundary + '--') - lines.append('') - body = '\r\n'.join(lines) - content_type = 'multipart/form-data; boundary=%s' % boundary + lines.append(b'--' + boundary + b'--') + lines.append(b'') + body = b'\r\n'.join(lines) return content_type, body def _get_file_info(self, file_info): @@ -433,10 +453,13 @@ class TestApp(object): if status is None: if res.status >= 200 and res.status < 400: return + body = res.body + if six.PY3: + body = body.decode('utf8', 'xmlcharrefreplace') raise AppError( "Bad response: %s (not 200 OK or 3xx redirect for %s)\n%s" % (res.full_status, res.request.url, - res.body)) + body)) if status != res.status: raise AppError( "Bad response: %s (not %s)" % (res.full_status, status)) @@ -446,7 +469,8 @@ class TestApp(object): raise AppError( "Application had errors logged:\n%s" % res.errors) - def _make_response(self, (status, headers, body, errors), total_time): + def _make_response(self, resp, total_time): + status, headers, body, errors = resp return TestResponse(self, status, headers, body, errors, total_time) @@ -597,8 +621,8 @@ class TestResponse(object): "You can only follow redirect responses (not %s)" % self.full_status) location = self.header('location') - type, rest = urllib.splittype(location) - host, path = urllib.splithost(rest) + type, rest = splittype(location) + host, path = splithost(rest) # @@: We should test that it's not a remote redirect return self.test_app.get(location, **kw) @@ -685,7 +709,7 @@ class TestResponse(object): def printlog(s): if verbose: - print s + print(s) found_links = [] total_links = 0 @@ -771,12 +795,12 @@ class TestResponse(object): method = self.test_app.post return method(href, **args) - _normal_body_regex = re.compile(r'[ \n\r\t]+') + _normal_body_regex = re.compile(br'[ \n\r\t]+') def normal_body__get(self): if self._normal_body is None: self._normal_body = self._normal_body_regex.sub( - ' ', self.body) + b' ', self.body) return self._normal_body normal_body = property(normal_body__get, @@ -790,9 +814,9 @@ class TestResponse(object): of the response. Whitespace is normalized when searching for a string. """ - if not isinstance(s, (str, unicode)): + if not isinstance(s, (six.binary_type, six.text_type)): s = str(s) - if isinstance(s, unicode): + if isinstance(s, six.text_type): ## FIXME: we don't know that this response uses utf8: s = s.encode('utf8') return (self.body.find(s) != -1 @@ -810,7 +834,7 @@ class TestResponse(object): if 'no' in kw: no = kw['no'] del kw['no'] - if isinstance(no, basestring): + if isinstance(no, (six.binary_type, six.text_type)): no = [no] else: no = [] @@ -819,23 +843,29 @@ class TestResponse(object): "The only keyword argument allowed is 'no'") for s in strings: if not s in self: - print >> sys.stderr, "Actual response (no %r):" % s - print >> sys.stderr, self + print("Actual response (no %r):" % s, file=sys.stderr) + print(self, file=sys.stderr) raise IndexError( "Body does not contain string %r" % s) for no_s in no: if no_s in self: - print >> sys.stderr, "Actual response (has %r)" % no_s - print >> sys.stderr, self + print("Actual response (has %r)" % no_s, file=sys.stderr) + print(self, file=sys.stderr) raise IndexError( "Body contains string %r" % s) def __repr__(self): - return '<Response %s %r>' % (self.full_status, self.body[:20]) + body = self.body + if six.PY3: + body = body.decode('utf8', 'xmlcharrefreplace') + body = body[:20] + return '<Response %s %r>' % (self.full_status, body) def __str__(self): - simple_body = '\n'.join([l for l in self.body.splitlines() + simple_body = b'\n'.join([l for l in self.body.splitlines() if l.strip()]) + if six.PY3: + simple_body = simple_body.decode('utf8', 'xmlcharrefreplace') return 'Response: %s\n%s\n%s' % ( self.status, '\n'.join(['%s: %s' % (n, v) for n, v in self.headers]), @@ -1382,7 +1412,7 @@ class TestFileEnvironment(object): cwd = _popget(kw, 'cwd', self.cwd) stdin = _popget(kw, 'stdin', None) printresult = _popget(kw, 'printresult', True) - args = map(str, args) + args = list(map(str, args)) assert not kw, ( "Arguments not expected: %s" % ', '.join(kw.keys())) if ' ' in script: @@ -1407,8 +1437,8 @@ class TestFileEnvironment(object): files_before=files_before, files_after=files_after) if printresult: - print result - print '-'*40 + print(result) + print('-'*40) if not expect_error: result.assert_no_error() if not expect_stderr: @@ -1534,8 +1564,8 @@ class ProcResult(object): def assert_no_stderr(self): __tracebackhide__ = True if self.stderr: - print 'Error output:' - print self.stderr + print('Error output:') + print(self.stderr) raise AssertionError("stderr output not expected") def __str__(self): @@ -1615,11 +1645,11 @@ class FoundFile(object): def mustcontain(self, s): __tracebackhide__ = True - bytes = self.bytes - if s not in bytes: - print 'Could not find %r in:' % s - print bytes - assert s in bytes + bytes_ = self.bytes + if s not in bytes_: + print('Could not find %r in:' % s) + print(bytes_) + assert s in bytes_ def __repr__(self): return '<%s %s:%s>' % ( @@ -1683,7 +1713,7 @@ def _space_prefix(pref, full, sep=None, indent=None, include_sep=True): def _make_pattern(pat): if pat is None: return None - if isinstance(pat, (str, unicode)): + if isinstance(pat, (six.binary_type, six.text_type)): pat = re.compile(pat) if hasattr(pat, 'search'): return pat.search @@ -1709,7 +1739,7 @@ def setup_module(module=None): if module is None: # The module we were called from must be the module... module = sys._getframe().f_back.f_globals['__name__'] - if isinstance(module, (str, unicode)): + if isinstance(module, (six.binary_type, six.text_type)): module = sys.modules[module] if hasattr(module, 'reset_state'): module.reset_state() diff --git a/paste/flup_session.py b/paste/flup_session.py index b230ab8..6f5c750 100644 --- a/paste/flup_session.py +++ b/paste/flup_session.py @@ -68,7 +68,7 @@ class SessionMiddleware(object): if cookie_name is NoDefault: cookie_name = global_conf.get('session_cookie', '_SID_') self.cookie_name = cookie_name - + def __call__(self, environ, start_response): service = flup_session.SessionService( self.store, environ, cookieName=self.cookie_name, @@ -81,7 +81,7 @@ class SessionMiddleware(object): try: app_iter = self.application(environ, cookie_start_response) - except httpexceptions.HTTPException, e: + except httpexceptions.HTTPException as e: headers = (e.headers or {}).items() service.addCookie(headers) e.headers = dict(headers) @@ -92,7 +92,7 @@ class SessionMiddleware(object): raise return wsgilib.add_close(app_iter, service.close) - + def make_session_middleware(app, global_conf, session_type=NoDefault, cookie_name=NoDefault, diff --git a/paste/gzipper.py b/paste/gzipper.py index 1431490..eca8775 100644 --- a/paste/gzipper.py +++ b/paste/gzipper.py @@ -13,11 +13,7 @@ Gzip-encodes the response. import gzip from paste.response import header_value, remove_header from paste.httpheaders import CONTENT_LENGTH - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO +import six class GzipOutput(object): pass @@ -46,7 +42,7 @@ class GzipResponse(object): def __init__(self, start_response, compress_level): self.start_response = start_response self.compress_level = compress_level - self.buffer = StringIO() + self.buffer = six.BytesIO() self.compressible = False self.content_length = None diff --git a/paste/httpexceptions.py b/paste/httpexceptions.py index 85e7c84..0b68c2d 100644 --- a/paste/httpexceptions.py +++ b/paste/httpexceptions.py @@ -74,7 +74,7 @@ References: """ -import types +import six from paste.wsgilib import catch_errors_app from paste.response import has_header, header_value, replace_header from paste.request import resolve_relative_url @@ -178,9 +178,9 @@ class HTTPException(Exception): assert isinstance(headers, (type(None), list)), ( "headers must be None or a list: %r" % headers) - assert isinstance(detail, (type(None), basestring)), ( + assert isinstance(detail, (type(None), six.binary_type, six.text_type)), ( "detail must be None or a string: %r" % detail) - assert isinstance(comment, (type(None), basestring)), ( + assert isinstance(comment, (type(None), six.binary_type, six.text_type)), ( "comment must be None or a string: %r" % comment) self.headers = headers or tuple() for req in self.required_headers: @@ -206,9 +206,10 @@ class HTTPException(Exception): if self.headers: for (k, v) in self.headers: args[k.lower()] = escfunc(v) - for key, value in args.items(): - if isinstance(value, unicode): - args[key] = value.encode('utf8', 'xmlcharrefreplace') + if six.PY2: + for key, value in args.items(): + if isinstance(value, six.text_type): + args[key] = value.encode('utf8', 'xmlcharrefreplace') return template % args def plain(self, environ): @@ -237,7 +238,7 @@ class HTTPException(Exception): else: replace_header(headers, 'content-type', 'text/plain') content = self.plain(environ) - if isinstance(content, unicode): + if isinstance(content, six.text_type): content = content.encode('utf8') cur_content_type = ( header_value(headers, 'content-type') @@ -593,8 +594,8 @@ class HTTPVersionNotSupported(HTTPServerError): __all__ = ['HTTPException', 'HTTPRedirection', 'HTTPError' ] _exceptions = {} -for name, value in globals().items(): - if (isinstance(value, (type, types.ClassType)) and +for name, value in six.iteritems(dict(globals())): + if (isinstance(value, (type, six.class_types)) and issubclass(value, HTTPException) and value.code): _exceptions[value.code] = value @@ -637,7 +638,7 @@ class HTTPExceptionHandler(object): []).append(HTTPException) try: return self.application(environ, start_response) - except HTTPException, exc: + except HTTPException as exc: return exc(environ, start_response) def middleware(*args, **kw): diff --git a/paste/httpheaders.py b/paste/httpheaders.py index 728fa39..5457138 100644 --- a/paste/httpheaders.py +++ b/paste/httpheaders.py @@ -135,11 +135,18 @@ dashes to give CamelCase style names. """ import mimetypes -import urllib2 -import re -from rfc822 import formatdate, parsedate_tz, mktime_tz +import six from time import time as now -from httpexceptions import HTTPBadRequest +try: + # Python 3 + from email.utils import formatdate, parsedate_tz, mktime_tz + from urllib.request import AbstractDigestAuthHandler, parse_keqv_list, parse_http_list +except ImportError: + # Python 2 + from rfc822 import formatdate, parsedate_tz, mktime_tz + from urllib2 import AbstractDigestAuthHandler, parse_keqv_list, parse_http_list + +from .httpexceptions import HTTPBadRequest __all__ = ['get_header', 'list_headers', 'normalize_headers', 'HTTPHeader', 'EnvironVariable' ] @@ -164,7 +171,7 @@ REQUEST_METHOD = EnvironVariable("REQUEST_METHOD") SCRIPT_NAME = EnvironVariable("SCRIPT_NAME") PATH_INFO = EnvironVariable("PATH_INFO") -for _name, _obj in globals().items(): +for _name, _obj in six.iteritems(dict(globals())): if isinstance(_obj, EnvironVariable): __all__.append(_name) @@ -582,13 +589,10 @@ def normalize_headers(response_headers, strict=True): continue response_headers[idx] = (str(head), val) category[str(head)] = head.sort_order - def compare(a, b): - ac = category[a[0]] - bc = category[b[0]] - if ac == bc: - return cmp(a[0], b[0]) - return cmp(ac, bc) - response_headers.sort(compare) + def key_func(item): + value = item[0] + return (category[value], value) + response_headers.sort(key=key_func) class _DateHeader(_SingleValueHeader): """ @@ -733,7 +737,7 @@ class _CacheControl(_MultiValueHeader): result.append('max-age=%d' % max_age) if s_maxage is not None: result.append('s-maxage=%d' % s_maxage) - for (k, v) in extensions.items(): + for (k, v) in six.iteritems(extensions): if k not in self.extensions: raise AssertionError("unexpected extension used: '%s'" % k) result.append('%s="%s"' % (k.replace("_", "-"), v)) @@ -967,7 +971,7 @@ class _AcceptLanguage(_MultiValueHeader): if lvalue == "q": q = float(rvalue) qs.append((lang, q)) - qs.sort(lambda a, b: -cmp(a[1], b[1])) + qs.sort(key=lambda query: query[1], reverse=True) return [lang for (lang, q) in qs] _AcceptLanguage('Accept-Language', 'request', 'RFC 2616, 14.4') @@ -1007,18 +1011,33 @@ class _Authorization(_SingleValueHeader): path = path or "/" (_, realm) = challenge.split('realm="') (realm, _) = realm.split('"', 1) - auth = urllib2.AbstractDigestAuthHandler() + auth = AbstractDigestAuthHandler() auth.add_password(realm, path, username, password) (token, challenge) = challenge.split(' ', 1) - chal = urllib2.parse_keqv_list(urllib2.parse_http_list(challenge)) + chal = parse_keqv_list(parse_http_list(challenge)) class FakeRequest(object): - def get_full_url(self): - return path - def has_data(self): - return False + if six.PY3: + @property + def full_url(self): + return path + + selector = full_url + + @property + def data(self): + return None + else: + def get_full_url(self): + return path + + get_selector = get_full_url + + def has_data(self): + return False + def get_method(self): return method or "GET" - get_selector = get_full_url + retval = "Digest %s" % auth.get_authorization(FakeRequest(), chal) return (retval,) _Authorization('Authorization', 'request', 'RFC 2617') @@ -1092,6 +1111,6 @@ for head in _headers.values(): __all__.append(headname) __pudge_all__ = __all__[:] -for _name, _obj in globals().items(): +for _name, _obj in six.iteritems(dict(globals())): if isinstance(_obj, type) and issubclass(_obj, HTTPHeader): __pudge_all__.append(_name) diff --git a/paste/httpserver.py b/paste/httpserver.py index cd21713..179e526 100755 --- a/paste/httpserver.py +++ b/paste/httpserver.py @@ -17,16 +17,20 @@ if pyOpenSSL is installed, it also provides SSL capabilities. # @@: add support for chunked encoding, this is not a 1.1 server # till this is completed. +from __future__ import print_function import atexit import traceback -import socket, sys, threading, urlparse, Queue, urllib +import socket, sys, threading import posixpath +import six import time -import thread import os from itertools import count -from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer -from SocketServer import ThreadingMixIn +from six.moves import _thread +from six.moves import queue +from six.moves.BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer +from six.moves.socketserver import ThreadingMixIn +from six.moves.urllib.parse import unquote, urlsplit from paste.util import converters import logging try: @@ -38,6 +42,19 @@ except ImportError: __all__ = ['WSGIHandlerMixin', 'WSGIServer', 'WSGIHandler', 'serve'] __version__ = "0.5" + +def _get_headers(headers, k): + """ + Private function for abstracting differences in getting HTTP request + headers on Python 2 vs. Python 3 + """ + + if hasattr(headers, 'get_all'): + return headers.get_all(k) # Python 3 - email.message.Message + else: + return headers.getheaders(k) # Python 2 - mimetools.Message + + class ContinueHook(object): """ When a client request includes a 'Expect: 100-continue' header, then @@ -153,7 +170,7 @@ class WSGIHandlerMixin: if exc_info: try: if self.wsgi_headers_sent: - raise exc_info[0], exc_info[1], exc_info[2] + six.reraise(exc_info[0], exc_info[1], exc_info[2]) else: # In this case, we're going to assume that the # higher-level code is currently handling the @@ -176,8 +193,9 @@ class WSGIHandlerMixin: argument can be used to override any settings. """ - (scheme, netloc, path, query, fragment) = urlparse.urlsplit(self.path) - path = urllib.unquote(path) + dummy_url = 'http://dummy%s' % (self.path,) + (scheme, netloc, path, query, fragment) = urlsplit(dummy_url) + path = unquote(path) endslash = path.endswith('/') path = posixpath.normpath(path) if endslash and path != '/': @@ -244,14 +262,14 @@ class WSGIHandlerMixin: if hasattr(self.server, 'thread_pool'): # Now that we know what the request was for, we should # tell the thread pool what its worker is working on - self.server.thread_pool.worker_tracker[thread.get_ident()][1] = self.wsgi_environ + self.server.thread_pool.worker_tracker[_thread.get_ident()][1] = self.wsgi_environ self.wsgi_environ['paste.httpserver.thread_pool'] = self.server.thread_pool for k, v in self.headers.items(): key = 'HTTP_' + k.replace("-","_").upper() if key in ('HTTP_CONTENT_TYPE','HTTP_CONTENT_LENGTH'): continue - self.wsgi_environ[key] = ','.join(self.headers.getheaders(k)) + self.wsgi_environ[key] = ','.join(_get_headers(self.headers, k)) if hasattr(self.connection,'get_context'): self.wsgi_environ['wsgi.url_scheme'] = 'https' @@ -293,7 +311,7 @@ class WSGIHandlerMixin: if hasattr(result,'close'): result.close() result = None - except socket.error, exce: + except socket.error as exce: self.wsgi_connection_drop(exce, environ) return except: @@ -390,7 +408,7 @@ else: return (conn, info) def _auto_ssl_context(): - import OpenSSL, time, random + import OpenSSL, random pkey = OpenSSL.crypto.PKey() pkey.generate_key(OpenSSL.crypto.TYPE_RSA, 768) @@ -439,7 +457,7 @@ class WSGIHandler(WSGIHandlerMixin, BaseHTTPRequestHandler): # don't bother logging disconnects while handling a request try: BaseHTTPRequestHandler.handle(self) - except SocketErrors, exce: + except SocketErrors as exce: self.wsgi_connection_drop(exce) def address_string(self): @@ -575,12 +593,12 @@ class ThreadPool(object): self.nworkers = nworkers self.max_requests = max_requests self.name = name - self.queue = Queue.Queue() + self.queue = queue.Queue() self.workers = [] self.daemon = daemon if logger is None: logger = logging.getLogger('paste.httpserver.ThreadPool') - if isinstance(logger, basestring): + if isinstance(logger, six.string_types): logger = logging.getLogger(logger) self.logger = logger self.error_email = error_email @@ -734,7 +752,7 @@ class ThreadPool(object): return thread_id in threading._active def add_worker_thread(self, *args, **kwargs): - index = self._worker_count.next() + index = six.next(self._worker_count) worker = threading.Thread(target=self.worker_thread_callback, args=args, kwargs=kwargs, name=("worker %d" % index)) @@ -777,7 +795,7 @@ class ThreadPool(object): import pprint info_desc = pprint.pformat(info) except: - out = StringIO() + out = six.StringIO() traceback.print_exc(file=out) info_desc = 'Error:\n%s' % out.getvalue() self.notify_problem( @@ -839,7 +857,7 @@ class ThreadPool(object): ids="\n ".join(map(str, found))), subject="Process restart (too many zombie threads)") self.shutdown(10) - print 'Shutting down', threading.currentThread() + print('Shutting down', threading.currentThread()) raise ServerExit(3) def worker_thread_callback(self, message=None): @@ -848,7 +866,7 @@ class ThreadPool(object): callables. """ thread_obj = threading.currentThread() - thread_id = thread_obj.thread_id = thread.get_ident() + thread_id = thread_obj.thread_id = _thread.get_ident() self.workers.append(thread_obj) self.idle_workers.append(thread_id) requests_processed = 0 @@ -880,8 +898,8 @@ class ThreadPool(object): # removing all remnants of any exception, so # we should log it now. But ideally no # exception should reach this level - print >> sys.stderr, ( - 'Unexpected exception in worker %r' % runnable) + print('Unexpected exception in worker %r' % runnable, + file=sys.stderr) traceback.print_exc() if thread_id in self.dying_threads: # That last exception was intended to kill me @@ -891,7 +909,8 @@ class ThreadPool(object): del self.worker_tracker[thread_id] except KeyError: pass - sys.exc_clear() + if six.PY2: + sys.exc_clear() self.idle_workers.append(thread_id) finally: try: @@ -939,15 +958,14 @@ class ThreadPool(object): self.kill_worker(worker.thread_id) self.logger.info('Workers killed forcefully') if force_quit_timeout: - hung = [] timed_out = False need_force_quit = bool(zombies) - for workers in self.workers: + for worker in self.workers: if not timed_out and worker.isAlive(): timed_out = True worker.join(force_quit_timeout) if worker.isAlive(): - print "Worker %s won't die" % worker + print("Worker %s won't die" % worker) need_force_quit = True if need_force_quit: import atexit @@ -961,7 +979,7 @@ class ThreadPool(object): if mod == 'threading': atexit._exithandlers.remove(callback) atexit._run_exitfuncs() - print 'Forcefully exiting process' + print('Forcefully exiting process') os._exit(3) else: self.logger.info('All workers eventually killed') @@ -1020,7 +1038,7 @@ class ThreadPool(object): if e.strip()] server.sendmail(from_address, error_emails, message) server.quit() - print 'email sent to', error_emails, message + print('email sent to', error_emails, message) class ThreadPoolMixIn(object): """ @@ -1301,10 +1319,10 @@ def serve(application, host=None, port=None, handler=None, ssl_pem=None, protocol = is_ssl and 'https' or 'http' host, port = server.server_address[:2] if host == '0.0.0.0': - print 'serving on 0.0.0.0:%s view at %s://127.0.0.1:%s' % \ - (port, protocol, port) + print('serving on 0.0.0.0:%s view at %s://127.0.0.1:%s' + % (port, protocol, port)) else: - print "serving on %s://%s:%s" % (protocol, host, port) + print("serving on %s://%s:%s" % (protocol, host, port)) try: server.serve_forever() except KeyboardInterrupt: diff --git a/paste/lint.py b/paste/lint.py index 03b4a60..d781686 100644 --- a/paste/lint.py +++ b/paste/lint.py @@ -110,8 +110,8 @@ Some of the things this checks: """ import re +import six import sys -from types import DictType, StringType, TupleType, ListType import warnings header_re = re.compile(r'^[a-zA-Z][a-zA-Z0-9\-_]*$') @@ -133,7 +133,7 @@ def middleware(application, global_conf=None): will be printed to stderr -- there's no way to throw an exception at that point). """ - + def lint_app(*args, **kw): assert len(args) == 2, "Two arguments required" assert not kw, "No keyword arguments allowed" @@ -185,22 +185,22 @@ class InputWrapper(object): def read(self, *args): assert len(args) <= 1 v = self.input.read(*args) - assert type(v) is type("") + assert isinstance(v, six.binary_type) return v def readline(self, *args): v = self.input.readline(*args) - assert type(v) is type("") + assert isinstance(v, six.binary_type) return v def readlines(self, *args): assert len(args) <= 1 lines = self.input.readlines(*args) - assert type(lines) is type([]) + assert isinstance(lines, list) for line in lines: - assert type(line) is type("") + assert isinstance(line, six.binary_type) return lines - + def __iter__(self): while 1: line = self.readline() @@ -217,7 +217,7 @@ class ErrorWrapper(object): self.errors = wsgi_errors def write(self, s): - assert type(s) is type("") + assert isinstance(s, bytes) self.errors.write(s) def flush(self): @@ -236,7 +236,7 @@ class WriteWrapper(object): self.writer = wsgi_writer def __call__(self, s): - assert type(s) is type("") + assert isinstance(s, six.binary_type) self.writer(s) class PartialIteratorWrapper(object): @@ -262,13 +262,15 @@ class IteratorWrapper(object): def next(self): assert not self.closed, ( "Iterator read after closed") - v = self.iterator.next() + v = six.next(self.iterator) if self.check_start_response is not None: assert self.check_start_response, ( "The application returns and we started iterating over its body, but start_response has not yet been called") self.check_start_response = None return v - + + __next__ = next + def close(self): self.closed = True if hasattr(self.original_iterator, 'close'): @@ -282,10 +284,10 @@ class IteratorWrapper(object): "Iterator garbage collected without being closed") def check_environ(environ): - assert type(environ) is DictType, ( + assert isinstance(environ,dict), ( "Environment is not of the right type: %r (environment: %r)" % (type(environ), environ)) - + for key in ['REQUEST_METHOD', 'SERVER_NAME', 'SERVER_PORT', 'wsgi.version', 'wsgi.input', 'wsgi.errors', 'wsgi.multithread', 'wsgi.multiprocess', @@ -309,11 +311,11 @@ def check_environ(environ): if '.' in key: # Extension, we don't care about its type continue - assert type(environ[key]) is StringType, ( + assert isinstance(environ[key], str), ( "Environmental variable %s is not a string: %r (value: %r)" % (key, type(environ[key]), environ[key])) - - assert type(environ['wsgi.version']) is TupleType, ( + + assert isinstance(environ['wsgi.version'], tuple), ( "wsgi.version should be a tuple (%r)" % environ['wsgi.version']) assert environ['wsgi.url_scheme'] in ('http', 'https'), ( "wsgi.url_scheme unknown: %r" % environ['wsgi.url_scheme']) @@ -339,7 +341,7 @@ def check_environ(environ): "Invalid CONTENT_LENGTH: %r" % environ['CONTENT_LENGTH']) if not environ.get('SCRIPT_NAME'): - assert environ.has_key('PATH_INFO'), ( + assert 'PATH_INFO' in environ, ( "One of SCRIPT_NAME or PATH_INFO are required (PATH_INFO " "should at least be '/' if SCRIPT_NAME is empty)") assert environ.get('SCRIPT_NAME') != '/', ( @@ -359,7 +361,7 @@ def check_errors(wsgi_errors): % (wsgi_errors, attr)) def check_status(status): - assert type(status) is StringType, ( + assert isinstance(status, str), ( "Status must be a string (not %r)" % status) # Implicitly check that we can turn it into an integer: status_code = status.split(None, 1)[0] @@ -374,12 +376,12 @@ def check_status(status): % status, WSGIWarning) def check_headers(headers): - assert type(headers) is ListType, ( + assert isinstance(headers,list), ( "Headers (%r) must be of type list: %r" % (headers, type(headers))) header_names = {} for item in headers: - assert type(item) is TupleType, ( + assert isinstance(item, tuple), ( "Individual headers (%r) must be of type tuple: %r" % (item, type(item))) assert len(item) == 2 diff --git a/paste/modpython.py b/paste/modpython.py index a3fef55..d20a588 100644 --- a/paste/modpython.py +++ b/paste/modpython.py @@ -8,7 +8,7 @@ Example httpd.conf section for a Paste app with an ini file:: PythonHandler paste.modpython PythonOption paste.ini /some/location/your/pasteconfig.ini </Location> - + Or if you want to load a WSGI application under /your/homedir in the module ``startup`` and the WSGI app is ``app``:: @@ -49,6 +49,7 @@ This module highly based on Robert Brewer's, here: http://projects.amor.org/misc/svn/modpython_gateway.py """ +import six import traceback try: @@ -58,22 +59,22 @@ except: from paste.deploy import loadapp class InputWrapper(object): - + def __init__(self, req): self.req = req - + def close(self): pass - + def read(self, size=-1): return self.req.read(size) - + def readline(self, size=-1): return self.req.readline(size) - + def readlines(self, hint=-1): return self.req.readlines(hint) - + def __iter__(self): line = self.readline() while line: @@ -84,16 +85,16 @@ class InputWrapper(object): class ErrorWrapper(object): - + def __init__(self, req): self.req = req - + def flush(self): pass - + def write(self, msg): self.req.log_error(msg) - + def writelines(self, seq): self.write(''.join(seq)) @@ -103,12 +104,12 @@ bad_value = ("You must provide a PythonOption '%s', either 'on' or 'off', " class Handler(object): - + def __init__(self, req): self.started = False - + options = req.get_options() - + # Threading and forking try: q = apache.mpm_query @@ -122,7 +123,7 @@ class Handler(object): threaded = False else: raise ValueError(bad_value % "multithread") - + forked = options.get('multiprocess', '').lower() if forked == 'on': forked = True @@ -130,9 +131,9 @@ class Handler(object): forked = False else: raise ValueError(bad_value % "multiprocess") - + env = self.environ = dict(apache.build_cgi_env(req)) - + if 'SCRIPT_NAME' in options: # Override SCRIPT_NAME and PATH_INFO if requested. env['SCRIPT_NAME'] = options['SCRIPT_NAME'] @@ -140,7 +141,7 @@ class Handler(object): else: env['SCRIPT_NAME'] = '' env['PATH_INFO'] = req.uri - + env['wsgi.input'] = InputWrapper(req) env['wsgi.errors'] = ErrorWrapper(req) env['wsgi.version'] = (1, 0) @@ -151,9 +152,9 @@ class Handler(object): env['wsgi.url_scheme'] = 'http' env['wsgi.multithread'] = threaded env['wsgi.multiprocess'] = forked - + self.request = req - + def run(self, application): try: result = application(self.environ, self.start_response) @@ -171,17 +172,17 @@ class Handler(object): data = "A server error occurred. Please contact the administrator." self.request.set_content_length(len(data)) self.request.write(data) - + def start_response(self, status, headers, exc_info=None): if exc_info: try: if self.started: - raise exc_info[0], exc_info[1], exc_info[2] + six.reraise(exc_info[0], exc_info[1], exc_info[2]) finally: exc_info = None - + self.request.status = int(status[:3]) - + for key, val in headers: if key.lower() == 'content-length': self.request.set_content_length(int(val)) @@ -189,9 +190,9 @@ class Handler(object): self.request.content_type = val else: self.request.headers_out.add(key, val) - + return self.write - + def write(self, data): if not self.started: self.started = True @@ -213,7 +214,7 @@ def handler(req): module = __import__(module_name, globals(), locals(), ['']) startup = apache.resolve_object(module, object_str) startup(req) - + # Register a cleanup function if requested. global cleanup if 'wsgi.cleanup' in options and not cleanup: @@ -229,7 +230,7 @@ def handler(req): apache.register_cleanup(cleaner) except AttributeError: req.server.register_cleanup(req, cleaner) - + # Import the wsgi 'application' callable and pass it to Handler.run global wsgiapps appini = options.get('paste.ini') @@ -238,15 +239,15 @@ def handler(req): if appini not in wsgiapps: wsgiapps[appini] = loadapp("config:%s" % appini) app = wsgiapps[appini] - + # Import the wsgi 'application' callable and pass it to Handler.run appwsgi = options.get('wsgi.application') if appwsgi and not appini: modname, objname = appwsgi.split('::', 1) module = __import__(modname, globals(), locals(), ['']) app = getattr(module, objname) - + Handler(req).run(app) - + # status was set in Handler; always return apache.OK return apache.OK diff --git a/paste/proxy.py b/paste/proxy.py index 155128f..67d4b1b 100644 --- a/paste/proxy.py +++ b/paste/proxy.py @@ -21,24 +21,25 @@ TODO: * Rewriting body? (Probably not on this one -- that can be done with a different middleware that wraps this middleware) -* Example:: - +* Example:: + use = egg:Paste#proxy address = http://server3:8680/exist/rest/db/orgs/sch/config/ allowed_request_methods = GET - + """ -import httplib -import urlparse -import urllib +from six.moves import http_client as httplib +from six.moves.urllib import parse as urlparse +from six.moves.urllib.parse import quote +import six from paste import httpexceptions from paste.util.converters import aslist # Remove these headers from response (specify lower case header # names): -filtered_headers = ( +filtered_headers = ( 'transfer-encoding', 'connection', 'keep-alive', @@ -60,12 +61,12 @@ class Proxy(object): self.path = self.parsed[2] self.allowed_request_methods = [ x.lower() for x in allowed_request_methods if x] - + self.suppress_http_headers = [ x.lower() for x in suppress_http_headers if x] def __call__(self, environ, start_response): - if (self.allowed_request_methods and + if (self.allowed_request_methods and environ['REQUEST_METHOD'].lower() not in self.allowed_request_methods): return httpexceptions.HTTPBadRequest("Disallowed")(environ, start_response) @@ -95,30 +96,30 @@ class Proxy(object): body = environ['wsgi.input'].read(-1) headers['content-length'] = str(len(body)) else: - headers['content-length'] = environ['CONTENT_LENGTH'] + headers['content-length'] = environ['CONTENT_LENGTH'] length = int(environ['CONTENT_LENGTH']) body = environ['wsgi.input'].read(length) else: body = '' - - path_info = urllib.quote(environ['PATH_INFO']) - if self.path: + + path_info = quote(environ['PATH_INFO']) + if self.path: request_path = path_info if request_path and request_path[0] == '/': request_path = request_path[1:] - + path = urlparse.urljoin(self.path, request_path) else: path = path_info if environ.get('QUERY_STRING'): path += '?' + environ['QUERY_STRING'] - + conn.request(environ['REQUEST_METHOD'], path, body, headers) res = conn.getresponse() headers_out = parse_headers(res.msg) - + status = '%s %s' % (res.status, res.reason) start_response(status, headers_out) # @@: Default? @@ -134,13 +135,13 @@ def make_proxy(global_conf, address, allowed_request_methods="", suppress_http_headers=""): """ Make a WSGI application that proxies to another address: - + ``address`` the full URL ending with a trailing ``/`` - + ``allowed_request_methods``: a space seperated list of request methods (e.g., ``GET POST``) - + ``suppress_http_headers`` a space seperated list of http headers (lower case, without the leading ``http_``) that should not be passed on to target @@ -224,17 +225,17 @@ class TransparentProxy(object): else: body = '' length = 0 - + path = (environ.get('SCRIPT_NAME', '') + environ.get('PATH_INFO', '')) - path = urllib.quote(path) + path = quote(path) if 'QUERY_STRING' in environ: path += '?' + environ['QUERY_STRING'] conn.request(environ['REQUEST_METHOD'], path, body, headers) res = conn.getresponse() headers_out = parse_headers(res.msg) - + status = '%s %s' % (res.status, res.reason) start_response(status, headers_out) # @@: Default? @@ -250,27 +251,32 @@ def parse_headers(message): """ Turn a Message object into a list of WSGI-style headers. """ - headers_out = [] - for full_header in message.headers: - if not full_header: - # Shouldn't happen, but we'll just ignore - continue - if full_header[0].isspace(): - # Continuation line, add to the last header - if not headers_out: - raise ValueError( - "First header starts with a space (%r)" % full_header) - last_header, last_value = headers_out.pop() - value = last_value + ' ' + full_header.strip() - headers_out.append((last_header, value)) - continue - try: - header, value = full_header.split(':', 1) - except: - raise ValueError("Invalid header: %r" % full_header) - value = value.strip() - if header.lower() not in filtered_headers: - headers_out.append((header, value)) + headers_out = [] + if six.PY3: + for header, value in message.items(): + if header.lower() not in filtered_headers: + headers_out.append((header, value)) + else: + for full_header in message.headers: + if not full_header: + # Shouldn't happen, but we'll just ignore + continue + if full_header[0].isspace(): + # Continuation line, add to the last header + if not headers_out: + raise ValueError( + "First header starts with a space (%r)" % full_header) + last_header, last_value = headers_out.pop() + value = last_value + ' ' + full_header.strip() + headers_out.append((last_header, value)) + continue + try: + header, value = full_header.split(':', 1) + except: + raise ValueError("Invalid header: %r" % full_header) + value = value.strip() + if header.lower() not in filtered_headers: + headers_out.append((header, value)) return headers_out def make_transparent_proxy( diff --git a/paste/recursive.py b/paste/recursive.py index 8ea7038..0bef920 100644 --- a/paste/recursive.py +++ b/paste/recursive.py @@ -23,8 +23,9 @@ Raise ``ForwardRequestException(new_path_info)`` to do a forward (aborting the current request). """ -from cStringIO import StringIO +import six import warnings +from six.moves import cStringIO as StringIO __all__ = ['RecursiveMiddleware'] __pudge_all__ = ['RecursiveMiddleware', 'ForwardRequestException'] @@ -82,7 +83,7 @@ class RecursiveMiddleware(object): environ['paste.recursive.script_name'] = my_script_name try: return self.application(environ, start_response) - except ForwardRequestException, e: + except ForwardRequestException as e: middleware = CheckForRecursionMiddleware( e.factory(self), environ) return middleware(environ, start_response) @@ -114,10 +115,10 @@ class ForwardRequestException(Exception): def app(environ, start_response): if environ['PATH_INFO'] == '/hello': start_response("200 OK", [('Content-type', 'text/plain')]) - return ['Hello World!'] + return [b'Hello World!'] elif environ['PATH_INFO'] == '/error': start_response("404 Not Found", [('Content-type', 'text/plain')]) - return ['Page not found'] + return [b'Page not found'] else: raise ForwardRequestException('/error') @@ -157,10 +158,10 @@ class ForwardRequestException(Exception): def app(environ, start_response): if environ['PATH_INFO'] == '/hello': start_response("200 OK", [('Content-type', 'text/plain')]) - return ['Hello World!'] + return [b'Hello World!'] elif environ['PATH_INFO'] == '/error': start_response("404 Not Found", [('Content-type', 'text/plain')]) - return ['Page not found'] + return [b'Page not found'] else: def factory(app): return StatusKeeper(app, status='404 Not Found', url='/error') @@ -319,7 +320,7 @@ class Includer(Recursive): response = IncludedResponse() def start_response(status, headers, exc_info=None): if exc_info: - raise exc_info[0], exc_info[1], exc_info[2] + six.reraise(exc_info[0], exc_info[1], exc_info[2]) response.status = status response.headers = headers return response.write @@ -373,7 +374,7 @@ class IncluderAppIter(Recursive): response = IncludedAppIterResponse() def start_response(status, headers, exc_info=None): if exc_info: - raise exc_info[0], exc_info[1], exc_info[2] + six.reraise(exc_info[0], exc_info[1], exc_info[2]) response.status = status response.headers = headers return response.write diff --git a/paste/registry.py b/paste/registry.py index ef6516c..908bc0d 100644 --- a/paste/registry.py +++ b/paste/registry.py @@ -89,7 +89,7 @@ is provided solely in the extremely rare case that it is an issue so that a quick way to work around it is documented. """ -import sys +import six import paste.util.threadinglocal as threadinglocal __all__ = ['StackedObjectProxy', 'RegistryManager', 'StackedObjectRestorer', @@ -345,7 +345,7 @@ class Registry(object): def cleanup(self): """Remove all objects from all StackedObjectProxy instances that were tracked at this Registry context""" - for stacked, obj in self.reglist[-1].itervalues(): + for stacked, obj in six.itervalues(self.reglist[-1]): stacked._pop_object(obj) self.reglist.pop() @@ -377,7 +377,7 @@ class RegistryManager(object): try: app_iter = self.application(environ, start_response) - except Exception, e: + except Exception as e: # Regardless of if the content is an iterable, generator, list # or tuple, we clean-up right now. If its an iterable/generator # care should be used to ensure the generator has its own ref @@ -408,7 +408,7 @@ class RegistryManager(object): try: for item in self.application(environ, start_response): yield item - except Exception, e: + except Exception as e: # Regardless of if the content is an iterable, generator, list # or tuple, we clean-up right now. If its an iterable/generator # care should be used to ensure the generator has its own ref @@ -498,7 +498,7 @@ class StackedObjectRestorer(object): # their methods to act differently when a restoration context is active # in the current thread for reglist in registry.reglist: - for stacked, obj in reglist.itervalues(): + for stacked, obj in six.itervalues(reglist): self.enable_restoration(stacked) def get_saved_proxied_obj(self, stacked, request_id): diff --git a/paste/reloader.py b/paste/reloader.py index a0e3850..15c4d1c 100644 --- a/paste/reloader.py +++ b/paste/reloader.py @@ -26,7 +26,7 @@ or is run from this .bat file (if you use Windows):: if %errorlevel% == 3 goto repeat or run a monitoring process in Python (``paster serve --reload`` does -this). +this). Use the ``watch_file(filename)`` function to cause a reload/restart for other other non-Python files (e.g., configuration files). If you have @@ -40,6 +40,7 @@ Then every time the reloader polls files it will call ``watch_config_files`` and check all the filenames it returns. """ +from __future__ import print_function import os import sys import time @@ -93,12 +94,13 @@ class Monitor(object): try: filenames.extend(file_callback()) except: - print >> sys.stderr, "Error calling paste.reloader callback %r:" % file_callback + print("Error calling paste.reloader callback %r:" % file_callback, + file=sys.stderr) traceback.print_exc() for module in sys.modules.values(): try: filename = module.__file__ - except (AttributeError, ImportError), exc: + except (AttributeError, ImportError): continue if filename is not None: filenames.append(filename) @@ -119,8 +121,7 @@ class Monitor(object): if not self.module_mtimes.has_key(filename): self.module_mtimes[filename] = mtime elif self.module_mtimes[filename] < mtime: - print >> sys.stderr, ( - "%s changed; reloading..." % filename) + print("%s changed; reloading..." % filename, file=sys.stderr) return False return True diff --git a/paste/request.py b/paste/request.py index 9af494d..f0d91c1 100644 --- a/paste/request.py +++ b/paste/request.py @@ -18,14 +18,21 @@ environment to solve common requirements. """ import cgi -from Cookie import SimpleCookie, CookieError -from StringIO import StringIO -import urlparse -import urllib +from six.moves.urllib import parse as urlparse +from six.moves.urllib.parse import quote, parse_qsl +try: + # Python 3 + from http.cookies import SimpleCookie, CookieError +except ImportError: + # Python 2 + from Cookie import SimpleCookie, CookieError + try: from UserDict import DictMixin except ImportError: - from paste.util.UserDict24 import DictMixin + from collections import MutableMapping as DictMixin +import six + from paste.util.multidict import MultiDict __all__ = ['get_cookies', 'get_cookie_dict', 'parse_querystring', @@ -40,7 +47,7 @@ def get_cookies(environ): """ header = environ.get('HTTP_COOKIE', '') - if environ.has_key('paste.cookies'): + if 'paste.cookies' in environ: cookies, check_header = environ['paste.cookies'] if check_header == header: return cookies @@ -63,7 +70,7 @@ def get_cookie_dict(environ): header = environ.get('HTTP_COOKIE') if not header: return {} - if environ.has_key('paste.cookies.dict'): + if 'paste.cookies.dict' in environ: cookies, check_header = environ['paste.cookies.dict'] if check_header == header: return cookies @@ -96,8 +103,8 @@ def parse_querystring(environ): parsed, check_source = environ['paste.parsed_querystring'] if check_source == source: return parsed - parsed = cgi.parse_qsl(source, keep_blank_values=True, - strict_parsing=False) + parsed = parse_qsl(source, keep_blank_values=True, + strict_parsing=False) environ['paste.parsed_querystring'] = (parsed, source) return parsed @@ -127,8 +134,8 @@ def parse_dict_querystring(environ): parsed, check_source = environ['paste.parsed_dict_querystring'] if check_source == source: return parsed - parsed = cgi.parse_qsl(source, keep_blank_values=True, - strict_parsing=False) + parsed = parse_qsl(source, keep_blank_values=True, + strict_parsing=False) multi = MultiDict(parsed) environ['paste.parsed_dict_querystring'] = (multi, source) return multi @@ -168,11 +175,11 @@ def parse_formvars(environ, include_get_vars=True): old_query_string = environ.get('QUERY_STRING','') environ['QUERY_STRING'] = '' if fake_out_cgi: - input = StringIO('') + input = six.BytesIO(b'') old_content_type = environ.get('CONTENT_TYPE') old_content_length = environ.get('CONTENT_LENGTH') environ['CONTENT_LENGTH'] = '0' - environ['CONTENT_TYPE'] = '' + environ['CONTENT_TYPE'] = '' else: input = environ['wsgi.input'] fs = cgi.FieldStorage(fp=input, @@ -231,14 +238,14 @@ def construct_url(environ, with_query_string=True, with_path_info=True, url += ':' + environ['SERVER_PORT'] if script_name is None: - url += urllib.quote(environ.get('SCRIPT_NAME','')) + url += quote(environ.get('SCRIPT_NAME','')) else: - url += urllib.quote(script_name) + url += quote(script_name) if with_path_info: if path_info is None: - url += urllib.quote(environ.get('PATH_INFO','')) + url += quote(environ.get('PATH_INFO','')) else: - url += urllib.quote(path_info) + url += quote(path_info) if with_query_string: if querystring is None: if environ.get('QUERY_STRING'): @@ -288,8 +295,8 @@ def path_info_pop(environ): >>> def call_it(script_name, path_info): ... env = {'SCRIPT_NAME': script_name, 'PATH_INFO': path_info} ... result = path_info_pop(env) - ... print 'SCRIPT_NAME=%r; PATH_INFO=%r; returns=%r' % ( - ... env['SCRIPT_NAME'], env['PATH_INFO'], result) + ... print('SCRIPT_NAME=%r; PATH_INFO=%r; returns=%r' % ( + ... env['SCRIPT_NAME'], env['PATH_INFO'], result)) >>> call_it('/foo', '/bar') SCRIPT_NAME='/foo/bar'; PATH_INFO=''; returns='bar' >>> call_it('/foo/bar', '') @@ -368,7 +375,10 @@ class EnvironHeaders(DictMixin): return key[5:].replace('_', '-').title() else: return None - + + def __len__(self): + return len(self.environ) + def __getitem__(self, item): return self.environ[self._trans_name(item)] diff --git a/paste/session.py b/paste/session.py index 16a7739..ae208e7 100644 --- a/paste/session.py +++ b/paste/session.py @@ -23,11 +23,17 @@ session for each request, with no caching. Also, sessions aren't expired. """ -from Cookie import SimpleCookie +try: + # Python 3 + from http.cookies import SimpleCookie +except ImportError: + # Python 2 + from Cookie import SimpleCookie import time import random import os import datetime +import six import threading import tempfile @@ -100,7 +106,7 @@ class SessionFactory(object): return self.session.data() cookies = request.get_cookies(self.environ) session = None - if cookies.has_key(self.cookie_name): + if self.cookie_name in cookies: self.sid = cookies[self.cookie_name].value try: session = self.session_class(self.sid, create=False, @@ -142,7 +148,10 @@ class SessionFactory(object): r.append(os.times()) if for_object is not None: r.append(id(for_object)) - md5_hash = md5(str(r)) + content = str(r) + if six.PY3: + content = content.encode('utf8') + md5_hash = md5(content) try: return md5_hash.hexdigest() except AttributeError: @@ -179,7 +188,7 @@ class FileSession(object): chmod=None, expiration=2880, # in minutes: 48 hours ): - if chmod and isinstance(chmod, basestring): + if chmod and isinstance(chmod, (six.binary_type, six.text_type)): chmod = int(chmod, 8) self.chmod = chmod if not sid: diff --git a/paste/transaction.py b/paste/transaction.py index a283067..1347acd 100644 --- a/paste/transaction.py +++ b/paste/transaction.py @@ -116,5 +116,5 @@ if '__main__' == __name__ and False: curr = conn.cursor() curr.execute("SELECT now(), %s" % conn.quote("B'n\\'gles")) (time, bing) = curr.fetchone() - print bing, time + print(bing, time) diff --git a/paste/translogger.py b/paste/translogger.py index 47de2d3..794efd8 100644 --- a/paste/translogger.py +++ b/paste/translogger.py @@ -5,8 +5,9 @@ Middleware for logging requests, using Apache combined log format """ import logging +import six import time -import urllib +from six.moves.urllib.parse import quote class TransLogger(object): """ @@ -50,7 +51,7 @@ class TransLogger(object): def __call__(self, environ, start_response): start = time.localtime() - req_uri = urllib.quote(environ.get('SCRIPT_NAME', '') + req_uri = quote(environ.get('SCRIPT_NAME', '') + environ.get('PATH_INFO', '')) if environ.get('QUERY_STRING'): req_uri += '?'+environ['QUERY_STRING'] @@ -106,9 +107,9 @@ def make_filter( setup_console_handler=True, set_logger_level=logging.DEBUG): from paste.util.converters import asbool - if isinstance(logging_level, basestring): + if isinstance(logging_level, (six.binary_type, six.text_type)): logging_level = logging._levelNames[logging_level] - if isinstance(set_logger_level, basestring): + if isinstance(set_logger_level, (six.binary_type, six.text_type)): set_logger_level = logging._levelNames[set_logger_level] return TransLogger( app, diff --git a/paste/url.py b/paste/url.py index afc4fca..fb08d6d 100644 --- a/paste/url.py +++ b/paste/url.py @@ -4,9 +4,11 @@ """ This module implements a class for handling URLs. """ -import urllib +from six.moves.urllib.parse import parse_qsl, quote, unquote, urlencode import cgi from paste import request +import six + # Imported lazily from FormEncode: variabledecode = None @@ -20,9 +22,7 @@ def html_quote(v): def url_quote(v): if v is None: return '' - return urllib.quote(str(v)) - -url_unquote = urllib.unquote + return quote(str(v)) def js_repr(v): if v is None: @@ -83,7 +83,7 @@ class URLResource(object): if querystring is None: vars = request.parse_querystring(environ) else: - vars = cgi.parse_qsl( + vars = parse_qsl( querystring, keep_blank_values=True, strict_parsing=False) @@ -102,7 +102,7 @@ class URLResource(object): def __getitem__(self, item): if '=' in item: name, value = item.split('=', 1) - return self._add_vars({url_unquote(name): url_unquote(value)}) + return self._add_vars({unquote(name): unquote(value)}) return self._add_positional((item,)) def attr(self, **kw): @@ -138,10 +138,10 @@ class URLResource(object): vars = variabledecode.variable_encode(vars) return vars - + def var(self, **kw): kw = self.coerce_vars(kw) - new_vars = self.vars + kw.items() + new_vars = self.vars + list(kw.items()) return self.__class__(self.url, vars=new_vars, attrs=self.attrs, params=self.original_params) @@ -183,14 +183,17 @@ class URLResource(object): attrs=u.attrs, params=u.original_params) return u - - __div__ = addpath + + if six.PY3: + __truediv__ = addpath + else: + __div__ = addpath def become(self, OtherClass): return OtherClass(self.url, vars=self.vars, attrs=self.attrs, params=self.original_params) - + def href__get(self): s = self.url if self.vars: @@ -202,7 +205,7 @@ class URLResource(object): elif val is None: continue vars.append((name, val)) - s += urllib.urlencode(vars, True) + s += urlencode(vars, True) return s href = property(href__get) @@ -219,7 +222,7 @@ class URLResource(object): ', '.join(['%s=%r' % (n, v) for n, v in self.attrs.items()])) return base + '>' - + def html__get(self): if not self.params.get('tag'): raise ValueError( @@ -252,7 +255,7 @@ class URLResource(object): for an empty tag (like ``<img />``) """ raise NotImplementedError - + def _add_vars(self, vars): raise NotImplementedError @@ -309,7 +312,7 @@ class URL(URLResource): return self.addpath(*args) def _html_attrs(self): - attrs = self.attrs.items() + attrs = list(self.attrs.items()) attrs.insert(0, ('href', self.href)) if self.params.get('confirm'): attrs.append(('onclick', 'return confirm(%s)' @@ -330,7 +333,7 @@ class URL(URLResource): return self.become(JSPopup) js_popup = property(js_popup__get) - + class Image(URLResource): r""" @@ -343,7 +346,7 @@ class Image(URLResource): >>> i.href '/images/foo.png' """ - + default_params = {'tag': 'img'} def __str__(self): @@ -359,7 +362,7 @@ class Image(URLResource): return self.addpath(*args) def _html_attrs(self): - attrs = self.attrs.items() + attrs = list(self.attrs.items()) attrs.insert(0, ('src', self.href)) return attrs @@ -398,7 +401,7 @@ class Button(URLResource): return self.addpath(*args) def _html_attrs(self): - attrs = self.attrs.items() + attrs = list(self.attrs.items()) onclick = 'location.href=%s' % js_repr(self.href) if self.params.get('confirm'): onclick = 'if (confirm(%s)) {%s}' % ( @@ -451,7 +454,7 @@ class JSPopup(URLResource): return ', '.join(map(js_repr, args)) def _html_attrs(self): - attrs = self.attrs.items() + attrs = list(self.attrs.items()) onclick = ('window.open(%s); return false' % self._window_args()) attrs.insert(0, ('target', self.params['target'])) @@ -472,4 +475,4 @@ class JSPopup(URLResource): if __name__ == '__main__': import doctest doctest.testmod() - + diff --git a/paste/urlmap.py b/paste/urlmap.py index a636531..f721f2d 100644 --- a/paste/urlmap.py +++ b/paste/urlmap.py @@ -4,10 +4,16 @@ Map URL prefixes to WSGI applications. See ``URLMap`` """ -from UserDict import DictMixin import re import os import cgi +try: + # Python 3 + from collections import MutableMapping as DictMixin +except ImportError: + # Python 2 + from UserDict import DictMixin + from paste import httpexceptions __all__ = ['URLMap', 'PathProxyURLMap'] @@ -90,6 +96,13 @@ class URLMap(DictMixin): not_found_app = self.not_found_app self.not_found_application = not_found_app + def __len__(self): + return len(self.applications) + + def __iter__(self): + for app_url, app in self.applications: + yield app_url + norm_url_re = re.compile('//+') domain_url_re = re.compile('^(http|https)://') @@ -101,9 +114,9 @@ class URLMap(DictMixin): ',\n '.join(map(repr, matches))) else: extra = '' - extra += '\nSCRIPT_NAME: %r' % environ.get('SCRIPT_NAME') - extra += '\nPATH_INFO: %r' % environ.get('PATH_INFO') - extra += '\nHTTP_HOST: %r' % environ.get('HTTP_HOST') + extra += '\nSCRIPT_NAME: %r' % cgi.escape(environ.get('SCRIPT_NAME')) + extra += '\nPATH_INFO: %r' % cgi.escape(environ.get('PATH_INFO')) + extra += '\nHTTP_HOST: %r' % cgi.escape(environ.get('HTTP_HOST')) app = httpexceptions.HTTPNotFound( environ['PATH_INFO'], comment=cgi.escape(extra)).wsgi_application diff --git a/paste/urlparser.py b/paste/urlparser.py index a7469e3..19bcbac 100644 --- a/paste/urlparser.py +++ b/paste/urlparser.py @@ -5,6 +5,7 @@ WSGI applications that parse the URL and dispatch to on-disk resources """ import os +import six import sys import imp import mimetypes @@ -16,7 +17,7 @@ from paste import request from paste import fileapp from paste.util import import_string from paste import httpexceptions -from httpheaders import ETAG +from .httpheaders import ETAG from paste.util import converters class NoDefault(object): @@ -298,7 +299,7 @@ class URLParser(object): ``environ`` and ``filename``, and returns a WSGI application. """ d = cls.global_constructors - assert not d.has_key(extension), ( + assert extension not in d, ( "A constructor already exists for the extension %r (%r) " "when attemption to register constructor %r" % (extension, d[extension], constructor)) @@ -362,13 +363,13 @@ def load_module(environ, filename): environ['wsgi.errors']) def load_module_from_name(environ, filename, module_name, errors): - if sys.modules.has_key(module_name): + if module_name in sys.modules: return sys.modules[module_name] init_filename = os.path.join(os.path.dirname(filename), '__init__.py') if not os.path.exists(init_filename): try: f = open(init_filename, 'w') - except (OSError, IOError), e: + except (OSError, IOError) as e: errors.write( 'Cannot write __init__.py file into directory %s (%s)\n' % (os.path.dirname(filename), e)) @@ -376,7 +377,7 @@ def load_module_from_name(environ, filename, module_name, errors): f.write('#\n') f.close() fp = None - if sys.modules.has_key(module_name): + if module_name in sys.modules: return sys.modules[module_name] if '.' in module_name: parent_name = '.'.join(module_name.split('.')[:-1]) @@ -525,7 +526,7 @@ class PkgResourcesParser(StaticURLParser): def __init__(self, egg_or_spec, resource_name, manager=None, root_resource=None): if pkg_resources is None: raise NotImplementedError("This class requires pkg_resources.") - if isinstance(egg_or_spec, (str, unicode)): + if isinstance(egg_or_spec, (six.binary_type, six.text_type)): self.egg = pkg_resources.get_distribution(egg_or_spec) else: self.egg = egg_or_spec @@ -574,7 +575,7 @@ class PkgResourcesParser(StaticURLParser): # @@: I don't know what to do with the encoding. try: file = self.egg.get_resource_stream(self.manager, resource) - except (IOError, OSError), e: + except (IOError, OSError) as e: exc = httpexceptions.HTTPForbidden( 'You are not permitted to view this file (%s)' % e) return exc.wsgi_application(environ, start_response) diff --git a/paste/util/PySourceColor.py b/paste/util/PySourceColor.py index 65406ec..c576ead 100644 --- a/paste/util/PySourceColor.py +++ b/paste/util/PySourceColor.py @@ -1,2103 +1,2102 @@ -# -*- coding: Latin-1 -*-
-"""
-PySourceColor: color Python source code
-"""
-
-"""
- PySourceColor.py
-
-----------------------------------------------------------------------------
-
- A python source to colorized html/css/xhtml converter.
- Hacked by M.E.Farmer Jr. 2004, 2005
- Python license
-
-----------------------------------------------------------------------------
-
- - HTML markup does not create w3c valid html, but it works on every
- browser i've tried so far.(I.E.,Mozilla/Firefox,Opera,Konqueror,wxHTML).
- - CSS markup is w3c validated html 4.01 strict,
- but will not render correctly on all browsers.
- - XHTML markup is w3c validated xhtml 1.0 strict,
- like html 4.01, will not render correctly on all browsers.
-
-----------------------------------------------------------------------------
-
-Features:
-
- -Three types of markup:
- html (default)
- css/html 4.01 strict
- xhtml 1.0 strict
-
- -Can tokenize and colorize:
- 12 types of strings
- 2 comment types
- numbers
- operators
- brackets
- math operators
- class / name
- def / name
- decorator / name
- keywords
- arguments class/def/decorator
- linenumbers
- names
- text
-
- -Eight colorschemes built-in:
- null
- mono
- lite (default)
- dark
- dark2
- idle
- viewcvs
- pythonwin
-
- -Header and footer
- set to '' for builtin header / footer.
- give path to a file containing the html
- you want added as header or footer.
-
- -Arbitrary text and html
- html markup converts all to raw (TEXT token)
- #@# for raw -> send raw text.
- #$# for span -> inline html and text.
- #%# for div -> block level html and text.
-
- -Linenumbers
- Supports all styles. New token is called LINENUMBER.
- Defaults to NAME if not defined.
-
- Style options
-
- -ALL markups support these text styles:
- b = bold
- i = italic
- u = underline
- -CSS and XHTML has limited support for borders:
- HTML markup functions will ignore these.
- Optional: Border color in RGB hex
- Defaults to the text forecolor.
- #rrggbb = border color
- Border size:
- l = thick
- m = medium
- t = thin
- Border type:
- - = dashed
- . = dotted
- s = solid
- d = double
- g = groove
- r = ridge
- n = inset
- o = outset
- You can specify multiple sides,
- they will all use the same style.
- Optional: Default is full border.
- v = bottom
- < = left
- > = right
- ^ = top
- NOTE: Specify the styles you want.
- The markups will ignore unsupported styles
- Also note not all browsers can show these options
-
- -All tokens default to NAME if not defined
- so the only absolutely critical ones to define are:
- NAME, ERRORTOKEN, PAGEBACKGROUND
-
-----------------------------------------------------------------------------
-
-Example usage::
-
- # import
- import PySourceColor as psc
- psc.convert('c:/Python22/PySourceColor.py', colors=psc.idle, show=1)
-
- # from module import *
- from PySourceColor import *
- convert('c:/Python22/Lib', colors=lite, markup="css",
- header='#$#<b>This is a simpe heading</b><hr/>')
-
- # How to use a custom colorscheme, and most of the 'features'
- from PySourceColor import *
- new = {
- ERRORTOKEN: ('bui','#FF8080',''),
- DECORATOR_NAME: ('s','#AACBBC',''),
- DECORATOR: ('n','#333333',''),
- NAME: ('t.<v','#1133AA','#DDFF22'),
- NUMBER: ('','#236676','#FF5555'),
- OPERATOR: ('b','#454567','#BBBB11'),
- MATH_OPERATOR: ('','#935623','#423afb'),
- BRACKETS: ('b','#ac34bf','#6457a5'),
- COMMENT: ('t-#0022FF','#545366','#AABBFF'),
- DOUBLECOMMENT: ('<l#553455','#553455','#FF00FF'),
- CLASS_NAME: ('m^v-','#000000','#FFFFFF'),
- DEF_NAME: ('l=<v','#897845','#000022'),
- KEYWORD: ('.b','#345345','#FFFF22'),
- SINGLEQUOTE: ('mn','#223344','#AADDCC'),
- SINGLEQUOTE_R: ('','#344522',''),
- SINGLEQUOTE_U: ('','#234234',''),
- DOUBLEQUOTE: ('m#0022FF','#334421',''),
- DOUBLEQUOTE_R: ('','#345345',''),
- DOUBLEQUOTE_U: ('','#678673',''),
- TRIPLESINGLEQUOTE: ('tv','#FFFFFF','#000000'),
- TRIPLESINGLEQUOTE_R: ('tbu','#443256','#DDFFDA'),
- TRIPLESINGLEQUOTE_U: ('','#423454','#DDFFDA'),
- TRIPLEDOUBLEQUOTE: ('li#236fd3b<>','#000000','#FFFFFF'),
- TRIPLEDOUBLEQUOTE_R: ('tub','#000000','#FFFFFF'),
- TRIPLEDOUBLEQUOTE_U: ('-', '#CCAABB','#FFFAFF'),
- LINENUMBER: ('ib-','#ff66aa','#7733FF'),]
- TEXT: ('','#546634',''),
- PAGEBACKGROUND: '#FFFAAA',
- }
- if __name__ == '__main__':
- import sys
- convert(sys.argv[1], './xhtml.html', colors=new, markup='xhtml', show=1,
- linenumbers=1)
- convert(sys.argv[1], './html.html', colors=new, markup='html', show=1,
- linenumbers=1)
-
-"""
-
-__all__ = ['ERRORTOKEN','DECORATOR_NAME', 'DECORATOR', 'ARGS', 'EXTRASPACE',
- 'NAME', 'NUMBER', 'OPERATOR', 'COMMENT', 'MATH_OPERATOR',
- 'DOUBLECOMMENT', 'CLASS_NAME', 'DEF_NAME', 'KEYWORD', 'BRACKETS',
- 'SINGLEQUOTE','SINGLEQUOTE_R','SINGLEQUOTE_U','DOUBLEQUOTE',
- 'DOUBLEQUOTE_R', 'DOUBLEQUOTE_U', 'TRIPLESINGLEQUOTE', 'TEXT',
- 'TRIPLESINGLEQUOTE_R', 'TRIPLESINGLEQUOTE_U', 'TRIPLEDOUBLEQUOTE',
- 'TRIPLEDOUBLEQUOTE_R', 'TRIPLEDOUBLEQUOTE_U', 'PAGEBACKGROUND',
- 'LINENUMBER', 'CODESTART', 'CODEEND', 'PY', 'TOKEN_NAMES', 'CSSHOOK',
- 'null', 'mono', 'lite', 'dark','dark2', 'pythonwin','idle',
- 'viewcvs', 'Usage', 'cli', 'str2stdout', 'path2stdout', 'Parser',
- 'str2file', 'str2html', 'str2css', 'str2markup', 'path2file',
- 'path2html', 'convert', 'walkdir', 'defaultColors', 'showpage',
- 'pageconvert','tagreplace', 'MARKUPDICT']
-__title__ = 'PySourceColor'
-__version__ = "2.1a"
-__date__ = '25 April 2005'
-__author__ = "M.E.Farmer Jr."
-__credits__ = '''This was originally based on a python recipe
-submitted by Jürgen Hermann to ASPN. Now based on the voices in my head.
-M.E.Farmer 2004, 2005
-Python license
-'''
-import os
-import sys
-import time
-import glob
-import getopt
-import keyword
-import token
-import tokenize
-import traceback
-try :
- import cStringIO as StringIO
-except:
- import StringIO
-# Do not edit
-NAME = token.NAME
-NUMBER = token.NUMBER
-COMMENT = tokenize.COMMENT
-OPERATOR = token.OP
-ERRORTOKEN = token.ERRORTOKEN
-ARGS = token.NT_OFFSET + 1
-DOUBLECOMMENT = token.NT_OFFSET + 2
-CLASS_NAME = token.NT_OFFSET + 3
-DEF_NAME = token.NT_OFFSET + 4
-KEYWORD = token.NT_OFFSET + 5
-SINGLEQUOTE = token.NT_OFFSET + 6
-SINGLEQUOTE_R = token.NT_OFFSET + 7
-SINGLEQUOTE_U = token.NT_OFFSET + 8
-DOUBLEQUOTE = token.NT_OFFSET + 9
-DOUBLEQUOTE_R = token.NT_OFFSET + 10
-DOUBLEQUOTE_U = token.NT_OFFSET + 11
-TRIPLESINGLEQUOTE = token.NT_OFFSET + 12
-TRIPLESINGLEQUOTE_R = token.NT_OFFSET + 13
-TRIPLESINGLEQUOTE_U = token.NT_OFFSET + 14
-TRIPLEDOUBLEQUOTE = token.NT_OFFSET + 15
-TRIPLEDOUBLEQUOTE_R = token.NT_OFFSET + 16
-TRIPLEDOUBLEQUOTE_U = token.NT_OFFSET + 17
-PAGEBACKGROUND = token.NT_OFFSET + 18
-DECORATOR = token.NT_OFFSET + 19
-DECORATOR_NAME = token.NT_OFFSET + 20
-BRACKETS = token.NT_OFFSET + 21
-MATH_OPERATOR = token.NT_OFFSET + 22
-LINENUMBER = token.NT_OFFSET + 23
-TEXT = token.NT_OFFSET + 24
-PY = token.NT_OFFSET + 25
-CODESTART = token.NT_OFFSET + 26
-CODEEND = token.NT_OFFSET + 27
-CSSHOOK = token.NT_OFFSET + 28
-EXTRASPACE = token.NT_OFFSET + 29
-
-# markup classname lookup
-MARKUPDICT = {
- ERRORTOKEN: 'py_err',
- DECORATOR_NAME: 'py_decn',
- DECORATOR: 'py_dec',
- ARGS: 'py_args',
- NAME: 'py_name',
- NUMBER: 'py_num',
- OPERATOR: 'py_op',
- COMMENT: 'py_com',
- DOUBLECOMMENT: 'py_dcom',
- CLASS_NAME: 'py_clsn',
- DEF_NAME: 'py_defn',
- KEYWORD: 'py_key',
- SINGLEQUOTE: 'py_sq',
- SINGLEQUOTE_R: 'py_sqr',
- SINGLEQUOTE_U: 'py_squ',
- DOUBLEQUOTE: 'py_dq',
- DOUBLEQUOTE_R: 'py_dqr',
- DOUBLEQUOTE_U: 'py_dqu',
- TRIPLESINGLEQUOTE: 'py_tsq',
- TRIPLESINGLEQUOTE_R: 'py_tsqr',
- TRIPLESINGLEQUOTE_U: 'py_tsqu',
- TRIPLEDOUBLEQUOTE: 'py_tdq',
- TRIPLEDOUBLEQUOTE_R: 'py_tdqr',
- TRIPLEDOUBLEQUOTE_U: 'py_tdqu',
- BRACKETS: 'py_bra',
- MATH_OPERATOR: 'py_mop',
- LINENUMBER: 'py_lnum',
- TEXT: 'py_text',
- }
-# might help users that want to create custom schemes
-TOKEN_NAMES= {
- ERRORTOKEN:'ERRORTOKEN',
- DECORATOR_NAME:'DECORATOR_NAME',
- DECORATOR:'DECORATOR',
- ARGS:'ARGS',
- NAME:'NAME',
- NUMBER:'NUMBER',
- OPERATOR:'OPERATOR',
- COMMENT:'COMMENT',
- DOUBLECOMMENT:'DOUBLECOMMENT',
- CLASS_NAME:'CLASS_NAME',
- DEF_NAME:'DEF_NAME',
- KEYWORD:'KEYWORD',
- SINGLEQUOTE:'SINGLEQUOTE',
- SINGLEQUOTE_R:'SINGLEQUOTE_R',
- SINGLEQUOTE_U:'SINGLEQUOTE_U',
- DOUBLEQUOTE:'DOUBLEQUOTE',
- DOUBLEQUOTE_R:'DOUBLEQUOTE_R',
- DOUBLEQUOTE_U:'DOUBLEQUOTE_U',
- TRIPLESINGLEQUOTE:'TRIPLESINGLEQUOTE',
- TRIPLESINGLEQUOTE_R:'TRIPLESINGLEQUOTE_R',
- TRIPLESINGLEQUOTE_U:'TRIPLESINGLEQUOTE_U',
- TRIPLEDOUBLEQUOTE:'TRIPLEDOUBLEQUOTE',
- TRIPLEDOUBLEQUOTE_R:'TRIPLEDOUBLEQUOTE_R',
- TRIPLEDOUBLEQUOTE_U:'TRIPLEDOUBLEQUOTE_U',
- BRACKETS:'BRACKETS',
- MATH_OPERATOR:'MATH_OPERATOR',
- LINENUMBER:'LINENUMBER',
- TEXT:'TEXT',
- PAGEBACKGROUND:'PAGEBACKGROUND',
- }
-
-######################################################################
-# Edit colors and styles to taste
-# Create your own scheme, just copy one below , rename and edit.
-# Custom styles must at least define NAME, ERRORTOKEN, PAGEBACKGROUND,
-# all missing elements will default to NAME.
-# See module docstring for details on style attributes.
-######################################################################
-# Copy null and use it as a starter colorscheme.
-null = {# tokentype: ('tags border_color', 'textforecolor', 'textbackcolor')
- ERRORTOKEN: ('','#000000',''),# Error token
- DECORATOR_NAME: ('','#000000',''),# Decorator name
- DECORATOR: ('','#000000',''),# @ symbol
- ARGS: ('','#000000',''),# class,def,deco arguments
- NAME: ('','#000000',''),# All other python text
- NUMBER: ('','#000000',''),# 0->10
- OPERATOR: ('','#000000',''),# ':','<=',';',',','.','==', etc
- MATH_OPERATOR: ('','#000000',''),# '+','-','=','','**',etc
- BRACKETS: ('','#000000',''),# '[',']','(',')','{','}'
- COMMENT: ('','#000000',''),# Single comment
- DOUBLECOMMENT: ('','#000000',''),## Double comment
- CLASS_NAME: ('','#000000',''),# Class name
- DEF_NAME: ('','#000000',''),# Def name
- KEYWORD: ('','#000000',''),# Python keywords
- SINGLEQUOTE: ('','#000000',''),# 'SINGLEQUOTE'
- SINGLEQUOTE_R: ('','#000000',''),# r'SINGLEQUOTE'
- SINGLEQUOTE_U: ('','#000000',''),# u'SINGLEQUOTE'
- DOUBLEQUOTE: ('','#000000',''),# "DOUBLEQUOTE"
- DOUBLEQUOTE_R: ('','#000000',''),# r"DOUBLEQUOTE"
- DOUBLEQUOTE_U: ('','#000000',''),# u"DOUBLEQUOTE"
- TRIPLESINGLEQUOTE: ('','#000000',''),# '''TRIPLESINGLEQUOTE'''
- TRIPLESINGLEQUOTE_R: ('','#000000',''),# r'''TRIPLESINGLEQUOTE'''
- TRIPLESINGLEQUOTE_U: ('','#000000',''),# u'''TRIPLESINGLEQUOTE'''
- TRIPLEDOUBLEQUOTE: ('','#000000',''),# """TRIPLEDOUBLEQUOTE"""
- TRIPLEDOUBLEQUOTE_R: ('','#000000',''),# r"""TRIPLEDOUBLEQUOTE"""
- TRIPLEDOUBLEQUOTE_U: ('','#000000',''),# u"""TRIPLEDOUBLEQUOTE"""
- TEXT: ('','#000000',''),# non python text
- LINENUMBER: ('>ti#555555','#000000',''),# Linenumbers
- PAGEBACKGROUND: '#FFFFFF'# set the page background
- }
-
-mono = {
- ERRORTOKEN: ('s#FF0000','#FF8080',''),
- DECORATOR_NAME: ('bu','#000000',''),
- DECORATOR: ('b','#000000',''),
- ARGS: ('b','#555555',''),
- NAME: ('','#000000',''),
- NUMBER: ('b','#000000',''),
- OPERATOR: ('b','#000000',''),
- MATH_OPERATOR: ('b','#000000',''),
- BRACKETS: ('b','#000000',''),
- COMMENT: ('i','#999999',''),
- DOUBLECOMMENT: ('b','#999999',''),
- CLASS_NAME: ('bu','#000000',''),
- DEF_NAME: ('b','#000000',''),
- KEYWORD: ('b','#000000',''),
- SINGLEQUOTE: ('','#000000',''),
- SINGLEQUOTE_R: ('','#000000',''),
- SINGLEQUOTE_U: ('','#000000',''),
- DOUBLEQUOTE: ('','#000000',''),
- DOUBLEQUOTE_R: ('','#000000',''),
- DOUBLEQUOTE_U: ('','#000000',''),
- TRIPLESINGLEQUOTE: ('','#000000',''),
- TRIPLESINGLEQUOTE_R: ('','#000000',''),
- TRIPLESINGLEQUOTE_U: ('','#000000',''),
- TRIPLEDOUBLEQUOTE: ('i','#000000',''),
- TRIPLEDOUBLEQUOTE_R: ('i','#000000',''),
- TRIPLEDOUBLEQUOTE_U: ('i','#000000',''),
- TEXT: ('','#000000',''),
- LINENUMBER: ('>ti#555555','#000000',''),
- PAGEBACKGROUND: '#FFFFFF'
- }
-
-dark = {
- ERRORTOKEN: ('s#FF0000','#FF8080',''),
- DECORATOR_NAME: ('b','#FFBBAA',''),
- DECORATOR: ('b','#CC5511',''),
- ARGS: ('b','#DDDDFF',''),
- NAME: ('','#DDDDDD',''),
- NUMBER: ('','#FF0000',''),
- OPERATOR: ('b','#FAF785',''),
- MATH_OPERATOR: ('b','#FAF785',''),
- BRACKETS: ('b','#FAF785',''),
- COMMENT: ('','#45FCA0',''),
- DOUBLECOMMENT: ('i','#A7C7A9',''),
- CLASS_NAME: ('b','#B666FD',''),
- DEF_NAME: ('b','#EBAE5C',''),
- KEYWORD: ('b','#8680FF',''),
- SINGLEQUOTE: ('','#F8BAFE',''),
- SINGLEQUOTE_R: ('','#F8BAFE',''),
- SINGLEQUOTE_U: ('','#F8BAFE',''),
- DOUBLEQUOTE: ('','#FF80C0',''),
- DOUBLEQUOTE_R: ('','#FF80C0',''),
- DOUBLEQUOTE_U: ('','#FF80C0',''),
- TRIPLESINGLEQUOTE: ('','#FF9595',''),
- TRIPLESINGLEQUOTE_R: ('','#FF9595',''),
- TRIPLESINGLEQUOTE_U: ('','#FF9595',''),
- TRIPLEDOUBLEQUOTE: ('','#B3FFFF',''),
- TRIPLEDOUBLEQUOTE_R: ('','#B3FFFF',''),
- TRIPLEDOUBLEQUOTE_U: ('','#B3FFFF',''),
- TEXT: ('','#FFFFFF',''),
- LINENUMBER: ('>mi#555555','#bbccbb','#333333'),
- PAGEBACKGROUND: '#000000'
- }
-
-dark2 = {
- ERRORTOKEN: ('','#FF0000',''),
- DECORATOR_NAME: ('b','#FFBBAA',''),
- DECORATOR: ('b','#CC5511',''),
- ARGS: ('b','#DDDDDD',''),
- NAME: ('','#C0C0C0',''),
- NUMBER: ('b','#00FF00',''),
- OPERATOR: ('b','#FF090F',''),
- MATH_OPERATOR: ('b','#EE7020',''),
- BRACKETS: ('b','#FFB90F',''),
- COMMENT: ('i','#D0D000','#522000'),#'#88AA88','#11111F'),
- DOUBLECOMMENT: ('i','#D0D000','#522000'),#'#77BB77','#11111F'),
- CLASS_NAME: ('b','#DD4080',''),
- DEF_NAME: ('b','#FF8040',''),
- KEYWORD: ('b','#4726d1',''),
- SINGLEQUOTE: ('','#8080C0',''),
- SINGLEQUOTE_R: ('','#8080C0',''),
- SINGLEQUOTE_U: ('','#8080C0',''),
- DOUBLEQUOTE: ('','#ADB9F1',''),
- DOUBLEQUOTE_R: ('','#ADB9F1',''),
- DOUBLEQUOTE_U: ('','#ADB9F1',''),
- TRIPLESINGLEQUOTE: ('','#00C1C1',''),#A050C0
- TRIPLESINGLEQUOTE_R: ('','#00C1C1',''),#A050C0
- TRIPLESINGLEQUOTE_U: ('','#00C1C1',''),#A050C0
- TRIPLEDOUBLEQUOTE: ('','#33E3E3',''),#B090E0
- TRIPLEDOUBLEQUOTE_R: ('','#33E3E3',''),#B090E0
- TRIPLEDOUBLEQUOTE_U: ('','#33E3E3',''),#B090E0
- TEXT: ('','#C0C0C0',''),
- LINENUMBER: ('>mi#555555','#bbccbb','#333333'),
- PAGEBACKGROUND: '#000000'
- }
-
-lite = {
- ERRORTOKEN: ('s#FF0000','#FF8080',''),
- DECORATOR_NAME: ('b','#BB4422',''),
- DECORATOR: ('b','#3333AF',''),
- ARGS: ('b','#000000',''),
- NAME: ('','#333333',''),
- NUMBER: ('b','#DD2200',''),
- OPERATOR: ('b','#000000',''),
- MATH_OPERATOR: ('b','#000000',''),
- BRACKETS: ('b','#000000',''),
- COMMENT: ('','#007F00',''),
- DOUBLECOMMENT: ('','#608060',''),
- CLASS_NAME: ('b','#0000DF',''),
- DEF_NAME: ('b','#9C7A00',''),#f09030
- KEYWORD: ('b','#0000AF',''),
- SINGLEQUOTE: ('','#600080',''),
- SINGLEQUOTE_R: ('','#600080',''),
- SINGLEQUOTE_U: ('','#600080',''),
- DOUBLEQUOTE: ('','#A0008A',''),
- DOUBLEQUOTE_R: ('','#A0008A',''),
- DOUBLEQUOTE_U: ('','#A0008A',''),
- TRIPLESINGLEQUOTE: ('','#337799',''),
- TRIPLESINGLEQUOTE_R: ('','#337799',''),
- TRIPLESINGLEQUOTE_U: ('','#337799',''),
- TRIPLEDOUBLEQUOTE: ('','#1166AA',''),
- TRIPLEDOUBLEQUOTE_R: ('','#1166AA',''),
- TRIPLEDOUBLEQUOTE_U: ('','#1166AA',''),
- TEXT: ('','#000000',''),
- LINENUMBER: ('>ti#555555','#000000',''),
- PAGEBACKGROUND: '#FFFFFF'
- }
-
-idle = {
- ERRORTOKEN: ('s#FF0000','#FF8080',''),
- DECORATOR_NAME: ('','#900090',''),
- DECORATOR: ('','#FF7700',''),
- NAME: ('','#000000',''),
- NUMBER: ('','#000000',''),
- OPERATOR: ('','#000000',''),
- MATH_OPERATOR: ('','#000000',''),
- BRACKETS: ('','#000000',''),
- COMMENT: ('','#DD0000',''),
- DOUBLECOMMENT: ('','#DD0000',''),
- CLASS_NAME: ('','#0000FF',''),
- DEF_NAME: ('','#0000FF',''),
- KEYWORD: ('','#FF7700',''),
- SINGLEQUOTE: ('','#00AA00',''),
- SINGLEQUOTE_R: ('','#00AA00',''),
- SINGLEQUOTE_U: ('','#00AA00',''),
- DOUBLEQUOTE: ('','#00AA00',''),
- DOUBLEQUOTE_R: ('','#00AA00',''),
- DOUBLEQUOTE_U: ('','#00AA00',''),
- TRIPLESINGLEQUOTE: ('','#00AA00',''),
- TRIPLESINGLEQUOTE_R: ('','#00AA00',''),
- TRIPLESINGLEQUOTE_U: ('','#00AA00',''),
- TRIPLEDOUBLEQUOTE: ('','#00AA00',''),
- TRIPLEDOUBLEQUOTE_R: ('','#00AA00',''),
- TRIPLEDOUBLEQUOTE_U: ('','#00AA00',''),
- TEXT: ('','#000000',''),
- LINENUMBER: ('>ti#555555','#000000',''),
- PAGEBACKGROUND: '#FFFFFF'
- }
-
-pythonwin = {
- ERRORTOKEN: ('s#FF0000','#FF8080',''),
- DECORATOR_NAME: ('b','#DD0080',''),
- DECORATOR: ('b','#000080',''),
- ARGS: ('','#000000',''),
- NAME: ('','#303030',''),
- NUMBER: ('','#008080',''),
- OPERATOR: ('','#000000',''),
- MATH_OPERATOR: ('','#000000',''),
- BRACKETS: ('','#000000',''),
- COMMENT: ('','#007F00',''),
- DOUBLECOMMENT: ('','#7F7F7F',''),
- CLASS_NAME: ('b','#0000FF',''),
- DEF_NAME: ('b','#007F7F',''),
- KEYWORD: ('b','#000080',''),
- SINGLEQUOTE: ('','#808000',''),
- SINGLEQUOTE_R: ('','#808000',''),
- SINGLEQUOTE_U: ('','#808000',''),
- DOUBLEQUOTE: ('','#808000',''),
- DOUBLEQUOTE_R: ('','#808000',''),
- DOUBLEQUOTE_U: ('','#808000',''),
- TRIPLESINGLEQUOTE: ('','#808000',''),
- TRIPLESINGLEQUOTE_R: ('','#808000',''),
- TRIPLESINGLEQUOTE_U: ('','#808000',''),
- TRIPLEDOUBLEQUOTE: ('','#808000',''),
- TRIPLEDOUBLEQUOTE_R: ('','#808000',''),
- TRIPLEDOUBLEQUOTE_U: ('','#808000',''),
- TEXT: ('','#303030',''),
- LINENUMBER: ('>ti#555555','#000000',''),
- PAGEBACKGROUND: '#FFFFFF'
- }
-
-viewcvs = {
- ERRORTOKEN: ('s#FF0000','#FF8080',''),
- DECORATOR_NAME: ('','#000000',''),
- DECORATOR: ('','#000000',''),
- ARGS: ('','#000000',''),
- NAME: ('','#000000',''),
- NUMBER: ('','#000000',''),
- OPERATOR: ('','#000000',''),
- MATH_OPERATOR: ('','#000000',''),
- BRACKETS: ('','#000000',''),
- COMMENT: ('i','#b22222',''),
- DOUBLECOMMENT: ('i','#b22222',''),
- CLASS_NAME: ('','#000000',''),
- DEF_NAME: ('b','#0000ff',''),
- KEYWORD: ('b','#a020f0',''),
- SINGLEQUOTE: ('b','#bc8f8f',''),
- SINGLEQUOTE_R: ('b','#bc8f8f',''),
- SINGLEQUOTE_U: ('b','#bc8f8f',''),
- DOUBLEQUOTE: ('b','#bc8f8f',''),
- DOUBLEQUOTE_R: ('b','#bc8f8f',''),
- DOUBLEQUOTE_U: ('b','#bc8f8f',''),
- TRIPLESINGLEQUOTE: ('b','#bc8f8f',''),
- TRIPLESINGLEQUOTE_R: ('b','#bc8f8f',''),
- TRIPLESINGLEQUOTE_U: ('b','#bc8f8f',''),
- TRIPLEDOUBLEQUOTE: ('b','#bc8f8f',''),
- TRIPLEDOUBLEQUOTE_R: ('b','#bc8f8f',''),
- TRIPLEDOUBLEQUOTE_U: ('b','#bc8f8f',''),
- TEXT: ('','#000000',''),
- LINENUMBER: ('>ti#555555','#000000',''),
- PAGEBACKGROUND: '#FFFFFF'
- }
-
-defaultColors = lite
-
-def Usage():
- doc = """
- -----------------------------------------------------------------------------
- PySourceColor.py ver: %s
- -----------------------------------------------------------------------------
- Module summary:
- This module is designed to colorize python source code.
- Input--->python source
- Output-->colorized (html, html4.01/css, xhtml1.0)
- Standalone:
- This module will work from the command line with options.
- This module will work with redirected stdio.
- Imported:
- This module can be imported and used directly in your code.
- -----------------------------------------------------------------------------
- Command line options:
- -h, --help
- Optional-> Display this help message.
- -t, --test
- Optional-> Will ignore all others flags but --profile
- test all schemes and markup combinations
- -p, --profile
- Optional-> Works only with --test or -t
- runs profile.py and makes the test work in quiet mode.
- -i, --in, --input
- Optional-> If you give input on stdin.
- Use any of these for the current dir (.,cwd)
- Input can be file or dir.
- Input from stdin use one of the following (-,stdin)
- If stdin is used as input stdout is output unless specified.
- -o, --out, --output
- Optional-> output dir for the colorized source.
- default: output dir is the input dir.
- To output html to stdout use one of the following (-,stdout)
- Stdout can be used without stdin if you give a file as input.
- -c, --color
- Optional-> null, mono, dark, dark2, lite, idle, pythonwin, viewcvs
- default: dark
- -s, --show
- Optional-> Show page after creation.
- default: no show
- -m, --markup
- Optional-> html, css, xhtml
- css, xhtml also support external stylesheets (-e,--external)
- default: HTML
- -e, --external
- Optional-> use with css, xhtml
- Writes an style sheet instead of embedding it in the page
- saves it as pystyle.css in the same directory.
- html markup will silently ignore this flag.
- -H, --header
- Opional-> add a page header to the top of the output
- -H
- Builtin header (name,date,hrule)
- --header
- You must specify a filename.
- The header file must be valid html
- and must handle its own font colors.
- ex. --header c:/tmp/header.txt
- -F, --footer
- Opional-> add a page footer to the bottom of the output
- -F
- Builtin footer (hrule,name,date)
- --footer
- You must specify a filename.
- The footer file must be valid html
- and must handle its own font colors.
- ex. --footer c:/tmp/footer.txt
- -l, --linenumbers
- Optional-> default is no linenumbers
- Adds line numbers to the start of each line in the code.
- --convertpage
- Given a webpage that has code embedded in tags it will
- convert embedded code to colorized html.
- (see pageconvert for details)
- -----------------------------------------------------------------------------
- Option usage:
- # Test and show pages
- python PySourceColor.py -t -s
- # Test and only show profile results
- python PySourceColor.py -t -p
- # Colorize all .py,.pyw files in cwdir you can also use: (.,cwd)
- python PySourceColor.py -i .
- # Using long options w/ =
- python PySourceColor.py --in=c:/myDir/my.py --color=lite --show
- # Using short options w/out =
- python PySourceColor.py -i c:/myDir/ -c idle -m css -e
- # Using any mix
- python PySourceColor.py --in . -o=c:/myDir --show
- # Place a custom header on your files
- python PySourceColor.py -i . -o c:/tmp -m xhtml --header c:/header.txt
- -----------------------------------------------------------------------------
- Stdio usage:
- # Stdio using no options
- python PySourceColor.py < c:/MyFile.py > c:/tmp/MyFile.html
- # Using stdin alone automatically uses stdout for output: (stdin,-)
- python PySourceColor.py -i- < c:/MyFile.py > c:/tmp/myfile.html
- # Stdout can also be written to directly from a file instead of stdin
- python PySourceColor.py -i c:/MyFile.py -m css -o- > c:/tmp/myfile.html
- # Stdin can be used as input , but output can still be specified
- python PySourceColor.py -i- -o c:/pydoc.py.html -s < c:/Python22/my.py
- _____________________________________________________________________________
- """
- print doc % (__version__)
- sys.exit(1)
-
-###################################################### Command line interface
-
-def cli():
- """Handle command line args and redirections"""
- try:
- # try to get command line args
- opts, args = getopt.getopt(sys.argv[1:],
- "hseqtplHFi:o:c:m:h:f:",["help", "show", "quiet",
- "test", "external", "linenumbers", "convertpage", "profile",
- "input=", "output=", "color=", "markup=","header=", "footer="])
- except getopt.GetoptError:
- # on error print help information and exit:
- Usage()
- # init some names
- input = None
- output = None
- colorscheme = None
- markup = 'html'
- header = None
- footer = None
- linenumbers = 0
- show = 0
- quiet = 0
- test = 0
- profile = 0
- convertpage = 0
- form = None
- # if we have args then process them
- for o, a in opts:
- if o in ["-h", "--help"]:
- Usage()
- sys.exit()
- if o in ["-o", "--output", "--out"]:
- output = a
- if o in ["-i", "--input", "--in"]:
- input = a
- if input in [".", "cwd"]:
- input = os.getcwd()
- if o in ["-s", "--show"]:
- show = 1
- if o in ["-q", "--quiet"]:
- quiet = 1
- if o in ["-t", "--test"]:
- test = 1
- if o in ["--convertpage"]:
- convertpage = 1
- if o in ["-p", "--profile"]:
- profile = 1
- if o in ["-e", "--external"]:
- form = 'external'
- if o in ["-m", "--markup"]:
- markup = str(a)
- if o in ["-l", "--linenumbers"]:
- linenumbers = 1
- if o in ["--header"]:
- header = str(a)
- elif o == "-H":
- header = ''
- if o in ["--footer"]:
- footer = str(a)
- elif o == "-F":
- footer = ''
- if o in ["-c", "--color"]:
- try:
- colorscheme = globals().get(a.lower())
- except:
- traceback.print_exc()
- Usage()
- if test:
- if profile:
- import profile
- profile.run('_test(show=%s, quiet=%s)'%(show,quiet))
- else:
- # Parse this script in every possible colorscheme and markup
- _test(show,quiet)
- elif input in [None, "-", "stdin"] or output in ["-", "stdout"]:
- # determine if we are going to use stdio
- if input not in [None, "-", "stdin"]:
- if os.path.isfile(input) :
- path2stdout(input, colors=colorscheme, markup=markup,
- linenumbers=linenumbers, header=header,
- footer=footer, form=form)
- else:
- raise PathError, 'File does not exists!'
- else:
- try:
- if sys.stdin.isatty():
- raise InputError, 'Please check input!'
- else:
- if output in [None,"-","stdout"]:
- str2stdout(sys.stdin.read(), colors=colorscheme,
- markup=markup, header=header,
- footer=footer, linenumbers=linenumbers,
- form=form)
- else:
- str2file(sys.stdin.read(), outfile=output, show=show,
- markup=markup, header=header, footer=footer,
- linenumbers=linenumbers, form=form)
- except:
- traceback.print_exc()
- Usage()
- else:
- if os.path.exists(input):
- if convertpage:
- # if there was at least an input given we can proceed
- pageconvert(input, out=output, colors=colorscheme,
- show=show, markup=markup,linenumbers=linenumbers)
- else:
- # if there was at least an input given we can proceed
- convert(source=input, outdir=output, colors=colorscheme,
- show=show, markup=markup, quiet=quiet, header=header,
- footer=footer, linenumbers=linenumbers, form=form)
- else:
- raise PathError, 'File does not exists!'
- Usage()
-
-######################################################### Simple markup tests
-
-def _test(show=0, quiet=0):
- """Test the parser and most of the functions.
-
- There are 19 test total(eight colorschemes in three diffrent markups,
- and a str2file test. Most functions are tested by this.
- """
- fi = sys.argv[0]
- if not fi.endswith('.exe'):# Do not test if frozen as an archive
- # this is a collection of test, most things are covered.
- path2file(fi, '/tmp/null.html', null, show=show, quiet=quiet)
- path2file(fi, '/tmp/null_css.html', null, show=show,
- markup='css', quiet=quiet)
- path2file(fi, '/tmp/mono.html', mono, show=show, quiet=quiet)
- path2file(fi, '/tmp/mono_css.html', mono, show=show,
- markup='css', quiet=quiet)
- path2file(fi, '/tmp/lite.html', lite, show=show, quiet=quiet)
- path2file(fi, '/tmp/lite_css.html', lite, show=show,
- markup='css', quiet=quiet, header='', footer='',
- linenumbers=1)
- path2file(fi, '/tmp/lite_xhtml.html', lite, show=show,
- markup='xhtml', quiet=quiet)
- path2file(fi, '/tmp/dark.html', dark, show=show, quiet=quiet)
- path2file(fi, '/tmp/dark_css.html', dark, show=show,
- markup='css', quiet=quiet, linenumbers=1)
- path2file(fi, '/tmp/dark2.html', dark2, show=show, quiet=quiet)
- path2file(fi, '/tmp/dark2_css.html', dark2, show=show,
- markup='css', quiet=quiet)
- path2file(fi, '/tmp/dark2_xhtml.html', dark2, show=show,
- markup='xhtml', quiet=quiet, header='', footer='',
- linenumbers=1, form='external')
- path2file(fi, '/tmp/idle.html', idle, show=show, quiet=quiet)
- path2file(fi, '/tmp/idle_css.html', idle, show=show,
- markup='css', quiet=quiet)
- path2file(fi, '/tmp/viewcvs.html', viewcvs, show=show,
- quiet=quiet, linenumbers=1)
- path2file(fi, '/tmp/viewcvs_css.html', viewcvs, show=show,
- markup='css', linenumbers=1, quiet=quiet)
- path2file(fi, '/tmp/pythonwin.html', pythonwin, show=show,
- quiet=quiet)
- path2file(fi, '/tmp/pythonwin_css.html', pythonwin, show=show,
- markup='css', quiet=quiet)
- teststr=r'''"""This is a test of decorators and other things"""
-# This should be line 421...
-@whatever(arg,arg2)
-@A @B(arghh) @C
-def LlamaSaysNi(arg='Ni!',arg2="RALPH"):
- """This docstring is deeply disturbed by all the llama references"""
- print '%s The Wonder Llama says %s'% (arg2,arg)
-# So I was like duh!, and he was like ya know?!,
-# and so we were both like huh...wtf!? RTFM!! LOL!!;)
-@staticmethod## Double comments are KewL.
-def LlamasRLumpy():
- """This docstring is too sexy to be here.
- """
- u"""
-=============================
-A Møøse once bit my sister...
-=============================
- """
- ## Relax, this won't hurt a bit, just a simple, painless procedure,
- ## hold still while I get the anesthetizing hammer.
- m = {'three':'1','won':'2','too':'3'}
- o = r'fishy\fishy\fishy/fish\oh/where/is\my/little\..'
- python = uR"""
- No realli! She was Karving her initials øn the møøse with the sharpened end
- of an interspace tøøthbrush given her by Svenge - her brother-in-law -an Oslo
- dentist and star of many Norwegian møvies: "The Høt Hands of an Oslo
- Dentist", "Fillings of Passion", "The Huge Mølars of Horst Nordfink"..."""
- RU"""142 MEXICAN WHOOPING LLAMAS"""#<-Can you fit 142 llamas in a red box?
- n = u' HERMSGERVØRDENBRØTBØRDA ' + """ YUTTE """
- t = """SAMALLNIATNUOMNAIRODAUCE"""+"DENIARTYLLAICEPS04"
- ## We apologise for the fault in the
- ## comments. Those responsible have been
- ## sacked.
- y = '14 NORTH CHILEAN GUANACOS \
-(CLOSELY RELATED TO THE LLAMA)'
- rules = [0,1,2,3,4,5]
- print y'''
- htmlPath = os.path.abspath('/tmp/strtest_lines.html')
- str2file(teststr, htmlPath, colors=dark, markup='xhtml',
- linenumbers=420, show=show)
- _printinfo(" wrote %s" % htmlPath, quiet)
- htmlPath = os.path.abspath('/tmp/strtest_nolines.html')
- str2file(teststr, htmlPath, colors=dark, markup='xhtml',
- show=show)
- _printinfo(" wrote %s" % htmlPath, quiet)
- else:
- Usage()
- return
-
-# emacs wants this: '
-
-####################################################### User funtctions
-
-def str2stdout(sourcestring, colors=None, title='', markup='html',
- header=None, footer=None,
- linenumbers=0, form=None):
- """Converts a code(string) to colorized HTML. Writes to stdout.
-
- form='code',or'snip' (for "<pre>yourcode</pre>" only)
- colors=null,mono,lite,dark,dark2,idle,or pythonwin
- """
- Parser(sourcestring, colors=colors, title=title, markup=markup,
- header=header, footer=footer,
- linenumbers=linenumbers).format(form)
-
-def path2stdout(sourcepath, title='', colors=None, markup='html',
- header=None, footer=None,
- linenumbers=0, form=None):
- """Converts code(file) to colorized HTML. Writes to stdout.
-
- form='code',or'snip' (for "<pre>yourcode</pre>" only)
- colors=null,mono,lite,dark,dark2,idle,or pythonwin
- """
- sourcestring = open(sourcepath).read()
- Parser(sourcestring, colors=colors, title=sourcepath,
- markup=markup, header=header, footer=footer,
- linenumbers=linenumbers).format(form)
-
-def str2html(sourcestring, colors=None, title='',
- markup='html', header=None, footer=None,
- linenumbers=0, form=None):
- """Converts a code(string) to colorized HTML. Returns an HTML string.
-
- form='code',or'snip' (for "<pre>yourcode</pre>" only)
- colors=null,mono,lite,dark,dark2,idle,or pythonwin
- """
- stringIO = StringIO.StringIO()
- Parser(sourcestring, colors=colors, title=title, out=stringIO,
- markup=markup, header=header, footer=footer,
- linenumbers=linenumbers).format(form)
- stringIO.seek(0)
- return stringIO.read()
-
-def str2css(sourcestring, colors=None, title='',
- markup='css', header=None, footer=None,
- linenumbers=0, form=None):
- """Converts a code string to colorized CSS/HTML. Returns CSS/HTML string
-
- If form != None then this will return (stylesheet_str, code_str)
- colors=null,mono,lite,dark,dark2,idle,or pythonwin
- """
- if markup.lower() not in ['css' ,'xhtml']:
- markup = 'css'
- stringIO = StringIO.StringIO()
- parse = Parser(sourcestring, colors=colors, title=title,
- out=stringIO, markup=markup,
- header=header, footer=footer,
- linenumbers=linenumbers)
- parse.format(form)
- stringIO.seek(0)
- if form != None:
- return parse._sendCSSStyle(external=1), stringIO.read()
- else:
- return None, stringIO.read()
-
-def str2markup(sourcestring, colors=None, title = '',
- markup='xhtml', header=None, footer=None,
- linenumbers=0, form=None):
- """ Convert code strings into ([stylesheet or None], colorized string) """
- if markup.lower() == 'html':
- return None, str2html(sourcestring, colors=colors, title=title,
- header=header, footer=footer, markup=markup,
- linenumbers=linenumbers, form=form)
- else:
- return str2css(sourcestring, colors=colors, title=title,
- header=header, footer=footer, markup=markup,
- linenumbers=linenumbers, form=form)
-
-def str2file(sourcestring, outfile, colors=None, title='',
- markup='html', header=None, footer=None,
- linenumbers=0, show=0, dosheet=1, form=None):
- """Converts a code string to a file.
-
- makes no attempt at correcting bad pathnames
- """
- css , html = str2markup(sourcestring, colors=colors, title='',
- markup=markup, header=header, footer=footer,
- linenumbers=linenumbers, form=form)
- # write html
- f = open(outfile,'wt')
- f.writelines(html)
- f.close()
- #write css
- if css != None and dosheet:
- dir = os.path.dirname(outfile)
- outcss = os.path.join(dir,'pystyle.css')
- f = open(outcss,'wt')
- f.writelines(css)
- f.close()
- if show:
- showpage(outfile)
-
-def path2html(sourcepath, colors=None, markup='html',
- header=None, footer=None,
- linenumbers=0, form=None):
- """Converts code(file) to colorized HTML. Returns an HTML string.
-
- form='code',or'snip' (for "<pre>yourcode</pre>" only)
- colors=null,mono,lite,dark,dark2,idle,or pythonwin
- """
- stringIO = StringIO.StringIO()
- sourcestring = open(sourcepath).read()
- Parser(sourcestring, colors, title=sourcepath, out=stringIO,
- markup=markup, header=header, footer=footer,
- linenumbers=linenumbers).format(form)
- stringIO.seek(0)
- return stringIO.read()
-
-def convert(source, outdir=None, colors=None,
- show=0, markup='html', quiet=0,
- header=None, footer=None, linenumbers=0, form=None):
- """Takes a file or dir as input and places the html in the outdir.
-
- If outdir is none it defaults to the input dir
- """
- count=0
- # If it is a filename then path2file
- if not os.path.isdir(source):
- if os.path.isfile(source):
- count+=1
- path2file(source, outdir, colors, show, markup,
- quiet, form, header, footer, linenumbers, count)
- else:
- raise PathError, 'File does not exist!'
- # If we pass in a dir we need to walkdir for files.
- # Then we need to colorize them with path2file
- else:
- fileList = walkdir(source)
- if fileList != None:
- # make sure outdir is a dir
- if outdir != None:
- if os.path.splitext(outdir)[1] != '':
- outdir = os.path.split(outdir)[0]
- for item in fileList:
- count+=1
- path2file(item, outdir, colors, show, markup,
- quiet, form, header, footer, linenumbers, count)
- _printinfo('Completed colorizing %s files.'%str(count), quiet)
- else:
- _printinfo("No files to convert in dir.", quiet)
-
-def path2file(sourcePath, out=None, colors=None, show=0,
- markup='html', quiet=0, form=None,
- header=None, footer=None, linenumbers=0, count=1):
- """ Converts python source to html file"""
- # If no outdir is given we use the sourcePath
- if out == None:#this is a guess
- htmlPath = sourcePath + '.html'
- else:
- # If we do give an out_dir, and it does
- # not exist , it will be created.
- if os.path.splitext(out)[1] == '':
- if not os.path.isdir(out):
- os.makedirs(out)
- sourceName = os.path.basename(sourcePath)
- htmlPath = os.path.join(out,sourceName)+'.html'
- # If we do give an out_name, and its dir does
- # not exist , it will be created.
- else:
- outdir = os.path.split(out)[0]
- if not os.path.isdir(outdir):
- os.makedirs(outdir)
- htmlPath = out
- htmlPath = os.path.abspath(htmlPath)
- # Open the text and do the parsing.
- source = open(sourcePath).read()
- parse = Parser(source, colors, sourcePath, open(htmlPath, 'wt'),
- markup, header, footer, linenumbers)
- parse.format(form)
- _printinfo(" wrote %s" % htmlPath, quiet)
- # html markup will ignore the external flag, but
- # we need to stop the blank file from being written.
- if form == 'external' and count == 1 and markup != 'html':
- cssSheet = parse._sendCSSStyle(external=1)
- cssPath = os.path.join(os.path.dirname(htmlPath),'pystyle.css')
- css = open(cssPath, 'wt')
- css.write(cssSheet)
- css.close()
- _printinfo(" wrote %s" % cssPath, quiet)
- if show:
- # load HTML page into the default web browser.
- showpage(htmlPath)
- return htmlPath
-
-def tagreplace(sourcestr, colors=lite, markup='xhtml',
- linenumbers=0, dosheet=1, tagstart='<PY>'.lower(),
- tagend='</PY>'.lower(), stylesheet='pystyle.css'):
- """This is a helper function for pageconvert. Returns css, page.
- """
- if markup.lower() != 'html':
- link = '<link rel="stylesheet" href="%s" type="text/css"/></head>'
- css = link%stylesheet
- if sourcestr.find(css) == -1:
- sourcestr = sourcestr.replace('</head>', css, 1)
- starttags = sourcestr.count(tagstart)
- endtags = sourcestr.count(tagend)
- if starttags:
- if starttags == endtags:
- for _ in range(starttags):
- datastart = sourcestr.find(tagstart)
- dataend = sourcestr.find(tagend)
- data = sourcestr[datastart+len(tagstart):dataend]
- data = unescape(data)
- css , data = str2markup(data, colors=colors,
- linenumbers=linenumbers, markup=markup, form='embed')
- start = sourcestr[:datastart]
- end = sourcestr[dataend+len(tagend):]
- sourcestr = ''.join([start,data,end])
- else:
- raise InputError,'Tag mismatch!\nCheck %s,%s tags'%tagstart,tagend
- if not dosheet:
- css = None
- return css, sourcestr
-
-def pageconvert(path, out=None, colors=lite, markup='xhtml', linenumbers=0,
- dosheet=1, tagstart='<PY>'.lower(), tagend='</PY>'.lower(),
- stylesheet='pystyle', show=1, returnstr=0):
- """This function can colorize Python source
-
- that is written in a webpage enclosed in tags.
- """
- if out == None:
- out = os.path.dirname(path)
- infile = open(path, 'r').read()
- css,page = tagreplace(sourcestr=infile,colors=colors,
- markup=markup, linenumbers=linenumbers, dosheet=dosheet,
- tagstart=tagstart, tagend=tagend, stylesheet=stylesheet)
- if not returnstr:
- newpath = os.path.abspath(os.path.join(
- out,'tmp', os.path.basename(path)))
- if not os.path.exists(newpath):
- try:
- os.makedirs(os.path.dirname(newpath))
- except:
- pass#traceback.print_exc()
- #Usage()
- y = open(newpath, 'w')
- y.write(page)
- y.close()
- if css:
- csspath = os.path.abspath(os.path.join(
- out,'tmp','%s.css'%stylesheet))
- x = open(csspath,'w')
- x.write(css)
- x.close()
- if show:
- try:
- os.startfile(newpath)
- except:
- traceback.print_exc()
- return newpath
- else:
- return css, page
-
-##################################################################### helpers
-
-def walkdir(dir):
- """Return a list of .py and .pyw files from a given directory.
-
- This function can be written as a generator Python 2.3, or a genexp
- in Python 2.4. But 2.2 and 2.1 would be left out....
- """
- # Get a list of files that match *.py*
- GLOB_PATTERN = os.path.join(dir, "*.[p][y]*")
- pathlist = glob.glob(GLOB_PATTERN)
- # Now filter out all but py and pyw
- filterlist = [x for x in pathlist
- if x.endswith('.py')
- or x.endswith('.pyw')]
- if filterlist != []:
- # if we have a list send it
- return filterlist
- else:
- return None
-
-def showpage(path):
- """Helper function to open webpages"""
- try:
- import webbrowser
- webbrowser.open_new(os.path.abspath(path))
- except:
- traceback.print_exc()
-
-def _printinfo(message, quiet):
- """Helper to print messages"""
- if not quiet:
- print message
-
-def escape(text):
- """escape text for html. similar to cgi.escape"""
- text = text.replace("&", "&")
- text = text.replace("<", "<")
- text = text.replace(">", ">")
- return text
-
-def unescape(text):
- """unsecape escaped text"""
- text = text.replace(""", '"')
- text = text.replace(">", ">")
- text = text.replace("<", "<")
- text = text.replace("&", "&")
- return text
-
-########################################################### Custom Exceptions
-
-class PySourceColorError(Exception):
- # Base for custom errors
- def __init__(self, msg=''):
- self._msg = msg
- Exception.__init__(self, msg)
- def __repr__(self):
- return self._msg
- __str__ = __repr__
-
-class PathError(PySourceColorError):
- def __init__(self, msg):
- PySourceColorError.__init__(self,
- 'Path error! : %s'% msg)
-
-class InputError(PySourceColorError):
- def __init__(self, msg):
- PySourceColorError.__init__(self,
- 'Input error! : %s'% msg)
-
-########################################################## Python code parser
-
-class Parser(object):
-
- """MoinMoin python parser heavily chopped :)"""
-
- def __init__(self, raw, colors=None, title='', out=sys.stdout,
- markup='html', header=None, footer=None, linenumbers=0):
- """Store the source text & set some flags"""
- if colors == None:
- colors = defaultColors
- self.raw = raw.expandtabs().rstrip()
- self.title = os.path.basename(title)
- self.out = out
- self.line = ''
- self.lasttext = ''
- self.argFlag = 0
- self.classFlag = 0
- self.defFlag = 0
- self.decoratorFlag = 0
- self.external = 0
- self.markup = markup.upper()
- self.colors = colors
- self.header = header
- self.footer = footer
- self.doArgs = 1 # overrides the new tokens
- self.doNames = 1 # overrides the new tokens
- self.doMathOps = 1 # overrides the new tokens
- self.doBrackets = 1 # overrides the new tokens
- self.doURL = 1 # override url conversion
- self.LINENUMHOLDER = "___line___".upper()
- self.LINESTART = "___start___".upper()
- self.skip = 0
- # add space left side of code for padding.Override in color dict.
- self.extraspace = self.colors.get(EXTRASPACE, '')
- # Linenumbers less then zero also have numberlinks
- self.dolinenums = self.linenum = abs(linenumbers)
- if linenumbers < 0:
- self.numberlinks = 1
- else:
- self.numberlinks = 0
-
- def format(self, form=None):
- """Parse and send the colorized source"""
- if form in ('snip','code'):
- self.addEnds = 0
- elif form == 'embed':
- self.addEnds = 0
- self.external = 1
- else:
- if form == 'external':
- self.external = 1
- self.addEnds = 1
-
- # Store line offsets in self.lines
- self.lines = [0, 0]
- pos = 0
-
- # Add linenumbers
- if self.dolinenums:
- start=self.LINENUMHOLDER+' '+self.extraspace
- else:
- start=''+self.extraspace
- newlines = []
- lines = self.raw.splitlines(0)
- for l in lines:
- # span and div escape for customizing and embedding raw text
- if (l.startswith('#$#')
- or l.startswith('#%#')
- or l.startswith('#@#')):
- newlines.append(l)
- else:
- # kludge for line spans in css,xhtml
- if self.markup in ['XHTML','CSS']:
- newlines.append(self.LINESTART+' '+start+l)
- else:
- newlines.append(start+l)
- self.raw = "\n".join(newlines)+'\n'# plus an extra newline at the end
-
- # Gather lines
- while 1:
- pos = self.raw.find('\n', pos) + 1
- if not pos: break
- self.lines.append(pos)
- self.lines.append(len(self.raw))
-
- # Wrap text in a filelike object
- self.pos = 0
- text = StringIO.StringIO(self.raw)
-
- # Markup start
- if self.addEnds:
- self._doPageStart()
- else:
- self._doSnippetStart()
-
- ## Tokenize calls the __call__
- ## function for each token till done.
- # Parse the source and write out the results.
- try:
- tokenize.tokenize(text.readline, self)
- except tokenize.TokenError, ex:
- msg = ex[0]
- line = ex[1][0]
- self.out.write("<h3>ERROR: %s</h3>%s\n"%
- (msg, self.raw[self.lines[line]:]))
- #traceback.print_exc()
-
- # Markup end
- if self.addEnds:
- self._doPageEnd()
- else:
- self._doSnippetEnd()
-
- def __call__(self, toktype, toktext, (srow,scol), (erow,ecol), line):
- """Token handler. Order is important do not rearrange."""
- self.line = line
- # Calculate new positions
- oldpos = self.pos
- newpos = self.lines[srow] + scol
- self.pos = newpos + len(toktext)
- # Handle newlines
- if toktype in (token.NEWLINE, tokenize.NL):
- self.decoratorFlag = self.argFlag = 0
- # kludge for line spans in css,xhtml
- if self.markup in ['XHTML','CSS']:
- self.out.write('</span>')
- self.out.write('\n')
- return
-
- # Send the original whitespace, and tokenize backslashes if present.
- # Tokenizer.py just sends continued line backslashes with whitespace.
- # This is a hack to tokenize continued line slashes as operators.
- # Should continued line backslashes be treated as operators
- # or some other token?
-
- if newpos > oldpos:
- if self.raw[oldpos:newpos].isspace():
- # consume a single space after linestarts and linenumbers
- # had to have them so tokenizer could seperate them.
- # multiline strings are handled by do_Text functions
- if self.lasttext != self.LINESTART \
- and self.lasttext != self.LINENUMHOLDER:
- self.out.write(self.raw[oldpos:newpos])
- else:
- self.out.write(self.raw[oldpos+1:newpos])
- else:
- slash = self.raw[oldpos:newpos].find('\\')+oldpos
- self.out.write(self.raw[oldpos:slash])
- getattr(self, '_send%sText'%(self.markup))(OPERATOR, '\\')
- self.linenum+=1
- # kludge for line spans in css,xhtml
- if self.markup in ['XHTML','CSS']:
- self.out.write('</span>')
- self.out.write(self.raw[slash+1:newpos])
-
- # Skip indenting tokens
- if toktype in (token.INDENT, token.DEDENT):
- self.pos = newpos
- return
-
- # Look for operators
- if token.LPAR <= toktype and toktype <= token.OP:
- # Trap decorators py2.4 >
- if toktext == '@':
- toktype = DECORATOR
- # Set a flag if this was the decorator start so
- # the decorator name and arguments can be identified
- self.decoratorFlag = self.argFlag = 1
- else:
- if self.doArgs:
- # Find the start for arguments
- if toktext == '(' and self.argFlag:
- self.argFlag = 2
- # Find the end for arguments
- elif toktext == ':':
- self.argFlag = 0
- ## Seperate the diffrent operator types
- # Brackets
- if self.doBrackets and toktext in ['[',']','(',')','{','}']:
- toktype = BRACKETS
- # Math operators
- elif self.doMathOps and toktext in ['*=','**=','-=','+=','|=',
- '%=','>>=','<<=','=','^=',
- '/=', '+','-','**','*','/','%']:
- toktype = MATH_OPERATOR
- # Operator
- else:
- toktype = OPERATOR
- # example how flags should work.
- # def fun(arg=argvalue,arg2=argvalue2):
- # 0 1 2 A 1 N 2 A 1 N 0
- if toktext == "=" and self.argFlag == 2:
- self.argFlag = 1
- elif toktext == "," and self.argFlag == 1:
- self.argFlag = 2
- # Look for keywords
- elif toktype == NAME and keyword.iskeyword(toktext):
- toktype = KEYWORD
- # Set a flag if this was the class / def start so
- # the class / def name and arguments can be identified
- if toktext in ['class', 'def']:
- if toktext =='class' and \
- not line[:line.find('class')].endswith('.'):
- self.classFlag = self.argFlag = 1
- elif toktext == 'def' and \
- not line[:line.find('def')].endswith('.'):
- self.defFlag = self.argFlag = 1
- else:
- # must have used a keyword as a name i.e. self.class
- toktype = ERRORTOKEN
-
- # Look for class, def, decorator name
- elif (self.classFlag or self.defFlag or self.decoratorFlag) \
- and self.doNames:
- if self.classFlag:
- self.classFlag = 0
- toktype = CLASS_NAME
- elif self.defFlag:
- self.defFlag = 0
- toktype = DEF_NAME
- elif self.decoratorFlag:
- self.decoratorFlag = 0
- toktype = DECORATOR_NAME
-
- # Look for strings
- # Order of evaluation is important do not change.
- elif toktype == token.STRING:
- text = toktext.lower()
- # TRIPLE DOUBLE QUOTE's
- if (text[:3] == '"""'):
- toktype = TRIPLEDOUBLEQUOTE
- elif (text[:4] == 'r"""'):
- toktype = TRIPLEDOUBLEQUOTE_R
- elif (text[:4] == 'u"""' or
- text[:5] == 'ur"""'):
- toktype = TRIPLEDOUBLEQUOTE_U
- # DOUBLE QUOTE's
- elif (text[:1] == '"'):
- toktype = DOUBLEQUOTE
- elif (text[:2] == 'r"'):
- toktype = DOUBLEQUOTE_R
- elif (text[:2] == 'u"' or
- text[:3] == 'ur"'):
- toktype = DOUBLEQUOTE_U
- # TRIPLE SINGLE QUOTE's
- elif (text[:3] == "'''"):
- toktype = TRIPLESINGLEQUOTE
- elif (text[:4] == "r'''"):
- toktype = TRIPLESINGLEQUOTE_R
- elif (text[:4] == "u'''" or
- text[:5] == "ur'''"):
- toktype = TRIPLESINGLEQUOTE_U
- # SINGLE QUOTE's
- elif (text[:1] == "'"):
- toktype = SINGLEQUOTE
- elif (text[:2] == "r'"):
- toktype = SINGLEQUOTE_R
- elif (text[:2] == "u'" or
- text[:3] == "ur'"):
- toktype = SINGLEQUOTE_U
-
- # test for invalid string declaration
- if self.lasttext.lower() == 'ru':
- toktype = ERRORTOKEN
-
- # Look for comments
- elif toktype == COMMENT:
- if toktext[:2] == "##":
- toktype = DOUBLECOMMENT
- elif toktext[:3] == '#$#':
- toktype = TEXT
- self.textFlag = 'SPAN'
- toktext = toktext[3:]
- elif toktext[:3] == '#%#':
- toktype = TEXT
- self.textFlag = 'DIV'
- toktext = toktext[3:]
- elif toktext[:3] == '#@#':
- toktype = TEXT
- self.textFlag = 'RAW'
- toktext = toktext[3:]
- if self.doURL:
- # this is a 'fake helper function'
- # url(URI,Alias_name) or url(URI)
- url_pos = toktext.find('url(')
- if url_pos != -1:
- before = toktext[:url_pos]
- url = toktext[url_pos+4:]
- splitpoint = url.find(',')
- endpoint = url.find(')')
- after = url[endpoint+1:]
- url = url[:endpoint]
- if splitpoint != -1:
- urlparts = url.split(',',1)
- toktext = '%s<a href="%s">%s</a>%s'%(
- before,urlparts[0],urlparts[1].lstrip(),after)
- else:
- toktext = '%s<a href="%s">%s</a>%s'%(before,url,url,after)
-
- # Seperate errors from decorators
- elif toktype == ERRORTOKEN:
- # Bug fix for < py2.4
- # space between decorators
- if self.argFlag and toktext.isspace():
- #toktype = NAME
- self.out.write(toktext)
- return
- # Bug fix for py2.2 linenumbers with decorators
- elif toktext.isspace():
- # What if we have a decorator after a >>> or ...
- #p = line.find('@')
- #if p >= 0 and not line[:p].isspace():
- #self.out.write(toktext)
- #return
- if self.skip:
- self.skip=0
- return
- else:
- self.out.write(toktext)
- return
- # trap decorators < py2.4
- elif toktext == '@':
- toktype = DECORATOR
- # Set a flag if this was the decorator start so
- # the decorator name and arguments can be identified
- self.decoratorFlag = self.argFlag = 1
-
- # Seperate args from names
- elif (self.argFlag == 2 and
- toktype == NAME and
- toktext != 'None' and
- self.doArgs):
- toktype = ARGS
-
- # Look for line numbers
- # The conversion code for them is in the send_text functions.
- if toktext in [self.LINENUMHOLDER,self.LINESTART]:
- toktype = LINENUMBER
- # if we don't have linenumbers set flag
- # to skip the trailing space from linestart
- if toktext == self.LINESTART and not self.dolinenums \
- or toktext == self.LINENUMHOLDER:
- self.skip=1
-
-
- # Skip blank token that made it thru
- ## bugfix for the last empty tag.
- if toktext == '':
- return
-
- # Last token text history
- self.lasttext = toktext
-
- # escape all but the urls in the comments
- if toktype in (DOUBLECOMMENT, COMMENT):
- if toktext.find('<a href=') == -1:
- toktext = escape(toktext)
- else:
- pass
- elif toktype == TEXT:
- pass
- else:
- toktext = escape(toktext)
-
- # Send text for any markup
- getattr(self, '_send%sText'%(self.markup))(toktype, toktext)
- return
-
- ################################################################# Helpers
-
- def _doSnippetStart(self):
- if self.markup == 'HTML':
- # Start of html snippet
- self.out.write('<pre>\n')
- else:
- # Start of css/xhtml snippet
- self.out.write(self.colors.get(CODESTART,'<pre class="py">\n'))
-
- def _doSnippetEnd(self):
- # End of html snippet
- self.out.write(self.colors.get(CODEEND,'</pre>\n'))
-
- ######################################################## markup selectors
-
- def _getFile(self, filepath):
- try:
- _file = open(filepath,'r')
- content = _file.read()
- _file.close()
- except:
- traceback.print_exc()
- content = ''
- return content
-
- def _doPageStart(self):
- getattr(self, '_do%sStart'%(self.markup))()
-
- def _doPageHeader(self):
- if self.header != None:
- if self.header.find('#$#') != -1 or \
- self.header.find('#$#') != -1 or \
- self.header.find('#%#') != -1:
- self.out.write(self.header[3:])
- else:
- if self.header != '':
- self.header = self._getFile(self.header)
- getattr(self, '_do%sHeader'%(self.markup))()
-
- def _doPageFooter(self):
- if self.footer != None:
- if self.footer.find('#$#') != -1 or \
- self.footer.find('#@#') != -1 or \
- self.footer.find('#%#') != -1:
- self.out.write(self.footer[3:])
- else:
- if self.footer != '':
- self.footer = self._getFile(self.footer)
- getattr(self, '_do%sFooter'%(self.markup))()
-
- def _doPageEnd(self):
- getattr(self, '_do%sEnd'%(self.markup))()
-
- ################################################### color/style retrieval
- ## Some of these are not used anymore but are kept for documentation
-
- def _getLineNumber(self):
- num = self.linenum
- self.linenum+=1
- return str(num).rjust(5)+" "
-
- def _getTags(self, key):
- # style tags
- return self.colors.get(key, self.colors[NAME])[0]
-
- def _getForeColor(self, key):
- # get text foreground color, if not set to black
- color = self.colors.get(key, self.colors[NAME])[1]
- if color[:1] != '#':
- color = '#000000'
- return color
-
- def _getBackColor(self, key):
- # get text background color
- return self.colors.get(key, self.colors[NAME])[2]
-
- def _getPageColor(self):
- # get page background color
- return self.colors.get(PAGEBACKGROUND, '#FFFFFF')
-
- def _getStyle(self, key):
- # get the token style from the color dictionary
- return self.colors.get(key, self.colors[NAME])
-
- def _getMarkupClass(self, key):
- # get the markup class name from the markup dictionary
- return MARKUPDICT.get(key, MARKUPDICT[NAME])
-
- def _getDocumentCreatedBy(self):
- return '<!--This document created by %s ver.%s on: %s-->\n'%(
- __title__,__version__,time.ctime())
-
- ################################################### HTML markup functions
-
- def _doHTMLStart(self):
- # Start of html page
- self.out.write('<!DOCTYPE html PUBLIC \
-"-//W3C//DTD HTML 4.01//EN">\n')
- self.out.write('<html><head><title>%s</title>\n'%(self.title))
- self.out.write(self._getDocumentCreatedBy())
- self.out.write('<meta http-equiv="Content-Type" \
-content="text/html;charset=iso-8859-1">\n')
- # Get background
- self.out.write('</head><body bgcolor="%s">\n'%self._getPageColor())
- self._doPageHeader()
- self.out.write('<pre>')
-
- def _getHTMLStyles(self, toktype, toktext):
- # Get styles
- tags, color = self.colors.get(toktype, self.colors[NAME])[:2]#
- tagstart=[]
- tagend=[]
- # check for styles and set them if needed.
- if 'b' in tags:#Bold
- tagstart.append('<b>')
- tagend.append('</b>')
- if 'i' in tags:#Italics
- tagstart.append('<i>')
- tagend.append('</i>')
- if 'u' in tags:#Underline
- tagstart.append('<u>')
- tagend.append('</u>')
- # HTML tags should be paired like so : <b><i><u>Doh!</u></i></b>
- tagend.reverse()
- starttags="".join(tagstart)
- endtags="".join(tagend)
- return starttags,endtags,color
-
- def _sendHTMLText(self, toktype, toktext):
- numberlinks = self.numberlinks
-
- # If it is an error, set a red box around the bad tokens
- # older browsers should ignore it
- if toktype == ERRORTOKEN:
- style = ' style="border: solid 1.5pt #FF0000;"'
- else:
- style = ''
- # Get styles
- starttag, endtag, color = self._getHTMLStyles(toktype, toktext)
- # This is a hack to 'fix' multi-line strings.
- # Multi-line strings are treated as only one token
- # even though they can be several physical lines.
- # That makes it hard to spot the start of a line,
- # because at this level all we know about are tokens.
-
- if toktext.count(self.LINENUMHOLDER):
- # rip apart the string and separate it by line.
- # count lines and change all linenum token to line numbers.
- # embedded all the new font tags inside the current one.
- # Do this by ending the tag first then writing our new tags,
- # then starting another font tag exactly like the first one.
- if toktype == LINENUMBER:
- splittext = toktext.split(self.LINENUMHOLDER)
- else:
- splittext = toktext.split(self.LINENUMHOLDER+' ')
- store = []
- store.append(splittext.pop(0))
- lstarttag, lendtag, lcolor = self._getHTMLStyles(LINENUMBER, toktext)
- count = len(splittext)
- for item in splittext:
- num = self._getLineNumber()
- if numberlinks:
- numstrip = num.strip()
- content = '<a name="%s" href="#%s">%s</a>' \
- %(numstrip,numstrip,num)
- else:
- content = num
- if count <= 1:
- endtag,starttag = '',''
- linenumber = ''.join([endtag,'<font color=', lcolor, '>',
- lstarttag, content, lendtag, '</font>' ,starttag])
- store.append(linenumber+item)
- toktext = ''.join(store)
- # send text
- ## Output optimization
- # skip font tag if black text, but styles will still be sent. (b,u,i)
- if color !='#000000':
- startfont = '<font color="%s"%s>'%(color, style)
- endfont = '</font>'
- else:
- startfont, endfont = ('','')
- if toktype != LINENUMBER:
- self.out.write(''.join([startfont,starttag,
- toktext,endtag,endfont]))
- else:
- self.out.write(toktext)
- return
-
- def _doHTMLHeader(self):
- # Optional
- if self.header != '':
- self.out.write('%s\n'%self.header)
- else:
- color = self._getForeColor(NAME)
- self.out.write('<b><font color="%s"># %s \
- <br># %s</font></b><hr>\n'%
- (color, self.title, time.ctime()))
-
- def _doHTMLFooter(self):
- # Optional
- if self.footer != '':
- self.out.write('%s\n'%self.footer)
- else:
- color = self._getForeColor(NAME)
- self.out.write('<b><font color="%s"> \
- <hr># %s<br># %s</font></b>\n'%
- (color, self.title, time.ctime()))
-
- def _doHTMLEnd(self):
- # End of html page
- self.out.write('</pre>\n')
- # Write a little info at the bottom
- self._doPageFooter()
- self.out.write('</body></html>\n')
-
- #################################################### CSS markup functions
-
- def _getCSSStyle(self, key):
- # Get the tags and colors from the dictionary
- tags, forecolor, backcolor = self._getStyle(key)
- style=[]
- border = None
- bordercolor = None
- tags = tags.lower()
- if tags:
- # get the border color if specified
- # the border color will be appended to
- # the list after we define a border
- if '#' in tags:# border color
- start = tags.find('#')
- end = start + 7
- bordercolor = tags[start:end]
- tags.replace(bordercolor,'',1)
- # text styles
- if 'b' in tags:# Bold
- style.append('font-weight:bold;')
- else:
- style.append('font-weight:normal;')
- if 'i' in tags:# Italic
- style.append('font-style:italic;')
- if 'u' in tags:# Underline
- style.append('text-decoration:underline;')
- # border size
- if 'l' in tags:# thick border
- size='thick'
- elif 'm' in tags:# medium border
- size='medium'
- elif 't' in tags:# thin border
- size='thin'
- else:# default
- size='medium'
- # border styles
- if 'n' in tags:# inset border
- border='inset'
- elif 'o' in tags:# outset border
- border='outset'
- elif 'r' in tags:# ridge border
- border='ridge'
- elif 'g' in tags:# groove border
- border='groove'
- elif '=' in tags:# double border
- border='double'
- elif '.' in tags:# dotted border
- border='dotted'
- elif '-' in tags:# dashed border
- border='dashed'
- elif 's' in tags:# solid border
- border='solid'
- # border type check
- seperate_sides=0
- for side in ['<','>','^','v']:
- if side in tags:
- seperate_sides+=1
- # border box or seperate sides
- if seperate_sides==0 and border:
- style.append('border: %s %s;'%(border,size))
- else:
- if border == None:
- border = 'solid'
- if 'v' in tags:# bottom border
- style.append('border-bottom:%s %s;'%(border,size))
- if '<' in tags:# left border
- style.append('border-left:%s %s;'%(border,size))
- if '>' in tags:# right border
- style.append('border-right:%s %s;'%(border,size))
- if '^' in tags:# top border
- style.append('border-top:%s %s;'%(border,size))
- else:
- style.append('font-weight:normal;')# css inherited style fix
- # we have to define our borders before we set colors
- if bordercolor:
- style.append('border-color:%s;'%bordercolor)
- # text forecolor
- style.append('color:%s;'% forecolor)
- # text backcolor
- if backcolor:
- style.append('background-color:%s;'%backcolor)
- return (self._getMarkupClass(key),' '.join(style))
-
- def _sendCSSStyle(self, external=0):
- """ create external and internal style sheets"""
- styles = []
- external += self.external
- if not external:
- styles.append('<style type="text/css">\n<!--\n')
- # Get page background color and write styles ignore any we don't know
- styles.append('body { background:%s; }\n'%self._getPageColor())
- # write out the various css styles
- for key in MARKUPDICT:
- styles.append('.%s { %s }\n'%self._getCSSStyle(key))
- # If you want to style the pre tag you must modify the color dict.
- # Example:
- # lite[PY] = .py {border: solid thin #000000;background:#555555}\n'''
- styles.append(self.colors.get(PY, '.py { }\n'))
- # Extra css can be added here
- # add CSSHOOK to the color dict if you need it.
- # Example:
- #lite[CSSHOOK] = """.mytag { border: solid thin #000000; } \n
- # .myothertag { font-weight:bold; )\n"""
- styles.append(self.colors.get(CSSHOOK,''))
- if not self.external:
- styles.append('--></style>\n')
- return ''.join(styles)
-
- def _doCSSStart(self):
- # Start of css/html 4.01 page
- self.out.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN">\n')
- self.out.write('<html><head><title>%s</title>\n'%(self.title))
- self.out.write(self._getDocumentCreatedBy())
- self.out.write('<meta http-equiv="Content-Type" \
-content="text/html;charset=iso-8859-1">\n')
- self._doCSSStyleSheet()
- self.out.write('</head>\n<body>\n')
- # Write a little info at the top.
- self._doPageHeader()
- self.out.write(self.colors.get(CODESTART,'<pre class="py">\n'))
- return
-
- def _doCSSStyleSheet(self):
- if not self.external:
- # write an embedded style sheet
- self.out.write(self._sendCSSStyle())
- else:
- # write a link to an external style sheet
- self.out.write('<link rel="stylesheet" \
-href="pystyle.css" type="text/css">')
- return
-
- def _sendCSSText(self, toktype, toktext):
- # This is a hack to 'fix' multi-line strings.
- # Multi-line strings are treated as only one token
- # even though they can be several physical lines.
- # That makes it hard to spot the start of a line,
- # because at this level all we know about are tokens.
- markupclass = MARKUPDICT.get(toktype, MARKUPDICT[NAME])
- # if it is a LINENUMBER type then we can skip the rest
- if toktext == self.LINESTART and toktype == LINENUMBER:
- self.out.write('<span class="py_line">')
- return
- if toktext.count(self.LINENUMHOLDER):
- # rip apart the string and separate it by line
- # count lines and change all linenum token to line numbers
- # also convert linestart and lineend tokens
- # <linestart> <lnumstart> lnum <lnumend> text <lineend>
- #################################################
- newmarkup = MARKUPDICT.get(LINENUMBER, MARKUPDICT[NAME])
- lstartspan = '<span class="%s">'%(newmarkup)
- if toktype == LINENUMBER:
- splittext = toktext.split(self.LINENUMHOLDER)
- else:
- splittext = toktext.split(self.LINENUMHOLDER+' ')
- store = []
- # we have already seen the first linenumber token
- # so we can skip the first one
- store.append(splittext.pop(0))
- for item in splittext:
- num = self._getLineNumber()
- if self.numberlinks:
- numstrip = num.strip()
- content= '<a name="%s" href="#%s">%s</a>' \
- %(numstrip,numstrip,num)
- else:
- content = num
- linenumber= ''.join([lstartspan,content,'</span>'])
- store.append(linenumber+item)
- toktext = ''.join(store)
- if toktext.count(self.LINESTART):
- # wraps the textline in a line span
- # this adds a lot of kludges, is it really worth it?
- store = []
- parts = toktext.split(self.LINESTART+' ')
- # handle the first part differently
- # the whole token gets wraqpped in a span later on
- first = parts.pop(0)
- # place spans before the newline
- pos = first.rfind('\n')
- if pos != -1:
- first=first[:pos]+'</span></span>'+first[pos:]
- store.append(first)
- #process the rest of the string
- for item in parts:
- #handle line numbers if present
- if self.dolinenums:
- item = item.replace('</span>',
- '</span><span class="%s">'%(markupclass))
- else:
- item = '<span class="%s">%s'%(markupclass,item)
- # add endings for line and string tokens
- pos = item.rfind('\n')
- if pos != -1:
- item=item[:pos]+'</span></span>\n'
- store.append(item)
- # add start tags for lines
- toktext = '<span class="py_line">'.join(store)
- # Send text
- if toktype != LINENUMBER:
- if toktype == TEXT and self.textFlag == 'DIV':
- startspan = '<div class="%s">'%(markupclass)
- endspan = '</div>'
- elif toktype == TEXT and self.textFlag == 'RAW':
- startspan,endspan = ('','')
- else:
- startspan = '<span class="%s">'%(markupclass)
- endspan = '</span>'
- self.out.write(''.join([startspan, toktext, endspan]))
- else:
- self.out.write(toktext)
- return
-
- def _doCSSHeader(self):
- if self.header != '':
- self.out.write('%s\n'%self.header)
- else:
- name = MARKUPDICT.get(NAME)
- self.out.write('<div class="%s"># %s <br> \
-# %s</div><hr>\n'%(name, self.title, time.ctime()))
-
- def _doCSSFooter(self):
- # Optional
- if self.footer != '':
- self.out.write('%s\n'%self.footer)
- else:
- self.out.write('<hr><div class="%s"># %s <br> \
-# %s</div>\n'%(MARKUPDICT.get(NAME),self.title, time.ctime()))
-
- def _doCSSEnd(self):
- # End of css/html page
- self.out.write(self.colors.get(CODEEND,'</pre>\n'))
- # Write a little info at the bottom
- self._doPageFooter()
- self.out.write('</body></html>\n')
- return
-
- ################################################## XHTML markup functions
-
- def _doXHTMLStart(self):
- # XHTML is really just XML + HTML 4.01.
- # We only need to change the page headers,
- # and a few tags to get valid XHTML.
- # Start of xhtml page
- self.out.write('<?xml version="1.0"?>\n \
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"\n \
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">\n \
-<html xmlns="http://www.w3.org/1999/xhtml">\n')
- self.out.write('<head><title>%s</title>\n'%(self.title))
- self.out.write(self._getDocumentCreatedBy())
- self.out.write('<meta http-equiv="Content-Type" \
-content="text/html;charset=iso-8859-1"/>\n')
- self._doXHTMLStyleSheet()
- self.out.write('</head>\n<body>\n')
- # Write a little info at the top.
- self._doPageHeader()
- self.out.write(self.colors.get(CODESTART,'<pre class="py">\n'))
- return
-
- def _doXHTMLStyleSheet(self):
- if not self.external:
- # write an embedded style sheet
- self.out.write(self._sendCSSStyle())
- else:
- # write a link to an external style sheet
- self.out.write('<link rel="stylesheet" \
-href="pystyle.css" type="text/css"/>\n')
- return
-
- def _sendXHTMLText(self, toktype, toktext):
- self._sendCSSText(toktype, toktext)
-
- def _doXHTMLHeader(self):
- # Optional
- if self.header:
- self.out.write('%s\n'%self.header)
- else:
- name = MARKUPDICT.get(NAME)
- self.out.write('<div class="%s"># %s <br/> \
-# %s</div><hr/>\n '%(
- name, self.title, time.ctime()))
-
- def _doXHTMLFooter(self):
- # Optional
- if self.footer:
- self.out.write('%s\n'%self.footer)
- else:
- self.out.write('<hr/><div class="%s"># %s <br/> \
-# %s</div>\n'%(MARKUPDICT.get(NAME), self.title, time.ctime()))
-
- def _doXHTMLEnd(self):
- self._doCSSEnd()
-
-#############################################################################
-
-if __name__ == '__main__':
- cli()
-
-#############################################################################
-# PySourceColor.py
-# 2004, 2005 M.E.Farmer Jr.
-# Python license
+# -*- coding: Latin-1 -*- +""" +PySourceColor: color Python source code +""" + +""" + PySourceColor.py + +---------------------------------------------------------------------------- + + A python source to colorized html/css/xhtml converter. + Hacked by M.E.Farmer Jr. 2004, 2005 + Python license + +---------------------------------------------------------------------------- + + - HTML markup does not create w3c valid html, but it works on every + browser i've tried so far.(I.E.,Mozilla/Firefox,Opera,Konqueror,wxHTML). + - CSS markup is w3c validated html 4.01 strict, + but will not render correctly on all browsers. + - XHTML markup is w3c validated xhtml 1.0 strict, + like html 4.01, will not render correctly on all browsers. + +---------------------------------------------------------------------------- + +Features: + + -Three types of markup: + html (default) + css/html 4.01 strict + xhtml 1.0 strict + + -Can tokenize and colorize: + 12 types of strings + 2 comment types + numbers + operators + brackets + math operators + class / name + def / name + decorator / name + keywords + arguments class/def/decorator + linenumbers + names + text + + -Eight colorschemes built-in: + null + mono + lite (default) + dark + dark2 + idle + viewcvs + pythonwin + + -Header and footer + set to '' for builtin header / footer. + give path to a file containing the html + you want added as header or footer. + + -Arbitrary text and html + html markup converts all to raw (TEXT token) + #@# for raw -> send raw text. + #$# for span -> inline html and text. + #%# for div -> block level html and text. + + -Linenumbers + Supports all styles. New token is called LINENUMBER. + Defaults to NAME if not defined. + + Style options + + -ALL markups support these text styles: + b = bold + i = italic + u = underline + -CSS and XHTML has limited support for borders: + HTML markup functions will ignore these. + Optional: Border color in RGB hex + Defaults to the text forecolor. + #rrggbb = border color + Border size: + l = thick + m = medium + t = thin + Border type: + - = dashed + . = dotted + s = solid + d = double + g = groove + r = ridge + n = inset + o = outset + You can specify multiple sides, + they will all use the same style. + Optional: Default is full border. + v = bottom + < = left + > = right + ^ = top + NOTE: Specify the styles you want. + The markups will ignore unsupported styles + Also note not all browsers can show these options + + -All tokens default to NAME if not defined + so the only absolutely critical ones to define are: + NAME, ERRORTOKEN, PAGEBACKGROUND + +---------------------------------------------------------------------------- + +Example usage:: + + # import + import PySourceColor as psc + psc.convert('c:/Python22/PySourceColor.py', colors=psc.idle, show=1) + + # from module import * + from PySourceColor import * + convert('c:/Python22/Lib', colors=lite, markup="css", + header='#$#<b>This is a simpe heading</b><hr/>') + + # How to use a custom colorscheme, and most of the 'features' + from PySourceColor import * + new = { + ERRORTOKEN: ('bui','#FF8080',''), + DECORATOR_NAME: ('s','#AACBBC',''), + DECORATOR: ('n','#333333',''), + NAME: ('t.<v','#1133AA','#DDFF22'), + NUMBER: ('','#236676','#FF5555'), + OPERATOR: ('b','#454567','#BBBB11'), + MATH_OPERATOR: ('','#935623','#423afb'), + BRACKETS: ('b','#ac34bf','#6457a5'), + COMMENT: ('t-#0022FF','#545366','#AABBFF'), + DOUBLECOMMENT: ('<l#553455','#553455','#FF00FF'), + CLASS_NAME: ('m^v-','#000000','#FFFFFF'), + DEF_NAME: ('l=<v','#897845','#000022'), + KEYWORD: ('.b','#345345','#FFFF22'), + SINGLEQUOTE: ('mn','#223344','#AADDCC'), + SINGLEQUOTE_R: ('','#344522',''), + SINGLEQUOTE_U: ('','#234234',''), + DOUBLEQUOTE: ('m#0022FF','#334421',''), + DOUBLEQUOTE_R: ('','#345345',''), + DOUBLEQUOTE_U: ('','#678673',''), + TRIPLESINGLEQUOTE: ('tv','#FFFFFF','#000000'), + TRIPLESINGLEQUOTE_R: ('tbu','#443256','#DDFFDA'), + TRIPLESINGLEQUOTE_U: ('','#423454','#DDFFDA'), + TRIPLEDOUBLEQUOTE: ('li#236fd3b<>','#000000','#FFFFFF'), + TRIPLEDOUBLEQUOTE_R: ('tub','#000000','#FFFFFF'), + TRIPLEDOUBLEQUOTE_U: ('-', '#CCAABB','#FFFAFF'), + LINENUMBER: ('ib-','#ff66aa','#7733FF'),] + TEXT: ('','#546634',''), + PAGEBACKGROUND: '#FFFAAA', + } + if __name__ == '__main__': + import sys + convert(sys.argv[1], './xhtml.html', colors=new, markup='xhtml', show=1, + linenumbers=1) + convert(sys.argv[1], './html.html', colors=new, markup='html', show=1, + linenumbers=1) + +""" + +__all__ = ['ERRORTOKEN','DECORATOR_NAME', 'DECORATOR', 'ARGS', 'EXTRASPACE', + 'NAME', 'NUMBER', 'OPERATOR', 'COMMENT', 'MATH_OPERATOR', + 'DOUBLECOMMENT', 'CLASS_NAME', 'DEF_NAME', 'KEYWORD', 'BRACKETS', + 'SINGLEQUOTE','SINGLEQUOTE_R','SINGLEQUOTE_U','DOUBLEQUOTE', + 'DOUBLEQUOTE_R', 'DOUBLEQUOTE_U', 'TRIPLESINGLEQUOTE', 'TEXT', + 'TRIPLESINGLEQUOTE_R', 'TRIPLESINGLEQUOTE_U', 'TRIPLEDOUBLEQUOTE', + 'TRIPLEDOUBLEQUOTE_R', 'TRIPLEDOUBLEQUOTE_U', 'PAGEBACKGROUND', + 'LINENUMBER', 'CODESTART', 'CODEEND', 'PY', 'TOKEN_NAMES', 'CSSHOOK', + 'null', 'mono', 'lite', 'dark','dark2', 'pythonwin','idle', + 'viewcvs', 'Usage', 'cli', 'str2stdout', 'path2stdout', 'Parser', + 'str2file', 'str2html', 'str2css', 'str2markup', 'path2file', + 'path2html', 'convert', 'walkdir', 'defaultColors', 'showpage', + 'pageconvert','tagreplace', 'MARKUPDICT'] +__title__ = 'PySourceColor' +__version__ = "2.1a" +__date__ = '25 April 2005' +__author__ = "M.E.Farmer Jr." +__credits__ = '''This was originally based on a python recipe +submitted by Jürgen Hermann to ASPN. Now based on the voices in my head. +M.E.Farmer 2004, 2005 +Python license +''' +import os +import sys +import time +import glob +import getopt +import keyword +import token +import tokenize +import traceback +from six.moves import cStringIO as StringIO +# Do not edit +NAME = token.NAME +NUMBER = token.NUMBER +COMMENT = tokenize.COMMENT +OPERATOR = token.OP +ERRORTOKEN = token.ERRORTOKEN +ARGS = token.NT_OFFSET + 1 +DOUBLECOMMENT = token.NT_OFFSET + 2 +CLASS_NAME = token.NT_OFFSET + 3 +DEF_NAME = token.NT_OFFSET + 4 +KEYWORD = token.NT_OFFSET + 5 +SINGLEQUOTE = token.NT_OFFSET + 6 +SINGLEQUOTE_R = token.NT_OFFSET + 7 +SINGLEQUOTE_U = token.NT_OFFSET + 8 +DOUBLEQUOTE = token.NT_OFFSET + 9 +DOUBLEQUOTE_R = token.NT_OFFSET + 10 +DOUBLEQUOTE_U = token.NT_OFFSET + 11 +TRIPLESINGLEQUOTE = token.NT_OFFSET + 12 +TRIPLESINGLEQUOTE_R = token.NT_OFFSET + 13 +TRIPLESINGLEQUOTE_U = token.NT_OFFSET + 14 +TRIPLEDOUBLEQUOTE = token.NT_OFFSET + 15 +TRIPLEDOUBLEQUOTE_R = token.NT_OFFSET + 16 +TRIPLEDOUBLEQUOTE_U = token.NT_OFFSET + 17 +PAGEBACKGROUND = token.NT_OFFSET + 18 +DECORATOR = token.NT_OFFSET + 19 +DECORATOR_NAME = token.NT_OFFSET + 20 +BRACKETS = token.NT_OFFSET + 21 +MATH_OPERATOR = token.NT_OFFSET + 22 +LINENUMBER = token.NT_OFFSET + 23 +TEXT = token.NT_OFFSET + 24 +PY = token.NT_OFFSET + 25 +CODESTART = token.NT_OFFSET + 26 +CODEEND = token.NT_OFFSET + 27 +CSSHOOK = token.NT_OFFSET + 28 +EXTRASPACE = token.NT_OFFSET + 29 + +# markup classname lookup +MARKUPDICT = { + ERRORTOKEN: 'py_err', + DECORATOR_NAME: 'py_decn', + DECORATOR: 'py_dec', + ARGS: 'py_args', + NAME: 'py_name', + NUMBER: 'py_num', + OPERATOR: 'py_op', + COMMENT: 'py_com', + DOUBLECOMMENT: 'py_dcom', + CLASS_NAME: 'py_clsn', + DEF_NAME: 'py_defn', + KEYWORD: 'py_key', + SINGLEQUOTE: 'py_sq', + SINGLEQUOTE_R: 'py_sqr', + SINGLEQUOTE_U: 'py_squ', + DOUBLEQUOTE: 'py_dq', + DOUBLEQUOTE_R: 'py_dqr', + DOUBLEQUOTE_U: 'py_dqu', + TRIPLESINGLEQUOTE: 'py_tsq', + TRIPLESINGLEQUOTE_R: 'py_tsqr', + TRIPLESINGLEQUOTE_U: 'py_tsqu', + TRIPLEDOUBLEQUOTE: 'py_tdq', + TRIPLEDOUBLEQUOTE_R: 'py_tdqr', + TRIPLEDOUBLEQUOTE_U: 'py_tdqu', + BRACKETS: 'py_bra', + MATH_OPERATOR: 'py_mop', + LINENUMBER: 'py_lnum', + TEXT: 'py_text', + } +# might help users that want to create custom schemes +TOKEN_NAMES= { + ERRORTOKEN:'ERRORTOKEN', + DECORATOR_NAME:'DECORATOR_NAME', + DECORATOR:'DECORATOR', + ARGS:'ARGS', + NAME:'NAME', + NUMBER:'NUMBER', + OPERATOR:'OPERATOR', + COMMENT:'COMMENT', + DOUBLECOMMENT:'DOUBLECOMMENT', + CLASS_NAME:'CLASS_NAME', + DEF_NAME:'DEF_NAME', + KEYWORD:'KEYWORD', + SINGLEQUOTE:'SINGLEQUOTE', + SINGLEQUOTE_R:'SINGLEQUOTE_R', + SINGLEQUOTE_U:'SINGLEQUOTE_U', + DOUBLEQUOTE:'DOUBLEQUOTE', + DOUBLEQUOTE_R:'DOUBLEQUOTE_R', + DOUBLEQUOTE_U:'DOUBLEQUOTE_U', + TRIPLESINGLEQUOTE:'TRIPLESINGLEQUOTE', + TRIPLESINGLEQUOTE_R:'TRIPLESINGLEQUOTE_R', + TRIPLESINGLEQUOTE_U:'TRIPLESINGLEQUOTE_U', + TRIPLEDOUBLEQUOTE:'TRIPLEDOUBLEQUOTE', + TRIPLEDOUBLEQUOTE_R:'TRIPLEDOUBLEQUOTE_R', + TRIPLEDOUBLEQUOTE_U:'TRIPLEDOUBLEQUOTE_U', + BRACKETS:'BRACKETS', + MATH_OPERATOR:'MATH_OPERATOR', + LINENUMBER:'LINENUMBER', + TEXT:'TEXT', + PAGEBACKGROUND:'PAGEBACKGROUND', + } + +###################################################################### +# Edit colors and styles to taste +# Create your own scheme, just copy one below , rename and edit. +# Custom styles must at least define NAME, ERRORTOKEN, PAGEBACKGROUND, +# all missing elements will default to NAME. +# See module docstring for details on style attributes. +###################################################################### +# Copy null and use it as a starter colorscheme. +null = {# tokentype: ('tags border_color', 'textforecolor', 'textbackcolor') + ERRORTOKEN: ('','#000000',''),# Error token + DECORATOR_NAME: ('','#000000',''),# Decorator name + DECORATOR: ('','#000000',''),# @ symbol + ARGS: ('','#000000',''),# class,def,deco arguments + NAME: ('','#000000',''),# All other python text + NUMBER: ('','#000000',''),# 0->10 + OPERATOR: ('','#000000',''),# ':','<=',';',',','.','==', etc + MATH_OPERATOR: ('','#000000',''),# '+','-','=','','**',etc + BRACKETS: ('','#000000',''),# '[',']','(',')','{','}' + COMMENT: ('','#000000',''),# Single comment + DOUBLECOMMENT: ('','#000000',''),## Double comment + CLASS_NAME: ('','#000000',''),# Class name + DEF_NAME: ('','#000000',''),# Def name + KEYWORD: ('','#000000',''),# Python keywords + SINGLEQUOTE: ('','#000000',''),# 'SINGLEQUOTE' + SINGLEQUOTE_R: ('','#000000',''),# r'SINGLEQUOTE' + SINGLEQUOTE_U: ('','#000000',''),# u'SINGLEQUOTE' + DOUBLEQUOTE: ('','#000000',''),# "DOUBLEQUOTE" + DOUBLEQUOTE_R: ('','#000000',''),# r"DOUBLEQUOTE" + DOUBLEQUOTE_U: ('','#000000',''),# u"DOUBLEQUOTE" + TRIPLESINGLEQUOTE: ('','#000000',''),# '''TRIPLESINGLEQUOTE''' + TRIPLESINGLEQUOTE_R: ('','#000000',''),# r'''TRIPLESINGLEQUOTE''' + TRIPLESINGLEQUOTE_U: ('','#000000',''),# u'''TRIPLESINGLEQUOTE''' + TRIPLEDOUBLEQUOTE: ('','#000000',''),# """TRIPLEDOUBLEQUOTE""" + TRIPLEDOUBLEQUOTE_R: ('','#000000',''),# r"""TRIPLEDOUBLEQUOTE""" + TRIPLEDOUBLEQUOTE_U: ('','#000000',''),# u"""TRIPLEDOUBLEQUOTE""" + TEXT: ('','#000000',''),# non python text + LINENUMBER: ('>ti#555555','#000000',''),# Linenumbers + PAGEBACKGROUND: '#FFFFFF'# set the page background + } + +mono = { + ERRORTOKEN: ('s#FF0000','#FF8080',''), + DECORATOR_NAME: ('bu','#000000',''), + DECORATOR: ('b','#000000',''), + ARGS: ('b','#555555',''), + NAME: ('','#000000',''), + NUMBER: ('b','#000000',''), + OPERATOR: ('b','#000000',''), + MATH_OPERATOR: ('b','#000000',''), + BRACKETS: ('b','#000000',''), + COMMENT: ('i','#999999',''), + DOUBLECOMMENT: ('b','#999999',''), + CLASS_NAME: ('bu','#000000',''), + DEF_NAME: ('b','#000000',''), + KEYWORD: ('b','#000000',''), + SINGLEQUOTE: ('','#000000',''), + SINGLEQUOTE_R: ('','#000000',''), + SINGLEQUOTE_U: ('','#000000',''), + DOUBLEQUOTE: ('','#000000',''), + DOUBLEQUOTE_R: ('','#000000',''), + DOUBLEQUOTE_U: ('','#000000',''), + TRIPLESINGLEQUOTE: ('','#000000',''), + TRIPLESINGLEQUOTE_R: ('','#000000',''), + TRIPLESINGLEQUOTE_U: ('','#000000',''), + TRIPLEDOUBLEQUOTE: ('i','#000000',''), + TRIPLEDOUBLEQUOTE_R: ('i','#000000',''), + TRIPLEDOUBLEQUOTE_U: ('i','#000000',''), + TEXT: ('','#000000',''), + LINENUMBER: ('>ti#555555','#000000',''), + PAGEBACKGROUND: '#FFFFFF' + } + +dark = { + ERRORTOKEN: ('s#FF0000','#FF8080',''), + DECORATOR_NAME: ('b','#FFBBAA',''), + DECORATOR: ('b','#CC5511',''), + ARGS: ('b','#DDDDFF',''), + NAME: ('','#DDDDDD',''), + NUMBER: ('','#FF0000',''), + OPERATOR: ('b','#FAF785',''), + MATH_OPERATOR: ('b','#FAF785',''), + BRACKETS: ('b','#FAF785',''), + COMMENT: ('','#45FCA0',''), + DOUBLECOMMENT: ('i','#A7C7A9',''), + CLASS_NAME: ('b','#B666FD',''), + DEF_NAME: ('b','#EBAE5C',''), + KEYWORD: ('b','#8680FF',''), + SINGLEQUOTE: ('','#F8BAFE',''), + SINGLEQUOTE_R: ('','#F8BAFE',''), + SINGLEQUOTE_U: ('','#F8BAFE',''), + DOUBLEQUOTE: ('','#FF80C0',''), + DOUBLEQUOTE_R: ('','#FF80C0',''), + DOUBLEQUOTE_U: ('','#FF80C0',''), + TRIPLESINGLEQUOTE: ('','#FF9595',''), + TRIPLESINGLEQUOTE_R: ('','#FF9595',''), + TRIPLESINGLEQUOTE_U: ('','#FF9595',''), + TRIPLEDOUBLEQUOTE: ('','#B3FFFF',''), + TRIPLEDOUBLEQUOTE_R: ('','#B3FFFF',''), + TRIPLEDOUBLEQUOTE_U: ('','#B3FFFF',''), + TEXT: ('','#FFFFFF',''), + LINENUMBER: ('>mi#555555','#bbccbb','#333333'), + PAGEBACKGROUND: '#000000' + } + +dark2 = { + ERRORTOKEN: ('','#FF0000',''), + DECORATOR_NAME: ('b','#FFBBAA',''), + DECORATOR: ('b','#CC5511',''), + ARGS: ('b','#DDDDDD',''), + NAME: ('','#C0C0C0',''), + NUMBER: ('b','#00FF00',''), + OPERATOR: ('b','#FF090F',''), + MATH_OPERATOR: ('b','#EE7020',''), + BRACKETS: ('b','#FFB90F',''), + COMMENT: ('i','#D0D000','#522000'),#'#88AA88','#11111F'), + DOUBLECOMMENT: ('i','#D0D000','#522000'),#'#77BB77','#11111F'), + CLASS_NAME: ('b','#DD4080',''), + DEF_NAME: ('b','#FF8040',''), + KEYWORD: ('b','#4726d1',''), + SINGLEQUOTE: ('','#8080C0',''), + SINGLEQUOTE_R: ('','#8080C0',''), + SINGLEQUOTE_U: ('','#8080C0',''), + DOUBLEQUOTE: ('','#ADB9F1',''), + DOUBLEQUOTE_R: ('','#ADB9F1',''), + DOUBLEQUOTE_U: ('','#ADB9F1',''), + TRIPLESINGLEQUOTE: ('','#00C1C1',''),#A050C0 + TRIPLESINGLEQUOTE_R: ('','#00C1C1',''),#A050C0 + TRIPLESINGLEQUOTE_U: ('','#00C1C1',''),#A050C0 + TRIPLEDOUBLEQUOTE: ('','#33E3E3',''),#B090E0 + TRIPLEDOUBLEQUOTE_R: ('','#33E3E3',''),#B090E0 + TRIPLEDOUBLEQUOTE_U: ('','#33E3E3',''),#B090E0 + TEXT: ('','#C0C0C0',''), + LINENUMBER: ('>mi#555555','#bbccbb','#333333'), + PAGEBACKGROUND: '#000000' + } + +lite = { + ERRORTOKEN: ('s#FF0000','#FF8080',''), + DECORATOR_NAME: ('b','#BB4422',''), + DECORATOR: ('b','#3333AF',''), + ARGS: ('b','#000000',''), + NAME: ('','#333333',''), + NUMBER: ('b','#DD2200',''), + OPERATOR: ('b','#000000',''), + MATH_OPERATOR: ('b','#000000',''), + BRACKETS: ('b','#000000',''), + COMMENT: ('','#007F00',''), + DOUBLECOMMENT: ('','#608060',''), + CLASS_NAME: ('b','#0000DF',''), + DEF_NAME: ('b','#9C7A00',''),#f09030 + KEYWORD: ('b','#0000AF',''), + SINGLEQUOTE: ('','#600080',''), + SINGLEQUOTE_R: ('','#600080',''), + SINGLEQUOTE_U: ('','#600080',''), + DOUBLEQUOTE: ('','#A0008A',''), + DOUBLEQUOTE_R: ('','#A0008A',''), + DOUBLEQUOTE_U: ('','#A0008A',''), + TRIPLESINGLEQUOTE: ('','#337799',''), + TRIPLESINGLEQUOTE_R: ('','#337799',''), + TRIPLESINGLEQUOTE_U: ('','#337799',''), + TRIPLEDOUBLEQUOTE: ('','#1166AA',''), + TRIPLEDOUBLEQUOTE_R: ('','#1166AA',''), + TRIPLEDOUBLEQUOTE_U: ('','#1166AA',''), + TEXT: ('','#000000',''), + LINENUMBER: ('>ti#555555','#000000',''), + PAGEBACKGROUND: '#FFFFFF' + } + +idle = { + ERRORTOKEN: ('s#FF0000','#FF8080',''), + DECORATOR_NAME: ('','#900090',''), + DECORATOR: ('','#FF7700',''), + NAME: ('','#000000',''), + NUMBER: ('','#000000',''), + OPERATOR: ('','#000000',''), + MATH_OPERATOR: ('','#000000',''), + BRACKETS: ('','#000000',''), + COMMENT: ('','#DD0000',''), + DOUBLECOMMENT: ('','#DD0000',''), + CLASS_NAME: ('','#0000FF',''), + DEF_NAME: ('','#0000FF',''), + KEYWORD: ('','#FF7700',''), + SINGLEQUOTE: ('','#00AA00',''), + SINGLEQUOTE_R: ('','#00AA00',''), + SINGLEQUOTE_U: ('','#00AA00',''), + DOUBLEQUOTE: ('','#00AA00',''), + DOUBLEQUOTE_R: ('','#00AA00',''), + DOUBLEQUOTE_U: ('','#00AA00',''), + TRIPLESINGLEQUOTE: ('','#00AA00',''), + TRIPLESINGLEQUOTE_R: ('','#00AA00',''), + TRIPLESINGLEQUOTE_U: ('','#00AA00',''), + TRIPLEDOUBLEQUOTE: ('','#00AA00',''), + TRIPLEDOUBLEQUOTE_R: ('','#00AA00',''), + TRIPLEDOUBLEQUOTE_U: ('','#00AA00',''), + TEXT: ('','#000000',''), + LINENUMBER: ('>ti#555555','#000000',''), + PAGEBACKGROUND: '#FFFFFF' + } + +pythonwin = { + ERRORTOKEN: ('s#FF0000','#FF8080',''), + DECORATOR_NAME: ('b','#DD0080',''), + DECORATOR: ('b','#000080',''), + ARGS: ('','#000000',''), + NAME: ('','#303030',''), + NUMBER: ('','#008080',''), + OPERATOR: ('','#000000',''), + MATH_OPERATOR: ('','#000000',''), + BRACKETS: ('','#000000',''), + COMMENT: ('','#007F00',''), + DOUBLECOMMENT: ('','#7F7F7F',''), + CLASS_NAME: ('b','#0000FF',''), + DEF_NAME: ('b','#007F7F',''), + KEYWORD: ('b','#000080',''), + SINGLEQUOTE: ('','#808000',''), + SINGLEQUOTE_R: ('','#808000',''), + SINGLEQUOTE_U: ('','#808000',''), + DOUBLEQUOTE: ('','#808000',''), + DOUBLEQUOTE_R: ('','#808000',''), + DOUBLEQUOTE_U: ('','#808000',''), + TRIPLESINGLEQUOTE: ('','#808000',''), + TRIPLESINGLEQUOTE_R: ('','#808000',''), + TRIPLESINGLEQUOTE_U: ('','#808000',''), + TRIPLEDOUBLEQUOTE: ('','#808000',''), + TRIPLEDOUBLEQUOTE_R: ('','#808000',''), + TRIPLEDOUBLEQUOTE_U: ('','#808000',''), + TEXT: ('','#303030',''), + LINENUMBER: ('>ti#555555','#000000',''), + PAGEBACKGROUND: '#FFFFFF' + } + +viewcvs = { + ERRORTOKEN: ('s#FF0000','#FF8080',''), + DECORATOR_NAME: ('','#000000',''), + DECORATOR: ('','#000000',''), + ARGS: ('','#000000',''), + NAME: ('','#000000',''), + NUMBER: ('','#000000',''), + OPERATOR: ('','#000000',''), + MATH_OPERATOR: ('','#000000',''), + BRACKETS: ('','#000000',''), + COMMENT: ('i','#b22222',''), + DOUBLECOMMENT: ('i','#b22222',''), + CLASS_NAME: ('','#000000',''), + DEF_NAME: ('b','#0000ff',''), + KEYWORD: ('b','#a020f0',''), + SINGLEQUOTE: ('b','#bc8f8f',''), + SINGLEQUOTE_R: ('b','#bc8f8f',''), + SINGLEQUOTE_U: ('b','#bc8f8f',''), + DOUBLEQUOTE: ('b','#bc8f8f',''), + DOUBLEQUOTE_R: ('b','#bc8f8f',''), + DOUBLEQUOTE_U: ('b','#bc8f8f',''), + TRIPLESINGLEQUOTE: ('b','#bc8f8f',''), + TRIPLESINGLEQUOTE_R: ('b','#bc8f8f',''), + TRIPLESINGLEQUOTE_U: ('b','#bc8f8f',''), + TRIPLEDOUBLEQUOTE: ('b','#bc8f8f',''), + TRIPLEDOUBLEQUOTE_R: ('b','#bc8f8f',''), + TRIPLEDOUBLEQUOTE_U: ('b','#bc8f8f',''), + TEXT: ('','#000000',''), + LINENUMBER: ('>ti#555555','#000000',''), + PAGEBACKGROUND: '#FFFFFF' + } + +defaultColors = lite + +def Usage(): + doc = """ + ----------------------------------------------------------------------------- + PySourceColor.py ver: %s + ----------------------------------------------------------------------------- + Module summary: + This module is designed to colorize python source code. + Input--->python source + Output-->colorized (html, html4.01/css, xhtml1.0) + Standalone: + This module will work from the command line with options. + This module will work with redirected stdio. + Imported: + This module can be imported and used directly in your code. + ----------------------------------------------------------------------------- + Command line options: + -h, --help + Optional-> Display this help message. + -t, --test + Optional-> Will ignore all others flags but --profile + test all schemes and markup combinations + -p, --profile + Optional-> Works only with --test or -t + runs profile.py and makes the test work in quiet mode. + -i, --in, --input + Optional-> If you give input on stdin. + Use any of these for the current dir (.,cwd) + Input can be file or dir. + Input from stdin use one of the following (-,stdin) + If stdin is used as input stdout is output unless specified. + -o, --out, --output + Optional-> output dir for the colorized source. + default: output dir is the input dir. + To output html to stdout use one of the following (-,stdout) + Stdout can be used without stdin if you give a file as input. + -c, --color + Optional-> null, mono, dark, dark2, lite, idle, pythonwin, viewcvs + default: dark + -s, --show + Optional-> Show page after creation. + default: no show + -m, --markup + Optional-> html, css, xhtml + css, xhtml also support external stylesheets (-e,--external) + default: HTML + -e, --external + Optional-> use with css, xhtml + Writes an style sheet instead of embedding it in the page + saves it as pystyle.css in the same directory. + html markup will silently ignore this flag. + -H, --header + Opional-> add a page header to the top of the output + -H + Builtin header (name,date,hrule) + --header + You must specify a filename. + The header file must be valid html + and must handle its own font colors. + ex. --header c:/tmp/header.txt + -F, --footer + Opional-> add a page footer to the bottom of the output + -F + Builtin footer (hrule,name,date) + --footer + You must specify a filename. + The footer file must be valid html + and must handle its own font colors. + ex. --footer c:/tmp/footer.txt + -l, --linenumbers + Optional-> default is no linenumbers + Adds line numbers to the start of each line in the code. + --convertpage + Given a webpage that has code embedded in tags it will + convert embedded code to colorized html. + (see pageconvert for details) + ----------------------------------------------------------------------------- + Option usage: + # Test and show pages + python PySourceColor.py -t -s + # Test and only show profile results + python PySourceColor.py -t -p + # Colorize all .py,.pyw files in cwdir you can also use: (.,cwd) + python PySourceColor.py -i . + # Using long options w/ = + python PySourceColor.py --in=c:/myDir/my.py --color=lite --show + # Using short options w/out = + python PySourceColor.py -i c:/myDir/ -c idle -m css -e + # Using any mix + python PySourceColor.py --in . -o=c:/myDir --show + # Place a custom header on your files + python PySourceColor.py -i . -o c:/tmp -m xhtml --header c:/header.txt + ----------------------------------------------------------------------------- + Stdio usage: + # Stdio using no options + python PySourceColor.py < c:/MyFile.py > c:/tmp/MyFile.html + # Using stdin alone automatically uses stdout for output: (stdin,-) + python PySourceColor.py -i- < c:/MyFile.py > c:/tmp/myfile.html + # Stdout can also be written to directly from a file instead of stdin + python PySourceColor.py -i c:/MyFile.py -m css -o- > c:/tmp/myfile.html + # Stdin can be used as input , but output can still be specified + python PySourceColor.py -i- -o c:/pydoc.py.html -s < c:/Python22/my.py + _____________________________________________________________________________ + """ + print(doc % (__version__)) + sys.exit(1) + +###################################################### Command line interface + +def cli(): + """Handle command line args and redirections""" + try: + # try to get command line args + opts, args = getopt.getopt(sys.argv[1:], + "hseqtplHFi:o:c:m:h:f:",["help", "show", "quiet", + "test", "external", "linenumbers", "convertpage", "profile", + "input=", "output=", "color=", "markup=","header=", "footer="]) + except getopt.GetoptError: + # on error print help information and exit: + Usage() + # init some names + input = None + output = None + colorscheme = None + markup = 'html' + header = None + footer = None + linenumbers = 0 + show = 0 + quiet = 0 + test = 0 + profile = 0 + convertpage = 0 + form = None + # if we have args then process them + for o, a in opts: + if o in ["-h", "--help"]: + Usage() + sys.exit() + if o in ["-o", "--output", "--out"]: + output = a + if o in ["-i", "--input", "--in"]: + input = a + if input in [".", "cwd"]: + input = os.getcwd() + if o in ["-s", "--show"]: + show = 1 + if o in ["-q", "--quiet"]: + quiet = 1 + if o in ["-t", "--test"]: + test = 1 + if o in ["--convertpage"]: + convertpage = 1 + if o in ["-p", "--profile"]: + profile = 1 + if o in ["-e", "--external"]: + form = 'external' + if o in ["-m", "--markup"]: + markup = str(a) + if o in ["-l", "--linenumbers"]: + linenumbers = 1 + if o in ["--header"]: + header = str(a) + elif o == "-H": + header = '' + if o in ["--footer"]: + footer = str(a) + elif o == "-F": + footer = '' + if o in ["-c", "--color"]: + try: + colorscheme = globals().get(a.lower()) + except: + traceback.print_exc() + Usage() + if test: + if profile: + import profile + profile.run('_test(show=%s, quiet=%s)'%(show,quiet)) + else: + # Parse this script in every possible colorscheme and markup + _test(show,quiet) + elif input in [None, "-", "stdin"] or output in ["-", "stdout"]: + # determine if we are going to use stdio + if input not in [None, "-", "stdin"]: + if os.path.isfile(input) : + path2stdout(input, colors=colorscheme, markup=markup, + linenumbers=linenumbers, header=header, + footer=footer, form=form) + else: + raise PathError('File does not exists!') + else: + try: + if sys.stdin.isatty(): + raise InputError('Please check input!') + else: + if output in [None,"-","stdout"]: + str2stdout(sys.stdin.read(), colors=colorscheme, + markup=markup, header=header, + footer=footer, linenumbers=linenumbers, + form=form) + else: + str2file(sys.stdin.read(), outfile=output, show=show, + markup=markup, header=header, footer=footer, + linenumbers=linenumbers, form=form) + except: + traceback.print_exc() + Usage() + else: + if os.path.exists(input): + if convertpage: + # if there was at least an input given we can proceed + pageconvert(input, out=output, colors=colorscheme, + show=show, markup=markup,linenumbers=linenumbers) + else: + # if there was at least an input given we can proceed + convert(source=input, outdir=output, colors=colorscheme, + show=show, markup=markup, quiet=quiet, header=header, + footer=footer, linenumbers=linenumbers, form=form) + else: + raise PathError('File does not exists!') + Usage() + +######################################################### Simple markup tests + +def _test(show=0, quiet=0): + """Test the parser and most of the functions. + + There are 19 test total(eight colorschemes in three diffrent markups, + and a str2file test. Most functions are tested by this. + """ + fi = sys.argv[0] + if not fi.endswith('.exe'):# Do not test if frozen as an archive + # this is a collection of test, most things are covered. + path2file(fi, '/tmp/null.html', null, show=show, quiet=quiet) + path2file(fi, '/tmp/null_css.html', null, show=show, + markup='css', quiet=quiet) + path2file(fi, '/tmp/mono.html', mono, show=show, quiet=quiet) + path2file(fi, '/tmp/mono_css.html', mono, show=show, + markup='css', quiet=quiet) + path2file(fi, '/tmp/lite.html', lite, show=show, quiet=quiet) + path2file(fi, '/tmp/lite_css.html', lite, show=show, + markup='css', quiet=quiet, header='', footer='', + linenumbers=1) + path2file(fi, '/tmp/lite_xhtml.html', lite, show=show, + markup='xhtml', quiet=quiet) + path2file(fi, '/tmp/dark.html', dark, show=show, quiet=quiet) + path2file(fi, '/tmp/dark_css.html', dark, show=show, + markup='css', quiet=quiet, linenumbers=1) + path2file(fi, '/tmp/dark2.html', dark2, show=show, quiet=quiet) + path2file(fi, '/tmp/dark2_css.html', dark2, show=show, + markup='css', quiet=quiet) + path2file(fi, '/tmp/dark2_xhtml.html', dark2, show=show, + markup='xhtml', quiet=quiet, header='', footer='', + linenumbers=1, form='external') + path2file(fi, '/tmp/idle.html', idle, show=show, quiet=quiet) + path2file(fi, '/tmp/idle_css.html', idle, show=show, + markup='css', quiet=quiet) + path2file(fi, '/tmp/viewcvs.html', viewcvs, show=show, + quiet=quiet, linenumbers=1) + path2file(fi, '/tmp/viewcvs_css.html', viewcvs, show=show, + markup='css', linenumbers=1, quiet=quiet) + path2file(fi, '/tmp/pythonwin.html', pythonwin, show=show, + quiet=quiet) + path2file(fi, '/tmp/pythonwin_css.html', pythonwin, show=show, + markup='css', quiet=quiet) + teststr=r'''"""This is a test of decorators and other things""" +# This should be line 421... +@whatever(arg,arg2) +@A @B(arghh) @C +def LlamaSaysNi(arg='Ni!',arg2="RALPH"): + """This docstring is deeply disturbed by all the llama references""" + print('%s The Wonder Llama says %s'% (arg2,arg)) +# So I was like duh!, and he was like ya know?!, +# and so we were both like huh...wtf!? RTFM!! LOL!!;) +@staticmethod## Double comments are KewL. +def LlamasRLumpy(): + """This docstring is too sexy to be here. + """ + u""" +============================= +A Møøse once bit my sister... +============================= + """ + ## Relax, this won't hurt a bit, just a simple, painless procedure, + ## hold still while I get the anesthetizing hammer. + m = {'three':'1','won':'2','too':'3'} + o = r'fishy\fishy\fishy/fish\oh/where/is\my/little\..' + python = uR""" + No realli! She was Karving her initials øn the møøse with the sharpened end + of an interspace tøøthbrush given her by Svenge - her brother-in-law -an Oslo + dentist and star of many Norwegian møvies: "The Høt Hands of an Oslo + Dentist", "Fillings of Passion", "The Huge Mølars of Horst Nordfink"...""" + RU"""142 MEXICAN WHOOPING LLAMAS"""#<-Can you fit 142 llamas in a red box? + n = u' HERMSGERVØRDENBRØTBØRDA ' + """ YUTTE """ + t = """SAMALLNIATNUOMNAIRODAUCE"""+"DENIARTYLLAICEPS04" + ## We apologise for the fault in the + ## comments. Those responsible have been + ## sacked. + y = '14 NORTH CHILEAN GUANACOS \ +(CLOSELY RELATED TO THE LLAMA)' + rules = [0,1,2,3,4,5] + print y''' + htmlPath = os.path.abspath('/tmp/strtest_lines.html') + str2file(teststr, htmlPath, colors=dark, markup='xhtml', + linenumbers=420, show=show) + _printinfo(" wrote %s" % htmlPath, quiet) + htmlPath = os.path.abspath('/tmp/strtest_nolines.html') + str2file(teststr, htmlPath, colors=dark, markup='xhtml', + show=show) + _printinfo(" wrote %s" % htmlPath, quiet) + else: + Usage() + return + +# emacs wants this: ' + +####################################################### User funtctions + +def str2stdout(sourcestring, colors=None, title='', markup='html', + header=None, footer=None, + linenumbers=0, form=None): + """Converts a code(string) to colorized HTML. Writes to stdout. + + form='code',or'snip' (for "<pre>yourcode</pre>" only) + colors=null,mono,lite,dark,dark2,idle,or pythonwin + """ + Parser(sourcestring, colors=colors, title=title, markup=markup, + header=header, footer=footer, + linenumbers=linenumbers).format(form) + +def path2stdout(sourcepath, title='', colors=None, markup='html', + header=None, footer=None, + linenumbers=0, form=None): + """Converts code(file) to colorized HTML. Writes to stdout. + + form='code',or'snip' (for "<pre>yourcode</pre>" only) + colors=null,mono,lite,dark,dark2,idle,or pythonwin + """ + sourcestring = open(sourcepath).read() + Parser(sourcestring, colors=colors, title=sourcepath, + markup=markup, header=header, footer=footer, + linenumbers=linenumbers).format(form) + +def str2html(sourcestring, colors=None, title='', + markup='html', header=None, footer=None, + linenumbers=0, form=None): + """Converts a code(string) to colorized HTML. Returns an HTML string. + + form='code',or'snip' (for "<pre>yourcode</pre>" only) + colors=null,mono,lite,dark,dark2,idle,or pythonwin + """ + stringIO = StringIO.StringIO() + Parser(sourcestring, colors=colors, title=title, out=stringIO, + markup=markup, header=header, footer=footer, + linenumbers=linenumbers).format(form) + stringIO.seek(0) + return stringIO.read() + +def str2css(sourcestring, colors=None, title='', + markup='css', header=None, footer=None, + linenumbers=0, form=None): + """Converts a code string to colorized CSS/HTML. Returns CSS/HTML string + + If form != None then this will return (stylesheet_str, code_str) + colors=null,mono,lite,dark,dark2,idle,or pythonwin + """ + if markup.lower() not in ['css' ,'xhtml']: + markup = 'css' + stringIO = StringIO.StringIO() + parse = Parser(sourcestring, colors=colors, title=title, + out=stringIO, markup=markup, + header=header, footer=footer, + linenumbers=linenumbers) + parse.format(form) + stringIO.seek(0) + if form != None: + return parse._sendCSSStyle(external=1), stringIO.read() + else: + return None, stringIO.read() + +def str2markup(sourcestring, colors=None, title = '', + markup='xhtml', header=None, footer=None, + linenumbers=0, form=None): + """ Convert code strings into ([stylesheet or None], colorized string) """ + if markup.lower() == 'html': + return None, str2html(sourcestring, colors=colors, title=title, + header=header, footer=footer, markup=markup, + linenumbers=linenumbers, form=form) + else: + return str2css(sourcestring, colors=colors, title=title, + header=header, footer=footer, markup=markup, + linenumbers=linenumbers, form=form) + +def str2file(sourcestring, outfile, colors=None, title='', + markup='html', header=None, footer=None, + linenumbers=0, show=0, dosheet=1, form=None): + """Converts a code string to a file. + + makes no attempt at correcting bad pathnames + """ + css , html = str2markup(sourcestring, colors=colors, title='', + markup=markup, header=header, footer=footer, + linenumbers=linenumbers, form=form) + # write html + f = open(outfile,'wt') + f.writelines(html) + f.close() + #write css + if css != None and dosheet: + dir = os.path.dirname(outfile) + outcss = os.path.join(dir,'pystyle.css') + f = open(outcss,'wt') + f.writelines(css) + f.close() + if show: + showpage(outfile) + +def path2html(sourcepath, colors=None, markup='html', + header=None, footer=None, + linenumbers=0, form=None): + """Converts code(file) to colorized HTML. Returns an HTML string. + + form='code',or'snip' (for "<pre>yourcode</pre>" only) + colors=null,mono,lite,dark,dark2,idle,or pythonwin + """ + stringIO = StringIO.StringIO() + sourcestring = open(sourcepath).read() + Parser(sourcestring, colors, title=sourcepath, out=stringIO, + markup=markup, header=header, footer=footer, + linenumbers=linenumbers).format(form) + stringIO.seek(0) + return stringIO.read() + +def convert(source, outdir=None, colors=None, + show=0, markup='html', quiet=0, + header=None, footer=None, linenumbers=0, form=None): + """Takes a file or dir as input and places the html in the outdir. + + If outdir is none it defaults to the input dir + """ + count=0 + # If it is a filename then path2file + if not os.path.isdir(source): + if os.path.isfile(source): + count+=1 + path2file(source, outdir, colors, show, markup, + quiet, form, header, footer, linenumbers, count) + else: + raise PathError('File does not exist!') + # If we pass in a dir we need to walkdir for files. + # Then we need to colorize them with path2file + else: + fileList = walkdir(source) + if fileList != None: + # make sure outdir is a dir + if outdir != None: + if os.path.splitext(outdir)[1] != '': + outdir = os.path.split(outdir)[0] + for item in fileList: + count+=1 + path2file(item, outdir, colors, show, markup, + quiet, form, header, footer, linenumbers, count) + _printinfo('Completed colorizing %s files.'%str(count), quiet) + else: + _printinfo("No files to convert in dir.", quiet) + +def path2file(sourcePath, out=None, colors=None, show=0, + markup='html', quiet=0, form=None, + header=None, footer=None, linenumbers=0, count=1): + """ Converts python source to html file""" + # If no outdir is given we use the sourcePath + if out == None:#this is a guess + htmlPath = sourcePath + '.html' + else: + # If we do give an out_dir, and it does + # not exist , it will be created. + if os.path.splitext(out)[1] == '': + if not os.path.isdir(out): + os.makedirs(out) + sourceName = os.path.basename(sourcePath) + htmlPath = os.path.join(out,sourceName)+'.html' + # If we do give an out_name, and its dir does + # not exist , it will be created. + else: + outdir = os.path.split(out)[0] + if not os.path.isdir(outdir): + os.makedirs(outdir) + htmlPath = out + htmlPath = os.path.abspath(htmlPath) + # Open the text and do the parsing. + source = open(sourcePath).read() + parse = Parser(source, colors, sourcePath, open(htmlPath, 'wt'), + markup, header, footer, linenumbers) + parse.format(form) + _printinfo(" wrote %s" % htmlPath, quiet) + # html markup will ignore the external flag, but + # we need to stop the blank file from being written. + if form == 'external' and count == 1 and markup != 'html': + cssSheet = parse._sendCSSStyle(external=1) + cssPath = os.path.join(os.path.dirname(htmlPath),'pystyle.css') + css = open(cssPath, 'wt') + css.write(cssSheet) + css.close() + _printinfo(" wrote %s" % cssPath, quiet) + if show: + # load HTML page into the default web browser. + showpage(htmlPath) + return htmlPath + +def tagreplace(sourcestr, colors=lite, markup='xhtml', + linenumbers=0, dosheet=1, tagstart='<PY>'.lower(), + tagend='</PY>'.lower(), stylesheet='pystyle.css'): + """This is a helper function for pageconvert. Returns css, page. + """ + if markup.lower() != 'html': + link = '<link rel="stylesheet" href="%s" type="text/css"/></head>' + css = link%stylesheet + if sourcestr.find(css) == -1: + sourcestr = sourcestr.replace('</head>', css, 1) + starttags = sourcestr.count(tagstart) + endtags = sourcestr.count(tagend) + if starttags: + if starttags == endtags: + for _ in range(starttags): + datastart = sourcestr.find(tagstart) + dataend = sourcestr.find(tagend) + data = sourcestr[datastart+len(tagstart):dataend] + data = unescape(data) + css , data = str2markup(data, colors=colors, + linenumbers=linenumbers, markup=markup, form='embed') + start = sourcestr[:datastart] + end = sourcestr[dataend+len(tagend):] + sourcestr = ''.join([start,data,end]) + else: + raise InputError('Tag mismatch!\nCheck %s,%s tags'%tagstart,tagend) + if not dosheet: + css = None + return css, sourcestr + +def pageconvert(path, out=None, colors=lite, markup='xhtml', linenumbers=0, + dosheet=1, tagstart='<PY>'.lower(), tagend='</PY>'.lower(), + stylesheet='pystyle', show=1, returnstr=0): + """This function can colorize Python source + + that is written in a webpage enclosed in tags. + """ + if out == None: + out = os.path.dirname(path) + infile = open(path, 'r').read() + css,page = tagreplace(sourcestr=infile,colors=colors, + markup=markup, linenumbers=linenumbers, dosheet=dosheet, + tagstart=tagstart, tagend=tagend, stylesheet=stylesheet) + if not returnstr: + newpath = os.path.abspath(os.path.join( + out,'tmp', os.path.basename(path))) + if not os.path.exists(newpath): + try: + os.makedirs(os.path.dirname(newpath)) + except: + pass#traceback.print_exc() + #Usage() + y = open(newpath, 'w') + y.write(page) + y.close() + if css: + csspath = os.path.abspath(os.path.join( + out,'tmp','%s.css'%stylesheet)) + x = open(csspath,'w') + x.write(css) + x.close() + if show: + try: + os.startfile(newpath) + except: + traceback.print_exc() + return newpath + else: + return css, page + +##################################################################### helpers + +def walkdir(dir): + """Return a list of .py and .pyw files from a given directory. + + This function can be written as a generator Python 2.3, or a genexp + in Python 2.4. But 2.2 and 2.1 would be left out.... + """ + # Get a list of files that match *.py* + GLOB_PATTERN = os.path.join(dir, "*.[p][y]*") + pathlist = glob.glob(GLOB_PATTERN) + # Now filter out all but py and pyw + filterlist = [x for x in pathlist + if x.endswith('.py') + or x.endswith('.pyw')] + if filterlist != []: + # if we have a list send it + return filterlist + else: + return None + +def showpage(path): + """Helper function to open webpages""" + try: + import webbrowser + webbrowser.open_new(os.path.abspath(path)) + except: + traceback.print_exc() + +def _printinfo(message, quiet): + """Helper to print messages""" + if not quiet: + print(message) + +def escape(text): + """escape text for html. similar to cgi.escape""" + text = text.replace("&", "&") + text = text.replace("<", "<") + text = text.replace(">", ">") + return text + +def unescape(text): + """unsecape escaped text""" + text = text.replace(""", '"') + text = text.replace(">", ">") + text = text.replace("<", "<") + text = text.replace("&", "&") + return text + +########################################################### Custom Exceptions + +class PySourceColorError(Exception): + # Base for custom errors + def __init__(self, msg=''): + self._msg = msg + Exception.__init__(self, msg) + def __repr__(self): + return self._msg + __str__ = __repr__ + +class PathError(PySourceColorError): + def __init__(self, msg): + PySourceColorError.__init__(self, + 'Path error! : %s'% msg) + +class InputError(PySourceColorError): + def __init__(self, msg): + PySourceColorError.__init__(self, + 'Input error! : %s'% msg) + +########################################################## Python code parser + +class Parser(object): + + """MoinMoin python parser heavily chopped :)""" + + def __init__(self, raw, colors=None, title='', out=sys.stdout, + markup='html', header=None, footer=None, linenumbers=0): + """Store the source text & set some flags""" + if colors == None: + colors = defaultColors + self.raw = raw.expandtabs().rstrip() + self.title = os.path.basename(title) + self.out = out + self.line = '' + self.lasttext = '' + self.argFlag = 0 + self.classFlag = 0 + self.defFlag = 0 + self.decoratorFlag = 0 + self.external = 0 + self.markup = markup.upper() + self.colors = colors + self.header = header + self.footer = footer + self.doArgs = 1 # overrides the new tokens + self.doNames = 1 # overrides the new tokens + self.doMathOps = 1 # overrides the new tokens + self.doBrackets = 1 # overrides the new tokens + self.doURL = 1 # override url conversion + self.LINENUMHOLDER = "___line___".upper() + self.LINESTART = "___start___".upper() + self.skip = 0 + # add space left side of code for padding.Override in color dict. + self.extraspace = self.colors.get(EXTRASPACE, '') + # Linenumbers less then zero also have numberlinks + self.dolinenums = self.linenum = abs(linenumbers) + if linenumbers < 0: + self.numberlinks = 1 + else: + self.numberlinks = 0 + + def format(self, form=None): + """Parse and send the colorized source""" + if form in ('snip','code'): + self.addEnds = 0 + elif form == 'embed': + self.addEnds = 0 + self.external = 1 + else: + if form == 'external': + self.external = 1 + self.addEnds = 1 + + # Store line offsets in self.lines + self.lines = [0, 0] + pos = 0 + + # Add linenumbers + if self.dolinenums: + start=self.LINENUMHOLDER+' '+self.extraspace + else: + start=''+self.extraspace + newlines = [] + lines = self.raw.splitlines(0) + for l in lines: + # span and div escape for customizing and embedding raw text + if (l.startswith('#$#') + or l.startswith('#%#') + or l.startswith('#@#')): + newlines.append(l) + else: + # kludge for line spans in css,xhtml + if self.markup in ['XHTML','CSS']: + newlines.append(self.LINESTART+' '+start+l) + else: + newlines.append(start+l) + self.raw = "\n".join(newlines)+'\n'# plus an extra newline at the end + + # Gather lines + while 1: + pos = self.raw.find('\n', pos) + 1 + if not pos: break + self.lines.append(pos) + self.lines.append(len(self.raw)) + + # Wrap text in a filelike object + self.pos = 0 + text = StringIO.StringIO(self.raw) + + # Markup start + if self.addEnds: + self._doPageStart() + else: + self._doSnippetStart() + + ## Tokenize calls the __call__ + ## function for each token till done. + # Parse the source and write out the results. + try: + tokenize.tokenize(text.readline, self) + except tokenize.TokenError as ex: + msg = ex[0] + line = ex[1][0] + self.out.write("<h3>ERROR: %s</h3>%s\n"% + (msg, self.raw[self.lines[line]:])) + #traceback.print_exc() + + # Markup end + if self.addEnds: + self._doPageEnd() + else: + self._doSnippetEnd() + + def __call__(self, toktype, toktext, srow_col, erow_col, line): + """Token handler. Order is important do not rearrange.""" + self.line = line + srow, scol = srow_col + erow, ecol = erow_col + # Calculate new positions + oldpos = self.pos + newpos = self.lines[srow] + scol + self.pos = newpos + len(toktext) + # Handle newlines + if toktype in (token.NEWLINE, tokenize.NL): + self.decoratorFlag = self.argFlag = 0 + # kludge for line spans in css,xhtml + if self.markup in ['XHTML','CSS']: + self.out.write('</span>') + self.out.write('\n') + return + + # Send the original whitespace, and tokenize backslashes if present. + # Tokenizer.py just sends continued line backslashes with whitespace. + # This is a hack to tokenize continued line slashes as operators. + # Should continued line backslashes be treated as operators + # or some other token? + + if newpos > oldpos: + if self.raw[oldpos:newpos].isspace(): + # consume a single space after linestarts and linenumbers + # had to have them so tokenizer could seperate them. + # multiline strings are handled by do_Text functions + if self.lasttext != self.LINESTART \ + and self.lasttext != self.LINENUMHOLDER: + self.out.write(self.raw[oldpos:newpos]) + else: + self.out.write(self.raw[oldpos+1:newpos]) + else: + slash = self.raw[oldpos:newpos].find('\\')+oldpos + self.out.write(self.raw[oldpos:slash]) + getattr(self, '_send%sText'%(self.markup))(OPERATOR, '\\') + self.linenum+=1 + # kludge for line spans in css,xhtml + if self.markup in ['XHTML','CSS']: + self.out.write('</span>') + self.out.write(self.raw[slash+1:newpos]) + + # Skip indenting tokens + if toktype in (token.INDENT, token.DEDENT): + self.pos = newpos + return + + # Look for operators + if token.LPAR <= toktype and toktype <= token.OP: + # Trap decorators py2.4 > + if toktext == '@': + toktype = DECORATOR + # Set a flag if this was the decorator start so + # the decorator name and arguments can be identified + self.decoratorFlag = self.argFlag = 1 + else: + if self.doArgs: + # Find the start for arguments + if toktext == '(' and self.argFlag: + self.argFlag = 2 + # Find the end for arguments + elif toktext == ':': + self.argFlag = 0 + ## Seperate the diffrent operator types + # Brackets + if self.doBrackets and toktext in ['[',']','(',')','{','}']: + toktype = BRACKETS + # Math operators + elif self.doMathOps and toktext in ['*=','**=','-=','+=','|=', + '%=','>>=','<<=','=','^=', + '/=', '+','-','**','*','/','%']: + toktype = MATH_OPERATOR + # Operator + else: + toktype = OPERATOR + # example how flags should work. + # def fun(arg=argvalue,arg2=argvalue2): + # 0 1 2 A 1 N 2 A 1 N 0 + if toktext == "=" and self.argFlag == 2: + self.argFlag = 1 + elif toktext == "," and self.argFlag == 1: + self.argFlag = 2 + # Look for keywords + elif toktype == NAME and keyword.iskeyword(toktext): + toktype = KEYWORD + # Set a flag if this was the class / def start so + # the class / def name and arguments can be identified + if toktext in ['class', 'def']: + if toktext =='class' and \ + not line[:line.find('class')].endswith('.'): + self.classFlag = self.argFlag = 1 + elif toktext == 'def' and \ + not line[:line.find('def')].endswith('.'): + self.defFlag = self.argFlag = 1 + else: + # must have used a keyword as a name i.e. self.class + toktype = ERRORTOKEN + + # Look for class, def, decorator name + elif (self.classFlag or self.defFlag or self.decoratorFlag) \ + and self.doNames: + if self.classFlag: + self.classFlag = 0 + toktype = CLASS_NAME + elif self.defFlag: + self.defFlag = 0 + toktype = DEF_NAME + elif self.decoratorFlag: + self.decoratorFlag = 0 + toktype = DECORATOR_NAME + + # Look for strings + # Order of evaluation is important do not change. + elif toktype == token.STRING: + text = toktext.lower() + # TRIPLE DOUBLE QUOTE's + if (text[:3] == '"""'): + toktype = TRIPLEDOUBLEQUOTE + elif (text[:4] == 'r"""'): + toktype = TRIPLEDOUBLEQUOTE_R + elif (text[:4] == 'u"""' or + text[:5] == 'ur"""'): + toktype = TRIPLEDOUBLEQUOTE_U + # DOUBLE QUOTE's + elif (text[:1] == '"'): + toktype = DOUBLEQUOTE + elif (text[:2] == 'r"'): + toktype = DOUBLEQUOTE_R + elif (text[:2] == 'u"' or + text[:3] == 'ur"'): + toktype = DOUBLEQUOTE_U + # TRIPLE SINGLE QUOTE's + elif (text[:3] == "'''"): + toktype = TRIPLESINGLEQUOTE + elif (text[:4] == "r'''"): + toktype = TRIPLESINGLEQUOTE_R + elif (text[:4] == "u'''" or + text[:5] == "ur'''"): + toktype = TRIPLESINGLEQUOTE_U + # SINGLE QUOTE's + elif (text[:1] == "'"): + toktype = SINGLEQUOTE + elif (text[:2] == "r'"): + toktype = SINGLEQUOTE_R + elif (text[:2] == "u'" or + text[:3] == "ur'"): + toktype = SINGLEQUOTE_U + + # test for invalid string declaration + if self.lasttext.lower() == 'ru': + toktype = ERRORTOKEN + + # Look for comments + elif toktype == COMMENT: + if toktext[:2] == "##": + toktype = DOUBLECOMMENT + elif toktext[:3] == '#$#': + toktype = TEXT + self.textFlag = 'SPAN' + toktext = toktext[3:] + elif toktext[:3] == '#%#': + toktype = TEXT + self.textFlag = 'DIV' + toktext = toktext[3:] + elif toktext[:3] == '#@#': + toktype = TEXT + self.textFlag = 'RAW' + toktext = toktext[3:] + if self.doURL: + # this is a 'fake helper function' + # url(URI,Alias_name) or url(URI) + url_pos = toktext.find('url(') + if url_pos != -1: + before = toktext[:url_pos] + url = toktext[url_pos+4:] + splitpoint = url.find(',') + endpoint = url.find(')') + after = url[endpoint+1:] + url = url[:endpoint] + if splitpoint != -1: + urlparts = url.split(',',1) + toktext = '%s<a href="%s">%s</a>%s'%( + before,urlparts[0],urlparts[1].lstrip(),after) + else: + toktext = '%s<a href="%s">%s</a>%s'%(before,url,url,after) + + # Seperate errors from decorators + elif toktype == ERRORTOKEN: + # Bug fix for < py2.4 + # space between decorators + if self.argFlag and toktext.isspace(): + #toktype = NAME + self.out.write(toktext) + return + # Bug fix for py2.2 linenumbers with decorators + elif toktext.isspace(): + # What if we have a decorator after a >>> or ... + #p = line.find('@') + #if p >= 0 and not line[:p].isspace(): + #self.out.write(toktext) + #return + if self.skip: + self.skip=0 + return + else: + self.out.write(toktext) + return + # trap decorators < py2.4 + elif toktext == '@': + toktype = DECORATOR + # Set a flag if this was the decorator start so + # the decorator name and arguments can be identified + self.decoratorFlag = self.argFlag = 1 + + # Seperate args from names + elif (self.argFlag == 2 and + toktype == NAME and + toktext != 'None' and + self.doArgs): + toktype = ARGS + + # Look for line numbers + # The conversion code for them is in the send_text functions. + if toktext in [self.LINENUMHOLDER,self.LINESTART]: + toktype = LINENUMBER + # if we don't have linenumbers set flag + # to skip the trailing space from linestart + if toktext == self.LINESTART and not self.dolinenums \ + or toktext == self.LINENUMHOLDER: + self.skip=1 + + + # Skip blank token that made it thru + ## bugfix for the last empty tag. + if toktext == '': + return + + # Last token text history + self.lasttext = toktext + + # escape all but the urls in the comments + if toktype in (DOUBLECOMMENT, COMMENT): + if toktext.find('<a href=') == -1: + toktext = escape(toktext) + else: + pass + elif toktype == TEXT: + pass + else: + toktext = escape(toktext) + + # Send text for any markup + getattr(self, '_send%sText'%(self.markup))(toktype, toktext) + return + + ################################################################# Helpers + + def _doSnippetStart(self): + if self.markup == 'HTML': + # Start of html snippet + self.out.write('<pre>\n') + else: + # Start of css/xhtml snippet + self.out.write(self.colors.get(CODESTART,'<pre class="py">\n')) + + def _doSnippetEnd(self): + # End of html snippet + self.out.write(self.colors.get(CODEEND,'</pre>\n')) + + ######################################################## markup selectors + + def _getFile(self, filepath): + try: + _file = open(filepath,'r') + content = _file.read() + _file.close() + except: + traceback.print_exc() + content = '' + return content + + def _doPageStart(self): + getattr(self, '_do%sStart'%(self.markup))() + + def _doPageHeader(self): + if self.header != None: + if self.header.find('#$#') != -1 or \ + self.header.find('#$#') != -1 or \ + self.header.find('#%#') != -1: + self.out.write(self.header[3:]) + else: + if self.header != '': + self.header = self._getFile(self.header) + getattr(self, '_do%sHeader'%(self.markup))() + + def _doPageFooter(self): + if self.footer != None: + if self.footer.find('#$#') != -1 or \ + self.footer.find('#@#') != -1 or \ + self.footer.find('#%#') != -1: + self.out.write(self.footer[3:]) + else: + if self.footer != '': + self.footer = self._getFile(self.footer) + getattr(self, '_do%sFooter'%(self.markup))() + + def _doPageEnd(self): + getattr(self, '_do%sEnd'%(self.markup))() + + ################################################### color/style retrieval + ## Some of these are not used anymore but are kept for documentation + + def _getLineNumber(self): + num = self.linenum + self.linenum+=1 + return str(num).rjust(5)+" " + + def _getTags(self, key): + # style tags + return self.colors.get(key, self.colors[NAME])[0] + + def _getForeColor(self, key): + # get text foreground color, if not set to black + color = self.colors.get(key, self.colors[NAME])[1] + if color[:1] != '#': + color = '#000000' + return color + + def _getBackColor(self, key): + # get text background color + return self.colors.get(key, self.colors[NAME])[2] + + def _getPageColor(self): + # get page background color + return self.colors.get(PAGEBACKGROUND, '#FFFFFF') + + def _getStyle(self, key): + # get the token style from the color dictionary + return self.colors.get(key, self.colors[NAME]) + + def _getMarkupClass(self, key): + # get the markup class name from the markup dictionary + return MARKUPDICT.get(key, MARKUPDICT[NAME]) + + def _getDocumentCreatedBy(self): + return '<!--This document created by %s ver.%s on: %s-->\n'%( + __title__,__version__,time.ctime()) + + ################################################### HTML markup functions + + def _doHTMLStart(self): + # Start of html page + self.out.write('<!DOCTYPE html PUBLIC \ +"-//W3C//DTD HTML 4.01//EN">\n') + self.out.write('<html><head><title>%s</title>\n'%(self.title)) + self.out.write(self._getDocumentCreatedBy()) + self.out.write('<meta http-equiv="Content-Type" \ +content="text/html;charset=iso-8859-1">\n') + # Get background + self.out.write('</head><body bgcolor="%s">\n'%self._getPageColor()) + self._doPageHeader() + self.out.write('<pre>') + + def _getHTMLStyles(self, toktype, toktext): + # Get styles + tags, color = self.colors.get(toktype, self.colors[NAME])[:2]# + tagstart=[] + tagend=[] + # check for styles and set them if needed. + if 'b' in tags:#Bold + tagstart.append('<b>') + tagend.append('</b>') + if 'i' in tags:#Italics + tagstart.append('<i>') + tagend.append('</i>') + if 'u' in tags:#Underline + tagstart.append('<u>') + tagend.append('</u>') + # HTML tags should be paired like so : <b><i><u>Doh!</u></i></b> + tagend.reverse() + starttags="".join(tagstart) + endtags="".join(tagend) + return starttags,endtags,color + + def _sendHTMLText(self, toktype, toktext): + numberlinks = self.numberlinks + + # If it is an error, set a red box around the bad tokens + # older browsers should ignore it + if toktype == ERRORTOKEN: + style = ' style="border: solid 1.5pt #FF0000;"' + else: + style = '' + # Get styles + starttag, endtag, color = self._getHTMLStyles(toktype, toktext) + # This is a hack to 'fix' multi-line strings. + # Multi-line strings are treated as only one token + # even though they can be several physical lines. + # That makes it hard to spot the start of a line, + # because at this level all we know about are tokens. + + if toktext.count(self.LINENUMHOLDER): + # rip apart the string and separate it by line. + # count lines and change all linenum token to line numbers. + # embedded all the new font tags inside the current one. + # Do this by ending the tag first then writing our new tags, + # then starting another font tag exactly like the first one. + if toktype == LINENUMBER: + splittext = toktext.split(self.LINENUMHOLDER) + else: + splittext = toktext.split(self.LINENUMHOLDER+' ') + store = [] + store.append(splittext.pop(0)) + lstarttag, lendtag, lcolor = self._getHTMLStyles(LINENUMBER, toktext) + count = len(splittext) + for item in splittext: + num = self._getLineNumber() + if numberlinks: + numstrip = num.strip() + content = '<a name="%s" href="#%s">%s</a>' \ + %(numstrip,numstrip,num) + else: + content = num + if count <= 1: + endtag,starttag = '','' + linenumber = ''.join([endtag,'<font color=', lcolor, '>', + lstarttag, content, lendtag, '</font>' ,starttag]) + store.append(linenumber+item) + toktext = ''.join(store) + # send text + ## Output optimization + # skip font tag if black text, but styles will still be sent. (b,u,i) + if color !='#000000': + startfont = '<font color="%s"%s>'%(color, style) + endfont = '</font>' + else: + startfont, endfont = ('','') + if toktype != LINENUMBER: + self.out.write(''.join([startfont,starttag, + toktext,endtag,endfont])) + else: + self.out.write(toktext) + return + + def _doHTMLHeader(self): + # Optional + if self.header != '': + self.out.write('%s\n'%self.header) + else: + color = self._getForeColor(NAME) + self.out.write('<b><font color="%s"># %s \ + <br># %s</font></b><hr>\n'% + (color, self.title, time.ctime())) + + def _doHTMLFooter(self): + # Optional + if self.footer != '': + self.out.write('%s\n'%self.footer) + else: + color = self._getForeColor(NAME) + self.out.write('<b><font color="%s"> \ + <hr># %s<br># %s</font></b>\n'% + (color, self.title, time.ctime())) + + def _doHTMLEnd(self): + # End of html page + self.out.write('</pre>\n') + # Write a little info at the bottom + self._doPageFooter() + self.out.write('</body></html>\n') + + #################################################### CSS markup functions + + def _getCSSStyle(self, key): + # Get the tags and colors from the dictionary + tags, forecolor, backcolor = self._getStyle(key) + style=[] + border = None + bordercolor = None + tags = tags.lower() + if tags: + # get the border color if specified + # the border color will be appended to + # the list after we define a border + if '#' in tags:# border color + start = tags.find('#') + end = start + 7 + bordercolor = tags[start:end] + tags.replace(bordercolor,'',1) + # text styles + if 'b' in tags:# Bold + style.append('font-weight:bold;') + else: + style.append('font-weight:normal;') + if 'i' in tags:# Italic + style.append('font-style:italic;') + if 'u' in tags:# Underline + style.append('text-decoration:underline;') + # border size + if 'l' in tags:# thick border + size='thick' + elif 'm' in tags:# medium border + size='medium' + elif 't' in tags:# thin border + size='thin' + else:# default + size='medium' + # border styles + if 'n' in tags:# inset border + border='inset' + elif 'o' in tags:# outset border + border='outset' + elif 'r' in tags:# ridge border + border='ridge' + elif 'g' in tags:# groove border + border='groove' + elif '=' in tags:# double border + border='double' + elif '.' in tags:# dotted border + border='dotted' + elif '-' in tags:# dashed border + border='dashed' + elif 's' in tags:# solid border + border='solid' + # border type check + seperate_sides=0 + for side in ['<','>','^','v']: + if side in tags: + seperate_sides+=1 + # border box or seperate sides + if seperate_sides==0 and border: + style.append('border: %s %s;'%(border,size)) + else: + if border == None: + border = 'solid' + if 'v' in tags:# bottom border + style.append('border-bottom:%s %s;'%(border,size)) + if '<' in tags:# left border + style.append('border-left:%s %s;'%(border,size)) + if '>' in tags:# right border + style.append('border-right:%s %s;'%(border,size)) + if '^' in tags:# top border + style.append('border-top:%s %s;'%(border,size)) + else: + style.append('font-weight:normal;')# css inherited style fix + # we have to define our borders before we set colors + if bordercolor: + style.append('border-color:%s;'%bordercolor) + # text forecolor + style.append('color:%s;'% forecolor) + # text backcolor + if backcolor: + style.append('background-color:%s;'%backcolor) + return (self._getMarkupClass(key),' '.join(style)) + + def _sendCSSStyle(self, external=0): + """ create external and internal style sheets""" + styles = [] + external += self.external + if not external: + styles.append('<style type="text/css">\n<!--\n') + # Get page background color and write styles ignore any we don't know + styles.append('body { background:%s; }\n'%self._getPageColor()) + # write out the various css styles + for key in MARKUPDICT: + styles.append('.%s { %s }\n'%self._getCSSStyle(key)) + # If you want to style the pre tag you must modify the color dict. + # Example: + # lite[PY] = .py {border: solid thin #000000;background:#555555}\n''' + styles.append(self.colors.get(PY, '.py { }\n')) + # Extra css can be added here + # add CSSHOOK to the color dict if you need it. + # Example: + #lite[CSSHOOK] = """.mytag { border: solid thin #000000; } \n + # .myothertag { font-weight:bold; )\n""" + styles.append(self.colors.get(CSSHOOK,'')) + if not self.external: + styles.append('--></style>\n') + return ''.join(styles) + + def _doCSSStart(self): + # Start of css/html 4.01 page + self.out.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN">\n') + self.out.write('<html><head><title>%s</title>\n'%(self.title)) + self.out.write(self._getDocumentCreatedBy()) + self.out.write('<meta http-equiv="Content-Type" \ +content="text/html;charset=iso-8859-1">\n') + self._doCSSStyleSheet() + self.out.write('</head>\n<body>\n') + # Write a little info at the top. + self._doPageHeader() + self.out.write(self.colors.get(CODESTART,'<pre class="py">\n')) + return + + def _doCSSStyleSheet(self): + if not self.external: + # write an embedded style sheet + self.out.write(self._sendCSSStyle()) + else: + # write a link to an external style sheet + self.out.write('<link rel="stylesheet" \ +href="pystyle.css" type="text/css">') + return + + def _sendCSSText(self, toktype, toktext): + # This is a hack to 'fix' multi-line strings. + # Multi-line strings are treated as only one token + # even though they can be several physical lines. + # That makes it hard to spot the start of a line, + # because at this level all we know about are tokens. + markupclass = MARKUPDICT.get(toktype, MARKUPDICT[NAME]) + # if it is a LINENUMBER type then we can skip the rest + if toktext == self.LINESTART and toktype == LINENUMBER: + self.out.write('<span class="py_line">') + return + if toktext.count(self.LINENUMHOLDER): + # rip apart the string and separate it by line + # count lines and change all linenum token to line numbers + # also convert linestart and lineend tokens + # <linestart> <lnumstart> lnum <lnumend> text <lineend> + ################################################# + newmarkup = MARKUPDICT.get(LINENUMBER, MARKUPDICT[NAME]) + lstartspan = '<span class="%s">'%(newmarkup) + if toktype == LINENUMBER: + splittext = toktext.split(self.LINENUMHOLDER) + else: + splittext = toktext.split(self.LINENUMHOLDER+' ') + store = [] + # we have already seen the first linenumber token + # so we can skip the first one + store.append(splittext.pop(0)) + for item in splittext: + num = self._getLineNumber() + if self.numberlinks: + numstrip = num.strip() + content= '<a name="%s" href="#%s">%s</a>' \ + %(numstrip,numstrip,num) + else: + content = num + linenumber= ''.join([lstartspan,content,'</span>']) + store.append(linenumber+item) + toktext = ''.join(store) + if toktext.count(self.LINESTART): + # wraps the textline in a line span + # this adds a lot of kludges, is it really worth it? + store = [] + parts = toktext.split(self.LINESTART+' ') + # handle the first part differently + # the whole token gets wraqpped in a span later on + first = parts.pop(0) + # place spans before the newline + pos = first.rfind('\n') + if pos != -1: + first=first[:pos]+'</span></span>'+first[pos:] + store.append(first) + #process the rest of the string + for item in parts: + #handle line numbers if present + if self.dolinenums: + item = item.replace('</span>', + '</span><span class="%s">'%(markupclass)) + else: + item = '<span class="%s">%s'%(markupclass,item) + # add endings for line and string tokens + pos = item.rfind('\n') + if pos != -1: + item=item[:pos]+'</span></span>\n' + store.append(item) + # add start tags for lines + toktext = '<span class="py_line">'.join(store) + # Send text + if toktype != LINENUMBER: + if toktype == TEXT and self.textFlag == 'DIV': + startspan = '<div class="%s">'%(markupclass) + endspan = '</div>' + elif toktype == TEXT and self.textFlag == 'RAW': + startspan,endspan = ('','') + else: + startspan = '<span class="%s">'%(markupclass) + endspan = '</span>' + self.out.write(''.join([startspan, toktext, endspan])) + else: + self.out.write(toktext) + return + + def _doCSSHeader(self): + if self.header != '': + self.out.write('%s\n'%self.header) + else: + name = MARKUPDICT.get(NAME) + self.out.write('<div class="%s"># %s <br> \ +# %s</div><hr>\n'%(name, self.title, time.ctime())) + + def _doCSSFooter(self): + # Optional + if self.footer != '': + self.out.write('%s\n'%self.footer) + else: + self.out.write('<hr><div class="%s"># %s <br> \ +# %s</div>\n'%(MARKUPDICT.get(NAME),self.title, time.ctime())) + + def _doCSSEnd(self): + # End of css/html page + self.out.write(self.colors.get(CODEEND,'</pre>\n')) + # Write a little info at the bottom + self._doPageFooter() + self.out.write('</body></html>\n') + return + + ################################################## XHTML markup functions + + def _doXHTMLStart(self): + # XHTML is really just XML + HTML 4.01. + # We only need to change the page headers, + # and a few tags to get valid XHTML. + # Start of xhtml page + self.out.write('<?xml version="1.0"?>\n \ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"\n \ + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">\n \ +<html xmlns="http://www.w3.org/1999/xhtml">\n') + self.out.write('<head><title>%s</title>\n'%(self.title)) + self.out.write(self._getDocumentCreatedBy()) + self.out.write('<meta http-equiv="Content-Type" \ +content="text/html;charset=iso-8859-1"/>\n') + self._doXHTMLStyleSheet() + self.out.write('</head>\n<body>\n') + # Write a little info at the top. + self._doPageHeader() + self.out.write(self.colors.get(CODESTART,'<pre class="py">\n')) + return + + def _doXHTMLStyleSheet(self): + if not self.external: + # write an embedded style sheet + self.out.write(self._sendCSSStyle()) + else: + # write a link to an external style sheet + self.out.write('<link rel="stylesheet" \ +href="pystyle.css" type="text/css"/>\n') + return + + def _sendXHTMLText(self, toktype, toktext): + self._sendCSSText(toktype, toktext) + + def _doXHTMLHeader(self): + # Optional + if self.header: + self.out.write('%s\n'%self.header) + else: + name = MARKUPDICT.get(NAME) + self.out.write('<div class="%s"># %s <br/> \ +# %s</div><hr/>\n '%( + name, self.title, time.ctime())) + + def _doXHTMLFooter(self): + # Optional + if self.footer: + self.out.write('%s\n'%self.footer) + else: + self.out.write('<hr/><div class="%s"># %s <br/> \ +# %s</div>\n'%(MARKUPDICT.get(NAME), self.title, time.ctime())) + + def _doXHTMLEnd(self): + self._doCSSEnd() + +############################################################################# + +if __name__ == '__main__': + cli() + +############################################################################# +# PySourceColor.py +# 2004, 2005 M.E.Farmer Jr. +# Python license diff --git a/paste/util/UserDict24.py b/paste/util/UserDict24.py deleted file mode 100644 index e5b64f5..0000000 --- a/paste/util/UserDict24.py +++ /dev/null @@ -1,167 +0,0 @@ -"""A more or less complete user-defined wrapper around dictionary objects.""" - -class UserDict: - def __init__(self, dict=None, **kwargs): - self.data = {} - if dict is not None: - if not hasattr(dict,'keys'): - dict = type({})(dict) # make mapping from a sequence - self.update(dict) - if len(kwargs): - self.update(kwargs) - def __repr__(self): return repr(self.data) - def __cmp__(self, dict): - if isinstance(dict, UserDict): - return cmp(self.data, dict.data) - else: - return cmp(self.data, dict) - def __len__(self): return len(self.data) - def __getitem__(self, key): return self.data[key] - def __setitem__(self, key, item): self.data[key] = item - def __delitem__(self, key): del self.data[key] - def clear(self): self.data.clear() - def copy(self): - if self.__class__ is UserDict: - return UserDict(self.data) - import copy - data = self.data - try: - self.data = {} - c = copy.copy(self) - finally: - self.data = data - c.update(self) - return c - def keys(self): return self.data.keys() - def items(self): return self.data.items() - def iteritems(self): return self.data.iteritems() - def iterkeys(self): return self.data.iterkeys() - def itervalues(self): return self.data.itervalues() - def values(self): return self.data.values() - def has_key(self, key): return self.data.has_key(key) - def update(self, dict): - if isinstance(dict, UserDict): - self.data.update(dict.data) - elif isinstance(dict, type(self.data)): - self.data.update(dict) - else: - for k, v in dict.items(): - self[k] = v - def get(self, key, failobj=None): - if not self.has_key(key): - return failobj - return self[key] - def setdefault(self, key, failobj=None): - if not self.has_key(key): - self[key] = failobj - return self[key] - def pop(self, key, *args): - return self.data.pop(key, *args) - def popitem(self): - return self.data.popitem() - def __contains__(self, key): - return key in self.data - def fromkeys(cls, iterable, value=None): - d = cls() - for key in iterable: - d[key] = value - return d - fromkeys = classmethod(fromkeys) - -class IterableUserDict(UserDict): - def __iter__(self): - return iter(self.data) - -class DictMixin: - # Mixin defining all dictionary methods for classes that already have - # a minimum dictionary interface including getitem, setitem, delitem, - # and keys. Without knowledge of the subclass constructor, the mixin - # does not define __init__() or copy(). In addition to the four base - # methods, progressively more efficiency comes with defining - # __contains__(), __iter__(), and iteritems(). - - # second level definitions support higher levels - def __iter__(self): - for k in self.keys(): - yield k - def has_key(self, key): - try: - value = self[key] - except KeyError: - return False - return True - def __contains__(self, key): - return self.has_key(key) - - # third level takes advantage of second level definitions - def iteritems(self): - for k in self: - yield (k, self[k]) - def iterkeys(self): - return self.__iter__() - - # fourth level uses definitions from lower levels - def itervalues(self): - for _, v in self.iteritems(): - yield v - def values(self): - return [v for _, v in self.iteritems()] - def items(self): - return list(self.iteritems()) - def clear(self): - for key in self.keys(): - del self[key] - def setdefault(self, key, default): - try: - return self[key] - except KeyError: - self[key] = default - return default - def pop(self, key, *args): - if len(args) > 1: - raise TypeError, "pop expected at most 2 arguments, got "\ - + repr(1 + len(args)) - try: - value = self[key] - except KeyError: - if args: - return args[0] - raise - del self[key] - return value - def popitem(self): - try: - k, v = self.iteritems().next() - except StopIteration: - raise KeyError, 'container is empty' - del self[k] - return (k, v) - def update(self, other): - # Make progressively weaker assumptions about "other" - if hasattr(other, 'iteritems'): # iteritems saves memory and lookups - for k, v in other.iteritems(): - self[k] = v - elif hasattr(other, '__iter__'): # iter saves memory - for k in other: - self[k] = other[k] - else: - for k in other.keys(): - self[k] = other[k] - def get(self, key, default=None): - try: - return self[key] - except KeyError: - return default - def __repr__(self): - return repr(dict(self.iteritems())) - def __cmp__(self, other): - if other is None: - return 1 - if isinstance(other, DictMixin): - other = dict(other.iteritems()) - return cmp(dict(self.iteritems()), other) - def __len__(self): - return len(self.keys()) - - def __nonzero__(self): - return bool(self.iteritems()) diff --git a/paste/util/converters.py b/paste/util/converters.py index f0ad349..11451bc 100644 --- a/paste/util/converters.py +++ b/paste/util/converters.py @@ -1,7 +1,11 @@ # (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org) # Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php + +import six + + def asbool(obj): - if isinstance(obj, (str, unicode)): + if isinstance(obj, (six.binary_type, six.text_type)): obj = obj.strip().lower() if obj in ['true', 'yes', 'on', 'y', 't', '1']: return True @@ -13,7 +17,7 @@ def asbool(obj): return bool(obj) def aslist(obj, sep=None, strip=True): - if isinstance(obj, (str, unicode)): + if isinstance(obj, (six.binary_type, six.text_type)): lst = obj.split(sep) if strip: lst = [v.strip() for v in lst] diff --git a/paste/util/dateinterval.py b/paste/util/dateinterval.py index 2195ab2..023bce4 100644 --- a/paste/util/dateinterval.py +++ b/paste/util/dateinterval.py @@ -7,7 +7,7 @@ years (leap years in particular). Accepts (y)ear, (b)month, (w)eek, (d)ay, (h)our, (m)inute, (s)econd. -Exports only timeEncode and timeDecode functions. +Exports only timeEncode and timeDecode functions. """ import re @@ -30,9 +30,10 @@ timeValues = { 'm': minute, 's': second, } -timeOrdered = timeValues.items() -timeOrdered.sort(lambda a, b: -cmp(a[1], b[1])) - +timeOrdered = list(timeValues.items()) +timeOrdered.sort(key=lambda x: x[1], reverse=True) + + def interval_encode(seconds, include_sign=False): """Encodes a number of seconds (representing a time interval) into a form like 1h2d3s. @@ -79,7 +80,7 @@ def interval_decode(s): s = s[1:] for match in allMatches(s, _timeRE): char = match.group(0)[-1].lower() - if not timeValues.has_key(char): + if char not in timeValues: # @@: should signal error continue time += int(match.group(0)[:-1]) * timeValues[char] diff --git a/paste/util/datetimeutil.py b/paste/util/datetimeutil.py index b1a7f38..3c6d7d9 100644 --- a/paste/util/datetimeutil.py +++ b/paste/util/datetimeutil.py @@ -1,361 +1,359 @@ -# (c) 2005 Clark C. Evans and contributors
-# This module is part of the Python Paste Project and is released under
-# the MIT License: http://www.opensource.org/licenses/mit-license.php
-# Some of this code was funded by: http://prometheusresearch.com
-"""
-Date, Time, and Timespan Parsing Utilities
-
-This module contains parsing support to create "human friendly"
-``datetime`` object parsing. The explicit goal of these routines is
-to provide a multi-format date/time support not unlike that found in
-Microsoft Excel. In most approaches, the input is very "strict" to
-prevent errors -- however, this approach is much more liberal since we
-are assuming the user-interface is parroting back the normalized value
-and thus the user has immediate feedback if the data is not typed in
-correctly.
-
- ``parse_date`` and ``normalize_date``
-
- These functions take a value like '9 jan 2007' and returns either an
- ``date`` object, or an ISO 8601 formatted date value such
- as '2007-01-09'. There is an option to provide an Oracle database
- style output as well, ``09 JAN 2007``, but this is not the default.
-
- This module always treats '/' delimiters as using US date order
- (since the author's clients are US based), hence '1/9/2007' is
- January 9th. Since this module treats the '-' as following
- European order this supports both modes of data-entry; together
- with immediate parroting back the result to the screen, the author
- has found this approach to work well in pratice.
-
- ``parse_time`` and ``normalize_time``
-
- These functions take a value like '1 pm' and returns either an
- ``time`` object, or an ISO 8601 formatted 24h clock time
- such as '13:00'. There is an option to provide for US style time
- values, '1:00 PM', however this is not the default.
-
- ``parse_datetime`` and ``normalize_datetime``
-
- These functions take a value like '9 jan 2007 at 1 pm' and returns
- either an ``datetime`` object, or an ISO 8601 formatted
- return (without the T) such as '2007-01-09 13:00'. There is an
- option to provide for Oracle / US style, '09 JAN 2007 @ 1:00 PM',
- however this is not the default.
-
- ``parse_delta`` and ``normalize_delta``
-
- These functions take a value like '1h 15m' and returns either an
- ``timedelta`` object, or an 2-decimal fixed-point
- numerical value in hours, such as '1.25'. The rationale is to
- support meeting or time-billing lengths, not to be an accurate
- representation in mili-seconds. As such not all valid
- ``timedelta`` values will have a normalized representation.
-
-"""
-from datetime import timedelta, time, date
-from time import localtime
-import string
-
-__all__ = ['parse_timedelta', 'normalize_timedelta',
- 'parse_time', 'normalize_time',
- 'parse_date', 'normalize_date']
-
-def _number(val):
- try:
- return string.atoi(val)
- except:
- return None
-
-#
-# timedelta
-#
-def parse_timedelta(val):
- """
- returns a ``timedelta`` object, or None
- """
- if not val:
- return None
- val = string.lower(val)
- if "." in val:
- val = float(val)
- return timedelta(hours=int(val), minutes=60*(val % 1.0))
- fHour = ("h" in val or ":" in val)
- fMin = ("m" in val or ":" in val)
- fFraction = "." in val
- for noise in "minu:teshour()":
- val = string.replace(val, noise, ' ')
- val = string.strip(val)
- val = string.split(val)
- hr = 0.0
- mi = 0
- val.reverse()
- if fHour:
- hr = int(val.pop())
- if fMin:
- mi = int(val.pop())
- if len(val) > 0 and not hr:
- hr = int(val.pop())
- return timedelta(hours=hr, minutes=mi)
-
-def normalize_timedelta(val):
- """
- produces a normalized string value of the timedelta
-
- This module returns a normalized time span value consisting of the
- number of hours in fractional form. For example '1h 15min' is
- formatted as 01.25.
- """
- if type(val) == str:
- val = parse_timedelta(val)
- if not val:
- return ''
- hr = val.seconds/3600
- mn = (val.seconds % 3600)/60
- return "%d.%02d" % (hr, mn * 100/60)
-
-#
-# time
-#
-def parse_time(val):
- if not val:
- return None
- hr = mi = 0
- val = string.lower(val)
- amflag = (-1 != string.find(val, 'a')) # set if AM is found
- pmflag = (-1 != string.find(val, 'p')) # set if PM is found
- for noise in ":amp.":
- val = string.replace(val, noise, ' ')
- val = string.split(val)
- if len(val) > 1:
- hr = int(val[0])
- mi = int(val[1])
- else:
- val = val[0]
- if len(val) < 1:
- pass
- elif 'now' == val:
- tm = localtime()
- hr = tm[3]
- mi = tm[4]
- elif 'noon' == val:
- hr = 12
- elif len(val) < 3:
- hr = int(val)
- if not amflag and not pmflag and hr < 7:
- hr += 12
- elif len(val) < 5:
- hr = int(val[:-2])
- mi = int(val[-2:])
- else:
- hr = int(val[:1])
- if amflag and hr >= 12:
- hr = hr - 12
- if pmflag and hr < 12:
- hr = hr + 12
- return time(hr, mi)
-
-def normalize_time(value, ampm):
- if not value:
- return ''
- if type(value) == str:
- value = parse_time(value)
- if not ampm:
- return "%02d:%02d" % (value.hour, value.minute)
- hr = value.hour
- am = "AM"
- if hr < 1 or hr > 23:
- hr = 12
- elif hr >= 12:
- am = "PM"
- if hr > 12:
- hr = hr - 12
- return "%02d:%02d %s" % (hr, value.minute, am)
-
-#
-# Date Processing
-#
-
-_one_day = timedelta(days=1)
-
-_str2num = {'jan':1, 'feb':2, 'mar':3, 'apr':4, 'may':5, 'jun':6,
- 'jul':7, 'aug':8, 'sep':9, 'oct':10, 'nov':11, 'dec':12 }
-
-def _month(val):
- for (key, mon) in _str2num.items():
- if key in val:
- return mon
- raise TypeError("unknown month '%s'" % val)
-
-_days_in_month = {1: 31, 2: 28, 3: 31, 4: 30, 5: 31, 6: 30,
- 7: 31, 8: 31, 9: 30, 10: 31, 11: 30, 12: 31,
- }
-_num2str = {1: 'Jan', 2: 'Feb', 3: 'Mar', 4: 'Apr', 5: 'May', 6: 'Jun',
- 7: 'Jul', 8: 'Aug', 9: 'Sep', 10: 'Oct', 11: 'Nov', 12: 'Dec',
- }
-_wkdy = ("mon", "tue", "wed", "thu", "fri", "sat", "sun")
-
-def parse_date(val):
- if not(val):
- return None
- val = string.lower(val)
- now = None
-
- # optimized check for YYYY-MM-DD
- strict = val.split("-")
- if len(strict) == 3:
- (y, m, d) = strict
- if "+" in d:
- d = d.split("+")[0]
- if " " in d:
- d = d.split(" ")[0]
- try:
- now = date(int(y), int(m), int(d))
- val = "xxx" + val[10:]
- except ValueError:
- pass
-
- # allow for 'now', 'mon', 'tue', etc.
- if not now:
- chk = val[:3]
- if chk in ('now','tod'):
- now = date.today()
- elif chk in _wkdy:
- now = date.today()
- idx = list(_wkdy).index(chk) + 1
- while now.isoweekday() != idx:
- now += _one_day
-
- # allow dates to be modified via + or - /w number of days, so
- # that now+3 is three days from now
- if now:
- tail = val[3:].strip()
- tail = tail.replace("+"," +").replace("-"," -")
- for item in tail.split():
- try:
- days = int(item)
- except ValueError:
- pass
- else:
- now += timedelta(days=days)
- return now
-
- # ok, standard parsing
- yr = mo = dy = None
- for noise in ('/', '-', ',', '*'):
- val = string.replace(val, noise, ' ')
- for noise in _wkdy:
- val = string.replace(val, noise, ' ')
- out = []
- last = False
- ldig = False
- for ch in val:
- if ch.isdigit():
- if last and not ldig:
- out.append(' ')
- last = ldig = True
- else:
- if ldig:
- out.append(' ')
- ldig = False
- last = True
- out.append(ch)
- val = string.split("".join(out))
- if 3 == len(val):
- a = _number(val[0])
- b = _number(val[1])
- c = _number(val[2])
- if len(val[0]) == 4:
- yr = a
- if b: # 1999 6 23
- mo = b
- dy = c
- else: # 1999 Jun 23
- mo = _month(val[1])
- dy = c
- elif a > 0:
- yr = c
- if len(val[2]) < 4:
- raise TypeError("four digit year required")
- if b: # 6 23 1999
- dy = b
- mo = a
- else: # 23 Jun 1999
- dy = a
- mo = _month(val[1])
- else: # Jun 23, 2000
- dy = b
- yr = c
- if len(val[2]) < 4:
- raise TypeError("four digit year required")
- mo = _month(val[0])
- elif 2 == len(val):
- a = _number(val[0])
- b = _number(val[1])
- if a > 999:
- yr = a
- dy = 1
- if b > 0: # 1999 6
- mo = b
- else: # 1999 Jun
- mo = _month(val[1])
- elif a > 0:
- if b > 999: # 6 1999
- mo = a
- yr = b
- dy = 1
- elif b > 0: # 6 23
- mo = a
- dy = b
- else: # 23 Jun
- dy = a
- mo = _month(val[1])
- else:
- if b > 999: # Jun 2001
- yr = b
- dy = 1
- else: # Jun 23
- dy = b
- mo = _month(val[0])
- elif 1 == len(val):
- val = val[0]
- if not val.isdigit():
- mo = _month(val)
- if mo is not None:
- dy = 1
- else:
- v = _number(val)
- val = str(v)
- if 8 == len(val): # 20010623
- yr = _number(val[:4])
- mo = _number(val[4:6])
- dy = _number(val[6:])
- elif len(val) in (3,4):
- if v > 1300: # 2004
- yr = v
- mo = 1
- dy = 1
- else: # 1202
- mo = _number(val[:-2])
- dy = _number(val[-2:])
- elif v < 32:
- dy = v
- else:
- raise TypeError("four digit year required")
- tm = localtime()
- if mo is None:
- mo = tm[1]
- if dy is None:
- dy = tm[2]
- if yr is None:
- yr = tm[0]
- return date(yr, mo, dy)
-
-def normalize_date(val, iso8601=True):
- if not val:
- return ''
- if type(val) == str:
- val = parse_date(val)
- if iso8601:
- return "%4d-%02d-%02d" % (val.year, val.month, val.day)
- return "%02d %s %4d" % (val.day, _num2str[val.month], val.year)
+# (c) 2005 Clark C. Evans and contributors +# This module is part of the Python Paste Project and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php +# Some of this code was funded by: http://prometheusresearch.com +""" +Date, Time, and Timespan Parsing Utilities + +This module contains parsing support to create "human friendly" +``datetime`` object parsing. The explicit goal of these routines is +to provide a multi-format date/time support not unlike that found in +Microsoft Excel. In most approaches, the input is very "strict" to +prevent errors -- however, this approach is much more liberal since we +are assuming the user-interface is parroting back the normalized value +and thus the user has immediate feedback if the data is not typed in +correctly. + + ``parse_date`` and ``normalize_date`` + + These functions take a value like '9 jan 2007' and returns either an + ``date`` object, or an ISO 8601 formatted date value such + as '2007-01-09'. There is an option to provide an Oracle database + style output as well, ``09 JAN 2007``, but this is not the default. + + This module always treats '/' delimiters as using US date order + (since the author's clients are US based), hence '1/9/2007' is + January 9th. Since this module treats the '-' as following + European order this supports both modes of data-entry; together + with immediate parroting back the result to the screen, the author + has found this approach to work well in pratice. + + ``parse_time`` and ``normalize_time`` + + These functions take a value like '1 pm' and returns either an + ``time`` object, or an ISO 8601 formatted 24h clock time + such as '13:00'. There is an option to provide for US style time + values, '1:00 PM', however this is not the default. + + ``parse_datetime`` and ``normalize_datetime`` + + These functions take a value like '9 jan 2007 at 1 pm' and returns + either an ``datetime`` object, or an ISO 8601 formatted + return (without the T) such as '2007-01-09 13:00'. There is an + option to provide for Oracle / US style, '09 JAN 2007 @ 1:00 PM', + however this is not the default. + + ``parse_delta`` and ``normalize_delta`` + + These functions take a value like '1h 15m' and returns either an + ``timedelta`` object, or an 2-decimal fixed-point + numerical value in hours, such as '1.25'. The rationale is to + support meeting or time-billing lengths, not to be an accurate + representation in mili-seconds. As such not all valid + ``timedelta`` values will have a normalized representation. + +""" +from datetime import timedelta, time, date +from time import localtime + +__all__ = ['parse_timedelta', 'normalize_timedelta', + 'parse_time', 'normalize_time', + 'parse_date', 'normalize_date'] + +def _number(val): + try: + return int(val) + except: + return None + +# +# timedelta +# +def parse_timedelta(val): + """ + returns a ``timedelta`` object, or None + """ + if not val: + return None + val = val.lower() + if "." in val: + val = float(val) + return timedelta(hours=int(val), minutes=60*(val % 1.0)) + fHour = ("h" in val or ":" in val) + fMin = ("m" in val or ":" in val) + for noise in "minu:teshour()": + val = val.replace(noise, ' ') + val = val.strip() + val = val.split() + hr = 0.0 + mi = 0 + val.reverse() + if fHour: + hr = int(val.pop()) + if fMin: + mi = int(val.pop()) + if len(val) > 0 and not hr: + hr = int(val.pop()) + return timedelta(hours=hr, minutes=mi) + +def normalize_timedelta(val): + """ + produces a normalized string value of the timedelta + + This module returns a normalized time span value consisting of the + number of hours in fractional form. For example '1h 15min' is + formatted as 01.25. + """ + if type(val) == str: + val = parse_timedelta(val) + if not val: + return '' + hr = val.seconds/3600 + mn = (val.seconds % 3600)/60 + return "%d.%02d" % (hr, mn * 100/60) + +# +# time +# +def parse_time(val): + if not val: + return None + hr = mi = 0 + val = val.lower() + amflag = (-1 != val.find('a')) # set if AM is found + pmflag = (-1 != val.find('p')) # set if PM is found + for noise in ":amp.": + val = val.replace(noise, ' ') + val = val.split() + if len(val) > 1: + hr = int(val[0]) + mi = int(val[1]) + else: + val = val[0] + if len(val) < 1: + pass + elif 'now' == val: + tm = localtime() + hr = tm[3] + mi = tm[4] + elif 'noon' == val: + hr = 12 + elif len(val) < 3: + hr = int(val) + if not amflag and not pmflag and hr < 7: + hr += 12 + elif len(val) < 5: + hr = int(val[:-2]) + mi = int(val[-2:]) + else: + hr = int(val[:1]) + if amflag and hr >= 12: + hr = hr - 12 + if pmflag and hr < 12: + hr = hr + 12 + return time(hr, mi) + +def normalize_time(value, ampm): + if not value: + return '' + if type(value) == str: + value = parse_time(value) + if not ampm: + return "%02d:%02d" % (value.hour, value.minute) + hr = value.hour + am = "AM" + if hr < 1 or hr > 23: + hr = 12 + elif hr >= 12: + am = "PM" + if hr > 12: + hr = hr - 12 + return "%02d:%02d %s" % (hr, value.minute, am) + +# +# Date Processing +# + +_one_day = timedelta(days=1) + +_str2num = {'jan':1, 'feb':2, 'mar':3, 'apr':4, 'may':5, 'jun':6, + 'jul':7, 'aug':8, 'sep':9, 'oct':10, 'nov':11, 'dec':12 } + +def _month(val): + for (key, mon) in _str2num.items(): + if key in val: + return mon + raise TypeError("unknown month '%s'" % val) + +_days_in_month = {1: 31, 2: 28, 3: 31, 4: 30, 5: 31, 6: 30, + 7: 31, 8: 31, 9: 30, 10: 31, 11: 30, 12: 31, + } +_num2str = {1: 'Jan', 2: 'Feb', 3: 'Mar', 4: 'Apr', 5: 'May', 6: 'Jun', + 7: 'Jul', 8: 'Aug', 9: 'Sep', 10: 'Oct', 11: 'Nov', 12: 'Dec', + } +_wkdy = ("mon", "tue", "wed", "thu", "fri", "sat", "sun") + +def parse_date(val): + if not(val): + return None + val = val.lower() + now = None + + # optimized check for YYYY-MM-DD + strict = val.split("-") + if len(strict) == 3: + (y, m, d) = strict + if "+" in d: + d = d.split("+")[0] + if " " in d: + d = d.split(" ")[0] + try: + now = date(int(y), int(m), int(d)) + val = "xxx" + val[10:] + except ValueError: + pass + + # allow for 'now', 'mon', 'tue', etc. + if not now: + chk = val[:3] + if chk in ('now','tod'): + now = date.today() + elif chk in _wkdy: + now = date.today() + idx = list(_wkdy).index(chk) + 1 + while now.isoweekday() != idx: + now += _one_day + + # allow dates to be modified via + or - /w number of days, so + # that now+3 is three days from now + if now: + tail = val[3:].strip() + tail = tail.replace("+"," +").replace("-"," -") + for item in tail.split(): + try: + days = int(item) + except ValueError: + pass + else: + now += timedelta(days=days) + return now + + # ok, standard parsing + yr = mo = dy = None + for noise in ('/', '-', ',', '*'): + val = val.replace(noise, ' ') + for noise in _wkdy: + val = val.replace(noise, ' ') + out = [] + last = False + ldig = False + for ch in val: + if ch.isdigit(): + if last and not ldig: + out.append(' ') + last = ldig = True + else: + if ldig: + out.append(' ') + ldig = False + last = True + out.append(ch) + val = "".join(out).split() + if 3 == len(val): + a = _number(val[0]) + b = _number(val[1]) + c = _number(val[2]) + if len(val[0]) == 4: + yr = a + if b: # 1999 6 23 + mo = b + dy = c + else: # 1999 Jun 23 + mo = _month(val[1]) + dy = c + elif a is not None and a > 0: + yr = c + if len(val[2]) < 4: + raise TypeError("four digit year required") + if b: # 6 23 1999 + dy = b + mo = a + else: # 23 Jun 1999 + dy = a + mo = _month(val[1]) + else: # Jun 23, 2000 + dy = b + yr = c + if len(val[2]) < 4: + raise TypeError("four digit year required") + mo = _month(val[0]) + elif 2 == len(val): + a = _number(val[0]) + b = _number(val[1]) + if a is not None and a > 999: + yr = a + dy = 1 + if b is not None and b > 0: # 1999 6 + mo = b + else: # 1999 Jun + mo = _month(val[1]) + elif a is not None and a > 0: + if b is not None and b > 999: # 6 1999 + mo = a + yr = b + dy = 1 + elif b is not None and b > 0: # 6 23 + mo = a + dy = b + else: # 23 Jun + dy = a + mo = _month(val[1]) + else: + if b > 999: # Jun 2001 + yr = b + dy = 1 + else: # Jun 23 + dy = b + mo = _month(val[0]) + elif 1 == len(val): + val = val[0] + if not val.isdigit(): + mo = _month(val) + if mo is not None: + dy = 1 + else: + v = _number(val) + val = str(v) + if 8 == len(val): # 20010623 + yr = _number(val[:4]) + mo = _number(val[4:6]) + dy = _number(val[6:]) + elif len(val) in (3,4): + if v is not None and v > 1300: # 2004 + yr = v + mo = 1 + dy = 1 + else: # 1202 + mo = _number(val[:-2]) + dy = _number(val[-2:]) + elif v < 32: + dy = v + else: + raise TypeError("four digit year required") + tm = localtime() + if mo is None: + mo = tm[1] + if dy is None: + dy = tm[2] + if yr is None: + yr = tm[0] + return date(yr, mo, dy) + +def normalize_date(val, iso8601=True): + if not val: + return '' + if type(val) == str: + val = parse_date(val) + if iso8601: + return "%4d-%02d-%02d" % (val.year, val.month, val.day) + return "%02d %s %4d" % (val.day, _num2str[val.month], val.year) diff --git a/paste/util/doctest24.py b/paste/util/doctest24.py deleted file mode 100644 index 28849ed..0000000 --- a/paste/util/doctest24.py +++ /dev/null @@ -1,2665 +0,0 @@ -# Module doctest. -# Released to the public domain 16-Jan-2001, by Tim Peters (tim@python.org). -# Major enhancements and refactoring by: -# Jim Fulton -# Edward Loper - -# Provided as-is; use at your own risk; no warranty; no promises; enjoy! - -r"""Module doctest -- a framework for running examples in docstrings. - -In simplest use, end each module M to be tested with: - -def _test(): - import doctest - doctest.testmod() - -if __name__ == "__main__": - _test() - -Then running the module as a script will cause the examples in the -docstrings to get executed and verified: - -python M.py - -This won't display anything unless an example fails, in which case the -failing example(s) and the cause(s) of the failure(s) are printed to stdout -(why not stderr? because stderr is a lame hack <0.2 wink>), and the final -line of output is "Test failed.". - -Run it with the -v switch instead: - -python M.py -v - -and a detailed report of all examples tried is printed to stdout, along -with assorted summaries at the end. - -You can force verbose mode by passing "verbose=True" to testmod, or prohibit -it by passing "verbose=False". In either of those cases, sys.argv is not -examined by testmod. - -There are a variety of other ways to run doctests, including integration -with the unittest framework, and support for running non-Python text -files containing doctests. There are also many ways to override parts -of doctest's default behaviors. See the Library Reference Manual for -details. -""" - -__docformat__ = 'reStructuredText en' - -__all__ = [ - # 0, Option Flags - 'register_optionflag', - 'DONT_ACCEPT_TRUE_FOR_1', - 'DONT_ACCEPT_BLANKLINE', - 'NORMALIZE_WHITESPACE', - 'ELLIPSIS', - 'IGNORE_EXCEPTION_DETAIL', - 'COMPARISON_FLAGS', - 'REPORT_UDIFF', - 'REPORT_CDIFF', - 'REPORT_NDIFF', - 'REPORT_ONLY_FIRST_FAILURE', - 'REPORTING_FLAGS', - # 1. Utility Functions - 'is_private', - # 2. Example & DocTest - 'Example', - 'DocTest', - # 3. Doctest Parser - 'DocTestParser', - # 4. Doctest Finder - 'DocTestFinder', - # 5. Doctest Runner - 'DocTestRunner', - 'OutputChecker', - 'DocTestFailure', - 'UnexpectedException', - 'DebugRunner', - # 6. Test Functions - 'testmod', - 'testfile', - 'run_docstring_examples', - # 7. Tester - 'Tester', - # 8. Unittest Support - 'DocTestSuite', - 'DocFileSuite', - 'set_unittest_reportflags', - # 9. Debugging Support - 'script_from_examples', - 'testsource', - 'debug_src', - 'debug', -] - -import __future__ - -import sys, traceback, inspect, linecache, os, re, types -import unittest, difflib, pdb, tempfile -import warnings -from StringIO import StringIO - -# Don't whine about the deprecated is_private function in this -# module's tests. -warnings.filterwarnings("ignore", "is_private", DeprecationWarning, - __name__, 0) - -# There are 4 basic classes: -# - Example: a <source, want> pair, plus an intra-docstring line number. -# - DocTest: a collection of examples, parsed from a docstring, plus -# info about where the docstring came from (name, filename, lineno). -# - DocTestFinder: extracts DocTests from a given object's docstring and -# its contained objects' docstrings. -# - DocTestRunner: runs DocTest cases, and accumulates statistics. -# -# So the basic picture is: -# -# list of: -# +------+ +---------+ +-------+ -# |object| --DocTestFinder-> | DocTest | --DocTestRunner-> |results| -# +------+ +---------+ +-------+ -# | Example | -# | ... | -# | Example | -# +---------+ - -# Option constants. - -OPTIONFLAGS_BY_NAME = {} -def register_optionflag(name): - flag = 1 << len(OPTIONFLAGS_BY_NAME) - OPTIONFLAGS_BY_NAME[name] = flag - return flag - -DONT_ACCEPT_TRUE_FOR_1 = register_optionflag('DONT_ACCEPT_TRUE_FOR_1') -DONT_ACCEPT_BLANKLINE = register_optionflag('DONT_ACCEPT_BLANKLINE') -NORMALIZE_WHITESPACE = register_optionflag('NORMALIZE_WHITESPACE') -ELLIPSIS = register_optionflag('ELLIPSIS') -IGNORE_EXCEPTION_DETAIL = register_optionflag('IGNORE_EXCEPTION_DETAIL') - -COMPARISON_FLAGS = (DONT_ACCEPT_TRUE_FOR_1 | - DONT_ACCEPT_BLANKLINE | - NORMALIZE_WHITESPACE | - ELLIPSIS | - IGNORE_EXCEPTION_DETAIL) - -REPORT_UDIFF = register_optionflag('REPORT_UDIFF') -REPORT_CDIFF = register_optionflag('REPORT_CDIFF') -REPORT_NDIFF = register_optionflag('REPORT_NDIFF') -REPORT_ONLY_FIRST_FAILURE = register_optionflag('REPORT_ONLY_FIRST_FAILURE') - -REPORTING_FLAGS = (REPORT_UDIFF | - REPORT_CDIFF | - REPORT_NDIFF | - REPORT_ONLY_FIRST_FAILURE) - -# Special string markers for use in `want` strings: -BLANKLINE_MARKER = '<BLANKLINE>' -ELLIPSIS_MARKER = '...' - -###################################################################### -## Table of Contents -###################################################################### -# 1. Utility Functions -# 2. Example & DocTest -- store test cases -# 3. DocTest Parser -- extracts examples from strings -# 4. DocTest Finder -- extracts test cases from objects -# 5. DocTest Runner -- runs test cases -# 6. Test Functions -- convenient wrappers for testing -# 7. Tester Class -- for backwards compatibility -# 8. Unittest Support -# 9. Debugging Support -# 10. Example Usage - -###################################################################### -## 1. Utility Functions -###################################################################### - -def is_private(prefix, base): - """prefix, base -> true iff name prefix + "." + base is "private". - - Prefix may be an empty string, and base does not contain a period. - Prefix is ignored (although functions you write conforming to this - protocol may make use of it). - Return true iff base begins with an (at least one) underscore, but - does not both begin and end with (at least) two underscores. - - >>> is_private("a.b", "my_func") - False - >>> is_private("____", "_my_func") - True - >>> is_private("someclass", "__init__") - False - >>> is_private("sometypo", "__init_") - True - >>> is_private("x.y.z", "_") - True - >>> is_private("_x.y.z", "__") - False - >>> is_private("", "") # senseless but consistent - False - """ - warnings.warn("is_private is deprecated; it wasn't useful; " - "examine DocTestFinder.find() lists instead", - DeprecationWarning, stacklevel=2) - return base[:1] == "_" and not base[:2] == "__" == base[-2:] - -def _extract_future_flags(globs): - """ - Return the compiler-flags associated with the future features that - have been imported into the given namespace (globs). - """ - flags = 0 - for fname in __future__.all_feature_names: - feature = globs.get(fname, None) - if feature is getattr(__future__, fname): - flags |= feature.compiler_flag - return flags - -def _normalize_module(module, depth=2): - """ - Return the module specified by `module`. In particular: - - If `module` is a module, then return module. - - If `module` is a string, then import and return the - module with that name. - - If `module` is None, then return the calling module. - The calling module is assumed to be the module of - the stack frame at the given depth in the call stack. - """ - if inspect.ismodule(module): - return module - elif isinstance(module, (str, unicode)): - return __import__(module, globals(), locals(), ["*"]) - elif module is None: - return sys.modules[sys._getframe(depth).f_globals['__name__']] - else: - raise TypeError("Expected a module, string, or None") - -def _indent(s, indent=4): - """ - Add the given number of space characters to the beginning every - non-blank line in `s`, and return the result. - """ - # This regexp matches the start of non-blank lines: - return re.sub('(?m)^(?!$)', indent*' ', s) - -def _exception_traceback(exc_info): - """ - Return a string containing a traceback message for the given - exc_info tuple (as returned by sys.exc_info()). - """ - # Get a traceback message. - excout = StringIO() - exc_type, exc_val, exc_tb = exc_info - traceback.print_exception(exc_type, exc_val, exc_tb, file=excout) - return excout.getvalue() - -# Override some StringIO methods. -class _SpoofOut(StringIO): - def getvalue(self): - result = StringIO.getvalue(self) - # If anything at all was written, make sure there's a trailing - # newline. There's no way for the expected output to indicate - # that a trailing newline is missing. - if result and not result.endswith("\n"): - result += "\n" - # Prevent softspace from screwing up the next test case, in - # case they used print with a trailing comma in an example. - if hasattr(self, "softspace"): - del self.softspace - return result - - def truncate(self, size=None): - StringIO.truncate(self, size) - if hasattr(self, "softspace"): - del self.softspace - -# Worst-case linear-time ellipsis matching. -def _ellipsis_match(want, got): - """ - Essentially the only subtle case: - >>> _ellipsis_match('aa...aa', 'aaa') - False - """ - if ELLIPSIS_MARKER not in want: - return want == got - - # Find "the real" strings. - ws = want.split(ELLIPSIS_MARKER) - assert len(ws) >= 2 - - # Deal with exact matches possibly needed at one or both ends. - startpos, endpos = 0, len(got) - w = ws[0] - if w: # starts with exact match - if got.startswith(w): - startpos = len(w) - del ws[0] - else: - return False - w = ws[-1] - if w: # ends with exact match - if got.endswith(w): - endpos -= len(w) - del ws[-1] - else: - return False - - if startpos > endpos: - # Exact end matches required more characters than we have, as in - # _ellipsis_match('aa...aa', 'aaa') - return False - - # For the rest, we only need to find the leftmost non-overlapping - # match for each piece. If there's no overall match that way alone, - # there's no overall match period. - for w in ws: - # w may be '' at times, if there are consecutive ellipses, or - # due to an ellipsis at the start or end of `want`. That's OK. - # Search for an empty string succeeds, and doesn't change startpos. - startpos = got.find(w, startpos, endpos) - if startpos < 0: - return False - startpos += len(w) - - return True - -def _comment_line(line): - "Return a commented form of the given line" - line = line.rstrip() - if line: - return '# '+line - else: - return '#' - -class _OutputRedirectingPdb(pdb.Pdb): - """ - A specialized version of the python debugger that redirects stdout - to a given stream when interacting with the user. Stdout is *not* - redirected when traced code is executed. - """ - def __init__(self, out): - self.__out = out - pdb.Pdb.__init__(self) - - def trace_dispatch(self, *args): - # Redirect stdout to the given stream. - save_stdout = sys.stdout - sys.stdout = self.__out - # Call Pdb's trace dispatch method. - try: - return pdb.Pdb.trace_dispatch(self, *args) - finally: - sys.stdout = save_stdout - -# [XX] Normalize with respect to os.path.pardir? -def _module_relative_path(module, path): - if not inspect.ismodule(module): - raise TypeError, 'Expected a module: %r' % module - if path.startswith('/'): - raise ValueError, 'Module-relative files may not have absolute paths' - - # Find the base directory for the path. - if hasattr(module, '__file__'): - # A normal module/package - basedir = os.path.split(module.__file__)[0] - elif module.__name__ == '__main__': - # An interactive session. - if len(sys.argv)>0 and sys.argv[0] != '': - basedir = os.path.split(sys.argv[0])[0] - else: - basedir = os.curdir - else: - # A module w/o __file__ (this includes builtins) - raise ValueError("Can't resolve paths relative to the module " + - module + " (it has no __file__)") - - # Combine the base directory and the path. - return os.path.join(basedir, *(path.split('/'))) - -###################################################################### -## 2. Example & DocTest -###################################################################### -## - An "example" is a <source, want> pair, where "source" is a -## fragment of source code, and "want" is the expected output for -## "source." The Example class also includes information about -## where the example was extracted from. -## -## - A "doctest" is a collection of examples, typically extracted from -## a string (such as an object's docstring). The DocTest class also -## includes information about where the string was extracted from. - -class Example: - """ - A single doctest example, consisting of source code and expected - output. `Example` defines the following attributes: - - - source: A single Python statement, always ending with a newline. - The constructor adds a newline if needed. - - - want: The expected output from running the source code (either - from stdout, or a traceback in case of exception). `want` ends - with a newline unless it's empty, in which case it's an empty - string. The constructor adds a newline if needed. - - - exc_msg: The exception message generated by the example, if - the example is expected to generate an exception; or `None` if - it is not expected to generate an exception. This exception - message is compared against the return value of - `traceback.format_exception_only()`. `exc_msg` ends with a - newline unless it's `None`. The constructor adds a newline - if needed. - - - lineno: The line number within the DocTest string containing - this Example where the Example begins. This line number is - zero-based, with respect to the beginning of the DocTest. - - - indent: The example's indentation in the DocTest string. - I.e., the number of space characters that preceed the - example's first prompt. - - - options: A dictionary mapping from option flags to True or - False, which is used to override default options for this - example. Any option flags not contained in this dictionary - are left at their default value (as specified by the - DocTestRunner's optionflags). By default, no options are set. - """ - def __init__(self, source, want, exc_msg=None, lineno=0, indent=0, - options=None): - # Normalize inputs. - if not source.endswith('\n'): - source += '\n' - if want and not want.endswith('\n'): - want += '\n' - if exc_msg is not None and not exc_msg.endswith('\n'): - exc_msg += '\n' - # Store properties. - self.source = source - self.want = want - self.lineno = lineno - self.indent = indent - if options is None: options = {} - self.options = options - self.exc_msg = exc_msg - -class DocTest: - """ - A collection of doctest examples that should be run in a single - namespace. Each `DocTest` defines the following attributes: - - - examples: the list of examples. - - - globs: The namespace (aka globals) that the examples should - be run in. - - - name: A name identifying the DocTest (typically, the name of - the object whose docstring this DocTest was extracted from). - - - filename: The name of the file that this DocTest was extracted - from, or `None` if the filename is unknown. - - - lineno: The line number within filename where this DocTest - begins, or `None` if the line number is unavailable. This - line number is zero-based, with respect to the beginning of - the file. - - - docstring: The string that the examples were extracted from, - or `None` if the string is unavailable. - """ - def __init__(self, examples, globs, name, filename, lineno, docstring): - """ - Create a new DocTest containing the given examples. The - DocTest's globals are initialized with a copy of `globs`. - """ - assert not isinstance(examples, basestring), \ - "DocTest no longer accepts str; use DocTestParser instead" - self.examples = examples - self.docstring = docstring - self.globs = globs.copy() - self.name = name - self.filename = filename - self.lineno = lineno - - def __repr__(self): - if len(self.examples) == 0: - examples = 'no examples' - elif len(self.examples) == 1: - examples = '1 example' - else: - examples = '%d examples' % len(self.examples) - return ('<DocTest %s from %s:%s (%s)>' % - (self.name, self.filename, self.lineno, examples)) - - - # This lets us sort tests by name: - def __cmp__(self, other): - if not isinstance(other, DocTest): - return -1 - return cmp((self.name, self.filename, self.lineno, id(self)), - (other.name, other.filename, other.lineno, id(other))) - -###################################################################### -## 3. DocTestParser -###################################################################### - -class DocTestParser: - """ - A class used to parse strings containing doctest examples. - """ - # This regular expression is used to find doctest examples in a - # string. It defines three groups: `source` is the source code - # (including leading indentation and prompts); `indent` is the - # indentation of the first (PS1) line of the source code; and - # `want` is the expected output (including leading indentation). - _EXAMPLE_RE = re.compile(r''' - # Source consists of a PS1 line followed by zero or more PS2 lines. - (?P<source> - (?:^(?P<indent> [ ]*) >>> .*) # PS1 line - (?:\n [ ]* \.\.\. .*)*) # PS2 lines - \n? - # Want consists of any non-blank lines that do not start with PS1. - (?P<want> (?:(?![ ]*$) # Not a blank line - (?![ ]*>>>) # Not a line starting with PS1 - .*$\n? # But any other line - )*) - ''', re.MULTILINE | re.VERBOSE) - - # A regular expression for handling `want` strings that contain - # expected exceptions. It divides `want` into three pieces: - # - the traceback header line (`hdr`) - # - the traceback stack (`stack`) - # - the exception message (`msg`), as generated by - # traceback.format_exception_only() - # `msg` may have multiple lines. We assume/require that the - # exception message is the first non-indented line starting with a word - # character following the traceback header line. - _EXCEPTION_RE = re.compile(r""" - # Grab the traceback header. Different versions of Python have - # said different things on the first traceback line. - ^(?P<hdr> Traceback\ \( - (?: most\ recent\ call\ last - | innermost\ last - ) \) : - ) - \s* $ # toss trailing whitespace on the header. - (?P<stack> .*?) # don't blink: absorb stuff until... - ^ (?P<msg> \w+ .*) # a line *starts* with alphanum. - """, re.VERBOSE | re.MULTILINE | re.DOTALL) - - # A callable returning a true value iff its argument is a blank line - # or contains a single comment. - _IS_BLANK_OR_COMMENT = re.compile(r'^[ ]*(#.*)?$').match - - def parse(self, string, name='<string>'): - """ - Divide the given string into examples and intervening text, - and return them as a list of alternating Examples and strings. - Line numbers for the Examples are 0-based. The optional - argument `name` is a name identifying this string, and is only - used for error messages. - """ - string = string.expandtabs() - # If all lines begin with the same indentation, then strip it. - min_indent = self._min_indent(string) - if min_indent > 0: - string = '\n'.join([l[min_indent:] for l in string.split('\n')]) - - output = [] - charno, lineno = 0, 0 - # Find all doctest examples in the string: - for m in self._EXAMPLE_RE.finditer(string): - # Add the pre-example text to `output`. - output.append(string[charno:m.start()]) - # Update lineno (lines before this example) - lineno += string.count('\n', charno, m.start()) - # Extract info from the regexp match. - (source, options, want, exc_msg) = \ - self._parse_example(m, name, lineno) - # Create an Example, and add it to the list. - if not self._IS_BLANK_OR_COMMENT(source): - output.append( Example(source, want, exc_msg, - lineno=lineno, - indent=min_indent+len(m.group('indent')), - options=options) ) - # Update lineno (lines inside this example) - lineno += string.count('\n', m.start(), m.end()) - # Update charno. - charno = m.end() - # Add any remaining post-example text to `output`. - output.append(string[charno:]) - return output - - def get_doctest(self, string, globs, name, filename, lineno): - """ - Extract all doctest examples from the given string, and - collect them into a `DocTest` object. - - `globs`, `name`, `filename`, and `lineno` are attributes for - the new `DocTest` object. See the documentation for `DocTest` - for more information. - """ - return DocTest(self.get_examples(string, name), globs, - name, filename, lineno, string) - - def get_examples(self, string, name='<string>'): - """ - Extract all doctest examples from the given string, and return - them as a list of `Example` objects. Line numbers are - 0-based, because it's most common in doctests that nothing - interesting appears on the same line as opening triple-quote, - and so the first interesting line is called \"line 1\" then. - - The optional argument `name` is a name identifying this - string, and is only used for error messages. - """ - return [x for x in self.parse(string, name) - if isinstance(x, Example)] - - def _parse_example(self, m, name, lineno): - """ - Given a regular expression match from `_EXAMPLE_RE` (`m`), - return a pair `(source, want)`, where `source` is the matched - example's source code (with prompts and indentation stripped); - and `want` is the example's expected output (with indentation - stripped). - - `name` is the string's name, and `lineno` is the line number - where the example starts; both are used for error messages. - """ - # Get the example's indentation level. - indent = len(m.group('indent')) - - # Divide source into lines; check that they're properly - # indented; and then strip their indentation & prompts. - source_lines = m.group('source').split('\n') - self._check_prompt_blank(source_lines, indent, name, lineno) - self._check_prefix(source_lines[1:], ' '*indent + '.', name, lineno) - source = '\n'.join([sl[indent+4:] for sl in source_lines]) - - # Divide want into lines; check that it's properly indented; and - # then strip the indentation. Spaces before the last newline should - # be preserved, so plain rstrip() isn't good enough. - want = m.group('want') - want_lines = want.split('\n') - if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]): - del want_lines[-1] # forget final newline & spaces after it - self._check_prefix(want_lines, ' '*indent, name, - lineno + len(source_lines)) - want = '\n'.join([wl[indent:] for wl in want_lines]) - - # If `want` contains a traceback message, then extract it. - m = self._EXCEPTION_RE.match(want) - if m: - exc_msg = m.group('msg') - else: - exc_msg = None - - # Extract options from the source. - options = self._find_options(source, name, lineno) - - return source, options, want, exc_msg - - # This regular expression looks for option directives in the - # source code of an example. Option directives are comments - # starting with "doctest:". Warning: this may give false - # positives for string-literals that contain the string - # "#doctest:". Eliminating these false positives would require - # actually parsing the string; but we limit them by ignoring any - # line containing "#doctest:" that is *followed* by a quote mark. - _OPTION_DIRECTIVE_RE = re.compile(r'#\s*doctest:\s*([^\n\'"]*)$', - re.MULTILINE) - - def _find_options(self, source, name, lineno): - """ - Return a dictionary containing option overrides extracted from - option directives in the given source string. - - `name` is the string's name, and `lineno` is the line number - where the example starts; both are used for error messages. - """ - options = {} - # (note: with the current regexp, this will match at most once:) - for m in self._OPTION_DIRECTIVE_RE.finditer(source): - option_strings = m.group(1).replace(',', ' ').split() - for option in option_strings: - if (option[0] not in '+-' or - option[1:] not in OPTIONFLAGS_BY_NAME): - raise ValueError('line %r of the doctest for %s ' - 'has an invalid option: %r' % - (lineno+1, name, option)) - flag = OPTIONFLAGS_BY_NAME[option[1:]] - options[flag] = (option[0] == '+') - if options and self._IS_BLANK_OR_COMMENT(source): - raise ValueError('line %r of the doctest for %s has an option ' - 'directive on a line with no example: %r' % - (lineno, name, source)) - return options - - # This regular expression finds the indentation of every non-blank - # line in a string. - _INDENT_RE = re.compile('^([ ]*)(?=\S)', re.MULTILINE) - - def _min_indent(self, s): - "Return the minimum indentation of any non-blank line in `s`" - indents = [len(indent) for indent in self._INDENT_RE.findall(s)] - if len(indents) > 0: - return min(indents) - else: - return 0 - - def _check_prompt_blank(self, lines, indent, name, lineno): - """ - Given the lines of a source string (including prompts and - leading indentation), check to make sure that every prompt is - followed by a space character. If any line is not followed by - a space character, then raise ValueError. - """ - for i, line in enumerate(lines): - if len(line) >= indent+4 and line[indent+3] != ' ': - raise ValueError('line %r of the docstring for %s ' - 'lacks blank after %s: %r' % - (lineno+i+1, name, - line[indent:indent+3], line)) - - def _check_prefix(self, lines, prefix, name, lineno): - """ - Check that every line in the given list starts with the given - prefix; if any line does not, then raise a ValueError. - """ - for i, line in enumerate(lines): - if line and not line.startswith(prefix): - raise ValueError('line %r of the docstring for %s has ' - 'inconsistent leading whitespace: %r' % - (lineno+i+1, name, line)) - - -###################################################################### -## 4. DocTest Finder -###################################################################### - -class DocTestFinder: - """ - A class used to extract the DocTests that are relevant to a given - object, from its docstring and the docstrings of its contained - objects. Doctests can currently be extracted from the following - object types: modules, functions, classes, methods, staticmethods, - classmethods, and properties. - """ - - def __init__(self, verbose=False, parser=DocTestParser(), - recurse=True, _namefilter=None, exclude_empty=True): - """ - Create a new doctest finder. - - The optional argument `parser` specifies a class or - function that should be used to create new DocTest objects (or - objects that implement the same interface as DocTest). The - signature for this factory function should match the signature - of the DocTest constructor. - - If the optional argument `recurse` is false, then `find` will - only examine the given object, and not any contained objects. - - If the optional argument `exclude_empty` is false, then `find` - will include tests for objects with empty docstrings. - """ - self._parser = parser - self._verbose = verbose - self._recurse = recurse - self._exclude_empty = exclude_empty - # _namefilter is undocumented, and exists only for temporary backward- - # compatibility support of testmod's deprecated isprivate mess. - self._namefilter = _namefilter - - def find(self, obj, name=None, module=None, globs=None, - extraglobs=None): - """ - Return a list of the DocTests that are defined by the given - object's docstring, or by any of its contained objects' - docstrings. - - The optional parameter `module` is the module that contains - the given object. If the module is not specified or is None, then - the test finder will attempt to automatically determine the - correct module. The object's module is used: - - - As a default namespace, if `globs` is not specified. - - To prevent the DocTestFinder from extracting DocTests - from objects that are imported from other modules. - - To find the name of the file containing the object. - - To help find the line number of the object within its - file. - - Contained objects whose module does not match `module` are ignored. - - If `module` is False, no attempt to find the module will be made. - This is obscure, of use mostly in tests: if `module` is False, or - is None but cannot be found automatically, then all objects are - considered to belong to the (non-existent) module, so all contained - objects will (recursively) be searched for doctests. - - The globals for each DocTest is formed by combining `globs` - and `extraglobs` (bindings in `extraglobs` override bindings - in `globs`). A new copy of the globals dictionary is created - for each DocTest. If `globs` is not specified, then it - defaults to the module's `__dict__`, if specified, or {} - otherwise. If `extraglobs` is not specified, then it defaults - to {}. - - """ - # If name was not specified, then extract it from the object. - if name is None: - name = getattr(obj, '__name__', None) - if name is None: - raise ValueError("DocTestFinder.find: name must be given " - "when obj.__name__ doesn't exist: %r" % - (type(obj),)) - - # Find the module that contains the given object (if obj is - # a module, then module=obj.). Note: this may fail, in which - # case module will be None. - if module is False: - module = None - elif module is None: - module = inspect.getmodule(obj) - - # Read the module's source code. This is used by - # DocTestFinder._find_lineno to find the line number for a - # given object's docstring. - try: - file = inspect.getsourcefile(obj) or inspect.getfile(obj) - source_lines = linecache.getlines(file) - if not source_lines: - source_lines = None - except TypeError: - source_lines = None - - # Initialize globals, and merge in extraglobs. - if globs is None: - if module is None: - globs = {} - else: - globs = module.__dict__.copy() - else: - globs = globs.copy() - if extraglobs is not None: - globs.update(extraglobs) - - # Recursively expore `obj`, extracting DocTests. - tests = [] - self._find(tests, obj, name, module, source_lines, globs, {}) - return tests - - def _filter(self, obj, prefix, base): - """ - Return true if the given object should not be examined. - """ - return (self._namefilter is not None and - self._namefilter(prefix, base)) - - def _from_module(self, module, object): - """ - Return true if the given object is defined in the given - module. - """ - if module is None: - return True - elif inspect.isfunction(object): - return module.__dict__ is object.func_globals - elif inspect.isclass(object): - return module.__name__ == object.__module__ - elif inspect.getmodule(object) is not None: - return module is inspect.getmodule(object) - elif hasattr(object, '__module__'): - return module.__name__ == object.__module__ - elif isinstance(object, property): - return True # [XX] no way not be sure. - else: - raise ValueError("object must be a class or function") - - def _find(self, tests, obj, name, module, source_lines, globs, seen): - """ - Find tests for the given object and any contained objects, and - add them to `tests`. - """ - if self._verbose: - print 'Finding tests in %s' % name - - # If we've already processed this object, then ignore it. - if id(obj) in seen: - return - seen[id(obj)] = 1 - - # Find a test for this object, and add it to the list of tests. - test = self._get_test(obj, name, module, globs, source_lines) - if test is not None: - tests.append(test) - - # Look for tests in a module's contained objects. - if inspect.ismodule(obj) and self._recurse: - for valname, val in obj.__dict__.items(): - # Check if this contained object should be ignored. - if self._filter(val, name, valname): - continue - valname = '%s.%s' % (name, valname) - # Recurse to functions & classes. - if ((inspect.isfunction(val) or inspect.isclass(val)) and - self._from_module(module, val)): - self._find(tests, val, valname, module, source_lines, - globs, seen) - - # Look for tests in a module's __test__ dictionary. - if inspect.ismodule(obj) and self._recurse: - for valname, val in getattr(obj, '__test__', {}).items(): - if not isinstance(valname, basestring): - raise ValueError("DocTestFinder.find: __test__ keys " - "must be strings: %r" % - (type(valname),)) - if not (inspect.isfunction(val) or inspect.isclass(val) or - inspect.ismethod(val) or inspect.ismodule(val) or - isinstance(val, basestring)): - raise ValueError("DocTestFinder.find: __test__ values " - "must be strings, functions, methods, " - "classes, or modules: %r" % - (type(val),)) - valname = '%s.__test__.%s' % (name, valname) - self._find(tests, val, valname, module, source_lines, - globs, seen) - - # Look for tests in a class's contained objects. - if inspect.isclass(obj) and self._recurse: - for valname, val in obj.__dict__.items(): - # Check if this contained object should be ignored. - if self._filter(val, name, valname): - continue - # Special handling for staticmethod/classmethod. - if isinstance(val, staticmethod): - val = getattr(obj, valname) - if isinstance(val, classmethod): - val = getattr(obj, valname).im_func - - # Recurse to methods, properties, and nested classes. - if ((inspect.isfunction(val) or inspect.isclass(val) or - isinstance(val, property)) and - self._from_module(module, val)): - valname = '%s.%s' % (name, valname) - self._find(tests, val, valname, module, source_lines, - globs, seen) - - def _get_test(self, obj, name, module, globs, source_lines): - """ - Return a DocTest for the given object, if it defines a docstring; - otherwise, return None. - """ - # Extract the object's docstring. If it doesn't have one, - # then return None (no test for this object). - if isinstance(obj, basestring): - docstring = obj - else: - try: - if obj.__doc__ is None: - docstring = '' - else: - docstring = obj.__doc__ - if not isinstance(docstring, basestring): - docstring = str(docstring) - except (TypeError, AttributeError): - docstring = '' - - # Find the docstring's location in the file. - lineno = self._find_lineno(obj, source_lines) - - # Don't bother if the docstring is empty. - if self._exclude_empty and not docstring: - return None - - # Return a DocTest for this object. - if module is None: - filename = None - else: - filename = getattr(module, '__file__', module.__name__) - if filename[-4:] in (".pyc", ".pyo"): - filename = filename[:-1] - return self._parser.get_doctest(docstring, globs, name, - filename, lineno) - - def _find_lineno(self, obj, source_lines): - """ - Return a line number of the given object's docstring. Note: - this method assumes that the object has a docstring. - """ - lineno = None - - # Find the line number for modules. - if inspect.ismodule(obj): - lineno = 0 - - # Find the line number for classes. - # Note: this could be fooled if a class is defined multiple - # times in a single file. - if inspect.isclass(obj): - if source_lines is None: - return None - pat = re.compile(r'^\s*class\s*%s\b' % - getattr(obj, '__name__', '-')) - for i, line in enumerate(source_lines): - if pat.match(line): - lineno = i - break - - # Find the line number for functions & methods. - if inspect.ismethod(obj): obj = obj.im_func - if inspect.isfunction(obj): obj = obj.func_code - if inspect.istraceback(obj): obj = obj.tb_frame - if inspect.isframe(obj): obj = obj.f_code - if inspect.iscode(obj): - lineno = getattr(obj, 'co_firstlineno', None)-1 - - # Find the line number where the docstring starts. Assume - # that it's the first line that begins with a quote mark. - # Note: this could be fooled by a multiline function - # signature, where a continuation line begins with a quote - # mark. - if lineno is not None: - if source_lines is None: - return lineno+1 - pat = re.compile('(^|.*:)\s*\w*("|\')') - for lineno in range(lineno, len(source_lines)): - if pat.match(source_lines[lineno]): - return lineno - - # We couldn't find the line number. - return None - -###################################################################### -## 5. DocTest Runner -###################################################################### - -class DocTestRunner: - """ - A class used to run DocTest test cases, and accumulate statistics. - The `run` method is used to process a single DocTest case. It - returns a tuple `(f, t)`, where `t` is the number of test cases - tried, and `f` is the number of test cases that failed. - - >>> tests = DocTestFinder().find(_TestClass) - >>> runner = DocTestRunner(verbose=False) - >>> for test in tests: - ... print runner.run(test) - (0, 2) - (0, 1) - (0, 2) - (0, 2) - - The `summarize` method prints a summary of all the test cases that - have been run by the runner, and returns an aggregated `(f, t)` - tuple: - - >>> runner.summarize(verbose=1) - 4 items passed all tests: - 2 tests in _TestClass - 2 tests in _TestClass.__init__ - 2 tests in _TestClass.get - 1 tests in _TestClass.square - 7 tests in 4 items. - 7 passed and 0 failed. - Test passed. - (0, 7) - - The aggregated number of tried examples and failed examples is - also available via the `tries` and `failures` attributes: - - >>> runner.tries - 7 - >>> runner.failures - 0 - - The comparison between expected outputs and actual outputs is done - by an `OutputChecker`. This comparison may be customized with a - number of option flags; see the documentation for `testmod` for - more information. If the option flags are insufficient, then the - comparison may also be customized by passing a subclass of - `OutputChecker` to the constructor. - - The test runner's display output can be controlled in two ways. - First, an output function (`out) can be passed to - `TestRunner.run`; this function will be called with strings that - should be displayed. It defaults to `sys.stdout.write`. If - capturing the output is not sufficient, then the display output - can be also customized by subclassing DocTestRunner, and - overriding the methods `report_start`, `report_success`, - `report_unexpected_exception`, and `report_failure`. - """ - # This divider string is used to separate failure messages, and to - # separate sections of the summary. - DIVIDER = "*" * 70 - - def __init__(self, checker=None, verbose=None, optionflags=0): - """ - Create a new test runner. - - Optional keyword arg `checker` is the `OutputChecker` that - should be used to compare the expected outputs and actual - outputs of doctest examples. - - Optional keyword arg 'verbose' prints lots of stuff if true, - only failures if false; by default, it's true iff '-v' is in - sys.argv. - - Optional argument `optionflags` can be used to control how the - test runner compares expected output to actual output, and how - it displays failures. See the documentation for `testmod` for - more information. - """ - self._checker = checker or OutputChecker() - if verbose is None: - verbose = '-v' in sys.argv - self._verbose = verbose - self.optionflags = optionflags - self.original_optionflags = optionflags - - # Keep track of the examples we've run. - self.tries = 0 - self.failures = 0 - self._name2ft = {} - - # Create a fake output target for capturing doctest output. - self._fakeout = _SpoofOut() - - #///////////////////////////////////////////////////////////////// - # Reporting methods - #///////////////////////////////////////////////////////////////// - - def report_start(self, out, test, example): - """ - Report that the test runner is about to process the given - example. (Only displays a message if verbose=True) - """ - if self._verbose: - if example.want: - out('Trying:\n' + _indent(example.source) + - 'Expecting:\n' + _indent(example.want)) - else: - out('Trying:\n' + _indent(example.source) + - 'Expecting nothing\n') - - def report_success(self, out, test, example, got): - """ - Report that the given example ran successfully. (Only - displays a message if verbose=True) - """ - if self._verbose: - out("ok\n") - - def report_failure(self, out, test, example, got): - """ - Report that the given example failed. - """ - out(self._failure_header(test, example) + - self._checker.output_difference(example, got, self.optionflags)) - - def report_unexpected_exception(self, out, test, example, exc_info): - """ - Report that the given example raised an unexpected exception. - """ - out(self._failure_header(test, example) + - 'Exception raised:\n' + _indent(_exception_traceback(exc_info))) - - def _failure_header(self, test, example): - out = [self.DIVIDER] - if test.filename: - if test.lineno is not None and example.lineno is not None: - lineno = test.lineno + example.lineno + 1 - else: - lineno = '?' - out.append('File "%s", line %s, in %s' % - (test.filename, lineno, test.name)) - else: - out.append('Line %s, in %s' % (example.lineno+1, test.name)) - out.append('Failed example:') - source = example.source - out.append(_indent(source)) - return '\n'.join(out) - - #///////////////////////////////////////////////////////////////// - # DocTest Running - #///////////////////////////////////////////////////////////////// - - def __run(self, test, compileflags, out): - """ - Run the examples in `test`. Write the outcome of each example - with one of the `DocTestRunner.report_*` methods, using the - writer function `out`. `compileflags` is the set of compiler - flags that should be used to execute examples. Return a tuple - `(f, t)`, where `t` is the number of examples tried, and `f` - is the number of examples that failed. The examples are run - in the namespace `test.globs`. - """ - # Keep track of the number of failures and tries. - failures = tries = 0 - - # Save the option flags (since option directives can be used - # to modify them). - original_optionflags = self.optionflags - - SUCCESS, FAILURE, BOOM = range(3) # `outcome` state - - check = self._checker.check_output - - # Process each example. - for examplenum, example in enumerate(test.examples): - - # If REPORT_ONLY_FIRST_FAILURE is set, then supress - # reporting after the first failure. - quiet = (self.optionflags & REPORT_ONLY_FIRST_FAILURE and - failures > 0) - - # Merge in the example's options. - self.optionflags = original_optionflags - if example.options: - for (optionflag, val) in example.options.items(): - if val: - self.optionflags |= optionflag - else: - self.optionflags &= ~optionflag - - # Record that we started this example. - tries += 1 - if not quiet: - self.report_start(out, test, example) - - # Use a special filename for compile(), so we can retrieve - # the source code during interactive debugging (see - # __patched_linecache_getlines). - filename = '<doctest %s[%d]>' % (test.name, examplenum) - - # Run the example in the given context (globs), and record - # any exception that gets raised. (But don't intercept - # keyboard interrupts.) - try: - # Don't blink! This is where the user's code gets run. - exec compile(example.source, filename, "single", - compileflags, 1) in test.globs - self.debugger.set_continue() # ==== Example Finished ==== - exception = None - except KeyboardInterrupt: - raise - except: - exception = sys.exc_info() - self.debugger.set_continue() # ==== Example Finished ==== - - got = self._fakeout.getvalue() # the actual output - self._fakeout.truncate(0) - outcome = FAILURE # guilty until proved innocent or insane - - # If the example executed without raising any exceptions, - # verify its output. - if exception is None: - if check(example.want, got, self.optionflags): - outcome = SUCCESS - - # The example raised an exception: check if it was expected. - else: - exc_info = sys.exc_info() - exc_msg = traceback.format_exception_only(*exc_info[:2])[-1] - if not quiet: - got += _exception_traceback(exc_info) - - # If `example.exc_msg` is None, then we weren't expecting - # an exception. - if example.exc_msg is None: - outcome = BOOM - - # We expected an exception: see whether it matches. - elif check(example.exc_msg, exc_msg, self.optionflags): - outcome = SUCCESS - - # Another chance if they didn't care about the detail. - elif self.optionflags & IGNORE_EXCEPTION_DETAIL: - m1 = re.match(r'[^:]*:', example.exc_msg) - m2 = re.match(r'[^:]*:', exc_msg) - if m1 and m2 and check(m1.group(0), m2.group(0), - self.optionflags): - outcome = SUCCESS - - # Report the outcome. - if outcome is SUCCESS: - if not quiet: - self.report_success(out, test, example, got) - elif outcome is FAILURE: - if not quiet: - self.report_failure(out, test, example, got) - failures += 1 - elif outcome is BOOM: - if not quiet: - self.report_unexpected_exception(out, test, example, - exc_info) - failures += 1 - else: - assert False, ("unknown outcome", outcome) - - # Restore the option flags (in case they were modified) - self.optionflags = original_optionflags - - # Record and return the number of failures and tries. - self.__record_outcome(test, failures, tries) - return failures, tries - - def __record_outcome(self, test, f, t): - """ - Record the fact that the given DocTest (`test`) generated `f` - failures out of `t` tried examples. - """ - f2, t2 = self._name2ft.get(test.name, (0,0)) - self._name2ft[test.name] = (f+f2, t+t2) - self.failures += f - self.tries += t - - __LINECACHE_FILENAME_RE = re.compile(r'<doctest ' - r'(?P<name>[\w\.]+)' - r'\[(?P<examplenum>\d+)\]>$') - def __patched_linecache_getlines(self, filename, module_globals=None): - m = self.__LINECACHE_FILENAME_RE.match(filename) - if m and m.group('name') == self.test.name: - example = self.test.examples[int(m.group('examplenum'))] - return example.source.splitlines(True) - else: - return self.save_linecache_getlines(filename)#?, module_globals) - - def run(self, test, compileflags=None, out=None, clear_globs=True): - """ - Run the examples in `test`, and display the results using the - writer function `out`. - - The examples are run in the namespace `test.globs`. If - `clear_globs` is true (the default), then this namespace will - be cleared after the test runs, to help with garbage - collection. If you would like to examine the namespace after - the test completes, then use `clear_globs=False`. - - `compileflags` gives the set of flags that should be used by - the Python compiler when running the examples. If not - specified, then it will default to the set of future-import - flags that apply to `globs`. - - The output of each example is checked using - `DocTestRunner.check_output`, and the results are formatted by - the `DocTestRunner.report_*` methods. - """ - self.test = test - - if compileflags is None: - compileflags = _extract_future_flags(test.globs) - - save_stdout = sys.stdout - if out is None: - out = save_stdout.write - sys.stdout = self._fakeout - - # Patch pdb.set_trace to restore sys.stdout during interactive - # debugging (so it's not still redirected to self._fakeout). - # Note that the interactive output will go to *our* - # save_stdout, even if that's not the real sys.stdout; this - # allows us to write test cases for the set_trace behavior. - save_set_trace = pdb.set_trace - self.debugger = _OutputRedirectingPdb(save_stdout) - self.debugger.reset() - pdb.set_trace = self.debugger.set_trace - - # Patch linecache.getlines, so we can see the example's source - # when we're inside the debugger. - self.save_linecache_getlines = linecache.getlines - linecache.getlines = self.__patched_linecache_getlines - - try: - return self.__run(test, compileflags, out) - finally: - sys.stdout = save_stdout - pdb.set_trace = save_set_trace - linecache.getlines = self.save_linecache_getlines - if clear_globs: - test.globs.clear() - - #///////////////////////////////////////////////////////////////// - # Summarization - #///////////////////////////////////////////////////////////////// - def summarize(self, verbose=None): - """ - Print a summary of all the test cases that have been run by - this DocTestRunner, and return a tuple `(f, t)`, where `f` is - the total number of failed examples, and `t` is the total - number of tried examples. - - The optional `verbose` argument controls how detailed the - summary is. If the verbosity is not specified, then the - DocTestRunner's verbosity is used. - """ - if verbose is None: - verbose = self._verbose - notests = [] - passed = [] - failed = [] - totalt = totalf = 0 - for x in self._name2ft.items(): - name, (f, t) = x - assert f <= t - totalt += t - totalf += f - if t == 0: - notests.append(name) - elif f == 0: - passed.append( (name, t) ) - else: - failed.append(x) - if verbose: - if notests: - print len(notests), "items had no tests:" - notests.sort() - for thing in notests: - print " ", thing - if passed: - print len(passed), "items passed all tests:" - passed.sort() - for thing, count in passed: - print " %3d tests in %s" % (count, thing) - if failed: - print self.DIVIDER - print len(failed), "items had failures:" - failed.sort() - for thing, (f, t) in failed: - print " %3d of %3d in %s" % (f, t, thing) - if verbose: - print totalt, "tests in", len(self._name2ft), "items." - print totalt - totalf, "passed and", totalf, "failed." - if totalf: - print "***Test Failed***", totalf, "failures." - elif verbose: - print "Test passed." - return totalf, totalt - - #///////////////////////////////////////////////////////////////// - # Backward compatibility cruft to maintain doctest.master. - #///////////////////////////////////////////////////////////////// - def merge(self, other): - d = self._name2ft - for name, (f, t) in other._name2ft.items(): - if name in d: - print "*** DocTestRunner.merge: '" + name + "' in both" \ - " testers; summing outcomes." - f2, t2 = d[name] - f = f + f2 - t = t + t2 - d[name] = f, t - -class OutputChecker: - """ - A class used to check the whether the actual output from a doctest - example matches the expected output. `OutputChecker` defines two - methods: `check_output`, which compares a given pair of outputs, - and returns true if they match; and `output_difference`, which - returns a string describing the differences between two outputs. - """ - def check_output(self, want, got, optionflags): - """ - Return True iff the actual output from an example (`got`) - matches the expected output (`want`). These strings are - always considered to match if they are identical; but - depending on what option flags the test runner is using, - several non-exact match types are also possible. See the - documentation for `TestRunner` for more information about - option flags. - """ - # Handle the common case first, for efficiency: - # if they're string-identical, always return true. - if got == want: - return True - - # The values True and False replaced 1 and 0 as the return - # value for boolean comparisons in Python 2.3. - if not (optionflags & DONT_ACCEPT_TRUE_FOR_1): - if (got,want) == ("True\n", "1\n"): - return True - if (got,want) == ("False\n", "0\n"): - return True - - # <BLANKLINE> can be used as a special sequence to signify a - # blank line, unless the DONT_ACCEPT_BLANKLINE flag is used. - if not (optionflags & DONT_ACCEPT_BLANKLINE): - # Replace <BLANKLINE> in want with a blank line. - want = re.sub('(?m)^%s\s*?$' % re.escape(BLANKLINE_MARKER), - '', want) - # If a line in got contains only spaces, then remove the - # spaces. - got = re.sub('(?m)^\s*?$', '', got) - if got == want: - return True - - # This flag causes doctest to ignore any differences in the - # contents of whitespace strings. Note that this can be used - # in conjunction with the ELLIPSIS flag. - if optionflags & NORMALIZE_WHITESPACE: - got = ' '.join(got.split()) - want = ' '.join(want.split()) - if got == want: - return True - - # The ELLIPSIS flag says to let the sequence "..." in `want` - # match any substring in `got`. - if optionflags & ELLIPSIS: - if _ellipsis_match(want, got): - return True - - # We didn't find any match; return false. - return False - - # Should we do a fancy diff? - def _do_a_fancy_diff(self, want, got, optionflags): - # Not unless they asked for a fancy diff. - if not optionflags & (REPORT_UDIFF | - REPORT_CDIFF | - REPORT_NDIFF): - return False - - # If expected output uses ellipsis, a meaningful fancy diff is - # too hard ... or maybe not. In two real-life failures Tim saw, - # a diff was a major help anyway, so this is commented out. - # [todo] _ellipsis_match() knows which pieces do and don't match, - # and could be the basis for a kick-ass diff in this case. - ##if optionflags & ELLIPSIS and ELLIPSIS_MARKER in want: - ## return False - - # ndiff does intraline difference marking, so can be useful even - # for 1-line differences. - if optionflags & REPORT_NDIFF: - return True - - # The other diff types need at least a few lines to be helpful. - return want.count('\n') > 2 and got.count('\n') > 2 - - def output_difference(self, example, got, optionflags): - """ - Return a string describing the differences between the - expected output for a given example (`example`) and the actual - output (`got`). `optionflags` is the set of option flags used - to compare `want` and `got`. - """ - want = example.want - # If <BLANKLINE>s are being used, then replace blank lines - # with <BLANKLINE> in the actual output string. - if not (optionflags & DONT_ACCEPT_BLANKLINE): - got = re.sub('(?m)^[ ]*(?=\n)', BLANKLINE_MARKER, got) - - # Check if we should use diff. - if self._do_a_fancy_diff(want, got, optionflags): - # Split want & got into lines. - want_lines = want.splitlines(True) # True == keep line ends - got_lines = got.splitlines(True) - # Use difflib to find their differences. - if optionflags & REPORT_UDIFF: - diff = difflib.unified_diff(want_lines, got_lines, n=2) - diff = list(diff)[2:] # strip the diff header - kind = 'unified diff with -expected +actual' - elif optionflags & REPORT_CDIFF: - diff = difflib.context_diff(want_lines, got_lines, n=2) - diff = list(diff)[2:] # strip the diff header - kind = 'context diff with expected followed by actual' - elif optionflags & REPORT_NDIFF: - engine = difflib.Differ(charjunk=difflib.IS_CHARACTER_JUNK) - diff = list(engine.compare(want_lines, got_lines)) - kind = 'ndiff with -expected +actual' - else: - assert 0, 'Bad diff option' - # Remove trailing whitespace on diff output. - diff = [line.rstrip() + '\n' for line in diff] - return 'Differences (%s):\n' % kind + _indent(''.join(diff)) - - # If we're not using diff, then simply list the expected - # output followed by the actual output. - if want and got: - return 'Expected:\n%sGot:\n%s' % (_indent(want), _indent(got)) - elif want: - return 'Expected:\n%sGot nothing\n' % _indent(want) - elif got: - return 'Expected nothing\nGot:\n%s' % _indent(got) - else: - return 'Expected nothing\nGot nothing\n' - -class DocTestFailure(Exception): - """A DocTest example has failed in debugging mode. - - The exception instance has variables: - - - test: the DocTest object being run - - - excample: the Example object that failed - - - got: the actual output - """ - def __init__(self, test, example, got): - self.test = test - self.example = example - self.got = got - - def __str__(self): - return str(self.test) - -class UnexpectedException(Exception): - """A DocTest example has encountered an unexpected exception - - The exception instance has variables: - - - test: the DocTest object being run - - - excample: the Example object that failed - - - exc_info: the exception info - """ - def __init__(self, test, example, exc_info): - self.test = test - self.example = example - self.exc_info = exc_info - - def __str__(self): - return str(self.test) - -class DebugRunner(DocTestRunner): - r"""Run doc tests but raise an exception as soon as there is a failure. - - If an unexpected exception occurs, an UnexpectedException is raised. - It contains the test, the example, and the original exception: - - >>> runner = DebugRunner(verbose=False) - >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42', - ... {}, 'foo', 'foo.py', 0) - >>> try: - ... runner.run(test) - ... except UnexpectedException, failure: - ... pass - - >>> failure.test is test - True - - >>> failure.example.want - '42\n' - - >>> exc_info = failure.exc_info - >>> raise exc_info[0], exc_info[1], exc_info[2] - Traceback (most recent call last): - ... - KeyError - - We wrap the original exception to give the calling application - access to the test and example information. - - If the output doesn't match, then a DocTestFailure is raised: - - >>> test = DocTestParser().get_doctest(''' - ... >>> x = 1 - ... >>> x - ... 2 - ... ''', {}, 'foo', 'foo.py', 0) - - >>> try: - ... runner.run(test) - ... except DocTestFailure, failure: - ... pass - - DocTestFailure objects provide access to the test: - - >>> failure.test is test - True - - As well as to the example: - - >>> failure.example.want - '2\n' - - and the actual output: - - >>> failure.got - '1\n' - - If a failure or error occurs, the globals are left intact: - - >>> del test.globs['__builtins__'] - >>> test.globs - {'x': 1} - - >>> test = DocTestParser().get_doctest(''' - ... >>> x = 2 - ... >>> raise KeyError - ... ''', {}, 'foo', 'foo.py', 0) - - >>> runner.run(test) - Traceback (most recent call last): - ... - UnexpectedException: <DocTest foo from foo.py:0 (2 examples)> - - >>> del test.globs['__builtins__'] - >>> test.globs - {'x': 2} - - But the globals are cleared if there is no error: - - >>> test = DocTestParser().get_doctest(''' - ... >>> x = 2 - ... ''', {}, 'foo', 'foo.py', 0) - - >>> runner.run(test) - (0, 1) - - >>> test.globs - {} - - """ - - def run(self, test, compileflags=None, out=None, clear_globs=True): - r = DocTestRunner.run(self, test, compileflags, out, False) - if clear_globs: - test.globs.clear() - return r - - def report_unexpected_exception(self, out, test, example, exc_info): - raise UnexpectedException(test, example, exc_info) - - def report_failure(self, out, test, example, got): - raise DocTestFailure(test, example, got) - -###################################################################### -## 6. Test Functions -###################################################################### -# These should be backwards compatible. - -# For backward compatibility, a global instance of a DocTestRunner -# class, updated by testmod. -master = None - -def testmod(m=None, name=None, globs=None, verbose=None, isprivate=None, - report=True, optionflags=0, extraglobs=None, - raise_on_error=False, exclude_empty=False): - """m=None, name=None, globs=None, verbose=None, isprivate=None, - report=True, optionflags=0, extraglobs=None, raise_on_error=False, - exclude_empty=False - - Test examples in docstrings in functions and classes reachable - from module m (or the current module if m is not supplied), starting - with m.__doc__. Unless isprivate is specified, private names - are not skipped. - - Also test examples reachable from dict m.__test__ if it exists and is - not None. m.__test__ maps names to functions, classes and strings; - function and class docstrings are tested even if the name is private; - strings are tested directly, as if they were docstrings. - - Return (#failures, #tests). - - See doctest.__doc__ for an overview. - - Optional keyword arg "name" gives the name of the module; by default - use m.__name__. - - Optional keyword arg "globs" gives a dict to be used as the globals - when executing examples; by default, use m.__dict__. A copy of this - dict is actually used for each docstring, so that each docstring's - examples start with a clean slate. - - Optional keyword arg "extraglobs" gives a dictionary that should be - merged into the globals that are used to execute examples. By - default, no extra globals are used. This is new in 2.4. - - Optional keyword arg "verbose" prints lots of stuff if true, prints - only failures if false; by default, it's true iff "-v" is in sys.argv. - - Optional keyword arg "report" prints a summary at the end when true, - else prints nothing at the end. In verbose mode, the summary is - detailed, else very brief (in fact, empty if all tests passed). - - Optional keyword arg "optionflags" or's together module constants, - and defaults to 0. This is new in 2.3. Possible values (see the - docs for details): - - DONT_ACCEPT_TRUE_FOR_1 - DONT_ACCEPT_BLANKLINE - NORMALIZE_WHITESPACE - ELLIPSIS - IGNORE_EXCEPTION_DETAIL - REPORT_UDIFF - REPORT_CDIFF - REPORT_NDIFF - REPORT_ONLY_FIRST_FAILURE - - Optional keyword arg "raise_on_error" raises an exception on the - first unexpected exception or failure. This allows failures to be - post-mortem debugged. - - Deprecated in Python 2.4: - Optional keyword arg "isprivate" specifies a function used to - determine whether a name is private. The default function is - treat all functions as public. Optionally, "isprivate" can be - set to doctest.is_private to skip over functions marked as private - using the underscore naming convention; see its docs for details. - - Advanced tomfoolery: testmod runs methods of a local instance of - class doctest.Tester, then merges the results into (or creates) - global Tester instance doctest.master. Methods of doctest.master - can be called directly too, if you want to do something unusual. - Passing report=0 to testmod is especially useful then, to delay - displaying a summary. Invoke doctest.master.summarize(verbose) - when you're done fiddling. - """ - global master - - if isprivate is not None: - warnings.warn("the isprivate argument is deprecated; " - "examine DocTestFinder.find() lists instead", - DeprecationWarning) - - # If no module was given, then use __main__. - if m is None: - # DWA - m will still be None if this wasn't invoked from the command - # line, in which case the following TypeError is about as good an error - # as we should expect - m = sys.modules.get('__main__') - - # Check that we were actually given a module. - if not inspect.ismodule(m): - raise TypeError("testmod: module required; %r" % (m,)) - - # If no name was given, then use the module's name. - if name is None: - name = m.__name__ - - # Find, parse, and run all tests in the given module. - finder = DocTestFinder(_namefilter=isprivate, exclude_empty=exclude_empty) - - if raise_on_error: - runner = DebugRunner(verbose=verbose, optionflags=optionflags) - else: - runner = DocTestRunner(verbose=verbose, optionflags=optionflags) - - for test in finder.find(m, name, globs=globs, extraglobs=extraglobs): - runner.run(test) - - if report: - runner.summarize() - - if master is None: - master = runner - else: - master.merge(runner) - - return runner.failures, runner.tries - -def testfile(filename, module_relative=True, name=None, package=None, - globs=None, verbose=None, report=True, optionflags=0, - extraglobs=None, raise_on_error=False, parser=DocTestParser()): - """ - Test examples in the given file. Return (#failures, #tests). - - Optional keyword arg "module_relative" specifies how filenames - should be interpreted: - - - If "module_relative" is True (the default), then "filename" - specifies a module-relative path. By default, this path is - relative to the calling module's directory; but if the - "package" argument is specified, then it is relative to that - package. To ensure os-independence, "filename" should use - "/" characters to separate path segments, and should not - be an absolute path (i.e., it may not begin with "/"). - - - If "module_relative" is False, then "filename" specifies an - os-specific path. The path may be absolute or relative (to - the current working directory). - - Optional keyword arg "name" gives the name of the test; by default - use the file's basename. - - Optional keyword argument "package" is a Python package or the - name of a Python package whose directory should be used as the - base directory for a module relative filename. If no package is - specified, then the calling module's directory is used as the base - directory for module relative filenames. It is an error to - specify "package" if "module_relative" is False. - - Optional keyword arg "globs" gives a dict to be used as the globals - when executing examples; by default, use {}. A copy of this dict - is actually used for each docstring, so that each docstring's - examples start with a clean slate. - - Optional keyword arg "extraglobs" gives a dictionary that should be - merged into the globals that are used to execute examples. By - default, no extra globals are used. - - Optional keyword arg "verbose" prints lots of stuff if true, prints - only failures if false; by default, it's true iff "-v" is in sys.argv. - - Optional keyword arg "report" prints a summary at the end when true, - else prints nothing at the end. In verbose mode, the summary is - detailed, else very brief (in fact, empty if all tests passed). - - Optional keyword arg "optionflags" or's together module constants, - and defaults to 0. Possible values (see the docs for details): - - DONT_ACCEPT_TRUE_FOR_1 - DONT_ACCEPT_BLANKLINE - NORMALIZE_WHITESPACE - ELLIPSIS - IGNORE_EXCEPTION_DETAIL - REPORT_UDIFF - REPORT_CDIFF - REPORT_NDIFF - REPORT_ONLY_FIRST_FAILURE - - Optional keyword arg "raise_on_error" raises an exception on the - first unexpected exception or failure. This allows failures to be - post-mortem debugged. - - Optional keyword arg "parser" specifies a DocTestParser (or - subclass) that should be used to extract tests from the files. - - Advanced tomfoolery: testmod runs methods of a local instance of - class doctest.Tester, then merges the results into (or creates) - global Tester instance doctest.master. Methods of doctest.master - can be called directly too, if you want to do something unusual. - Passing report=0 to testmod is especially useful then, to delay - displaying a summary. Invoke doctest.master.summarize(verbose) - when you're done fiddling. - """ - global master - - if package and not module_relative: - raise ValueError("Package may only be specified for module-" - "relative paths.") - - # Relativize the path - if module_relative: - package = _normalize_module(package) - filename = _module_relative_path(package, filename) - - # If no name was given, then use the file's name. - if name is None: - name = os.path.basename(filename) - - # Assemble the globals. - if globs is None: - globs = {} - else: - globs = globs.copy() - if extraglobs is not None: - globs.update(extraglobs) - - if raise_on_error: - runner = DebugRunner(verbose=verbose, optionflags=optionflags) - else: - runner = DocTestRunner(verbose=verbose, optionflags=optionflags) - - # Read the file, convert it to a test, and run it. - s = open(filename).read() - test = parser.get_doctest(s, globs, name, filename, 0) - runner.run(test) - - if report: - runner.summarize() - - if master is None: - master = runner - else: - master.merge(runner) - - return runner.failures, runner.tries - -def run_docstring_examples(f, globs, verbose=False, name="NoName", - compileflags=None, optionflags=0): - """ - Test examples in the given object's docstring (`f`), using `globs` - as globals. Optional argument `name` is used in failure messages. - If the optional argument `verbose` is true, then generate output - even if there are no failures. - - `compileflags` gives the set of flags that should be used by the - Python compiler when running the examples. If not specified, then - it will default to the set of future-import flags that apply to - `globs`. - - Optional keyword arg `optionflags` specifies options for the - testing and output. See the documentation for `testmod` for more - information. - """ - # Find, parse, and run all tests in the given module. - finder = DocTestFinder(verbose=verbose, recurse=False) - runner = DocTestRunner(verbose=verbose, optionflags=optionflags) - for test in finder.find(f, name, globs=globs): - runner.run(test, compileflags=compileflags) - -###################################################################### -## 7. Tester -###################################################################### -# This is provided only for backwards compatibility. It's not -# actually used in any way. - -class Tester: - def __init__(self, mod=None, globs=None, verbose=None, - isprivate=None, optionflags=0): - - warnings.warn("class Tester is deprecated; " - "use class doctest.DocTestRunner instead", - DeprecationWarning, stacklevel=2) - if mod is None and globs is None: - raise TypeError("Tester.__init__: must specify mod or globs") - if mod is not None and not inspect.ismodule(mod): - raise TypeError("Tester.__init__: mod must be a module; %r" % - (mod,)) - if globs is None: - globs = mod.__dict__ - self.globs = globs - - self.verbose = verbose - self.isprivate = isprivate - self.optionflags = optionflags - self.testfinder = DocTestFinder(_namefilter=isprivate) - self.testrunner = DocTestRunner(verbose=verbose, - optionflags=optionflags) - - def runstring(self, s, name): - test = DocTestParser().get_doctest(s, self.globs, name, None, None) - if self.verbose: - print "Running string", name - (f,t) = self.testrunner.run(test) - if self.verbose: - print f, "of", t, "examples failed in string", name - return (f,t) - - def rundoc(self, object, name=None, module=None): - f = t = 0 - tests = self.testfinder.find(object, name, module=module, - globs=self.globs) - for test in tests: - (f2, t2) = self.testrunner.run(test) - (f,t) = (f+f2, t+t2) - return (f,t) - - def rundict(self, d, name, module=None): - import new - m = new.module(name) - m.__dict__.update(d) - if module is None: - module = False - return self.rundoc(m, name, module) - - def run__test__(self, d, name): - import new - m = new.module(name) - m.__test__ = d - return self.rundoc(m, name) - - def summarize(self, verbose=None): - return self.testrunner.summarize(verbose) - - def merge(self, other): - self.testrunner.merge(other.testrunner) - -###################################################################### -## 8. Unittest Support -###################################################################### - -_unittest_reportflags = 0 - -def set_unittest_reportflags(flags): - """Sets the unittest option flags. - - The old flag is returned so that a runner could restore the old - value if it wished to: - - >>> old = _unittest_reportflags - >>> set_unittest_reportflags(REPORT_NDIFF | - ... REPORT_ONLY_FIRST_FAILURE) == old - True - - >>> import doctest - >>> doctest._unittest_reportflags == (REPORT_NDIFF | - ... REPORT_ONLY_FIRST_FAILURE) - True - - Only reporting flags can be set: - - >>> set_unittest_reportflags(ELLIPSIS) - Traceback (most recent call last): - ... - ValueError: ('Only reporting flags allowed', 8) - - >>> set_unittest_reportflags(old) == (REPORT_NDIFF | - ... REPORT_ONLY_FIRST_FAILURE) - True - """ - global _unittest_reportflags - - if (flags & REPORTING_FLAGS) != flags: - raise ValueError("Only reporting flags allowed", flags) - old = _unittest_reportflags - _unittest_reportflags = flags - return old - - -class DocTestCase(unittest.TestCase): - - def __init__(self, test, optionflags=0, setUp=None, tearDown=None, - checker=None): - - unittest.TestCase.__init__(self) - self._dt_optionflags = optionflags - self._dt_checker = checker - self._dt_test = test - self._dt_setUp = setUp - self._dt_tearDown = tearDown - - def setUp(self): - test = self._dt_test - - if self._dt_setUp is not None: - self._dt_setUp(test) - - def tearDown(self): - test = self._dt_test - - if self._dt_tearDown is not None: - self._dt_tearDown(test) - - test.globs.clear() - - def runTest(self): - test = self._dt_test - old = sys.stdout - new = StringIO() - optionflags = self._dt_optionflags - - if not (optionflags & REPORTING_FLAGS): - # The option flags don't include any reporting flags, - # so add the default reporting flags - optionflags |= _unittest_reportflags - - runner = DocTestRunner(optionflags=optionflags, - checker=self._dt_checker, verbose=False) - - try: - runner.DIVIDER = "-"*70 - failures, tries = runner.run( - test, out=new.write, clear_globs=False) - finally: - sys.stdout = old - - if failures: - raise self.failureException(self.format_failure(new.getvalue())) - - def format_failure(self, err): - test = self._dt_test - if test.lineno is None: - lineno = 'unknown line number' - else: - lineno = '%s' % test.lineno - lname = '.'.join(test.name.split('.')[-1:]) - return ('Failed doctest test for %s\n' - ' File "%s", line %s, in %s\n\n%s' - % (test.name, test.filename, lineno, lname, err) - ) - - def debug(self): - r"""Run the test case without results and without catching exceptions - - The unit test framework includes a debug method on test cases - and test suites to support post-mortem debugging. The test code - is run in such a way that errors are not caught. This way a - caller can catch the errors and initiate post-mortem debugging. - - The DocTestCase provides a debug method that raises - UnexpectedException errors if there is an unexepcted - exception: - - >>> test = DocTestParser().get_doctest('>>> raise KeyError\n42', - ... {}, 'foo', 'foo.py', 0) - >>> case = DocTestCase(test) - >>> try: - ... case.debug() - ... except UnexpectedException, failure: - ... pass - - The UnexpectedException contains the test, the example, and - the original exception: - - >>> failure.test is test - True - - >>> failure.example.want - '42\n' - - >>> exc_info = failure.exc_info - >>> raise exc_info[0], exc_info[1], exc_info[2] - Traceback (most recent call last): - ... - KeyError - - If the output doesn't match, then a DocTestFailure is raised: - - >>> test = DocTestParser().get_doctest(''' - ... >>> x = 1 - ... >>> x - ... 2 - ... ''', {}, 'foo', 'foo.py', 0) - >>> case = DocTestCase(test) - - >>> try: - ... case.debug() - ... except DocTestFailure, failure: - ... pass - - DocTestFailure objects provide access to the test: - - >>> failure.test is test - True - - As well as to the example: - - >>> failure.example.want - '2\n' - - and the actual output: - - >>> failure.got - '1\n' - - """ - - self.setUp() - runner = DebugRunner(optionflags=self._dt_optionflags, - checker=self._dt_checker, verbose=False) - runner.run(self._dt_test) - self.tearDown() - - def id(self): - return self._dt_test.name - - def __repr__(self): - name = self._dt_test.name.split('.') - return "%s (%s)" % (name[-1], '.'.join(name[:-1])) - - __str__ = __repr__ - - def shortDescription(self): - return "Doctest: " + self._dt_test.name - -def DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None, - **options): - """ - Convert doctest tests for a module to a unittest test suite. - - This converts each documentation string in a module that - contains doctest tests to a unittest test case. If any of the - tests in a doc string fail, then the test case fails. An exception - is raised showing the name of the file containing the test and a - (sometimes approximate) line number. - - The `module` argument provides the module to be tested. The argument - can be either a module or a module name. - - If no argument is given, the calling module is used. - - A number of options may be provided as keyword arguments: - - setUp - A set-up function. This is called before running the - tests in each file. The setUp function will be passed a DocTest - object. The setUp function can access the test globals as the - globs attribute of the test passed. - - tearDown - A tear-down function. This is called after running the - tests in each file. The tearDown function will be passed a DocTest - object. The tearDown function can access the test globals as the - globs attribute of the test passed. - - globs - A dictionary containing initial global variables for the tests. - - optionflags - A set of doctest option flags expressed as an integer. - """ - - if test_finder is None: - test_finder = DocTestFinder() - - module = _normalize_module(module) - tests = test_finder.find(module, globs=globs, extraglobs=extraglobs) - if globs is None: - globs = module.__dict__ - if not tests: - # Why do we want to do this? Because it reveals a bug that might - # otherwise be hidden. - raise ValueError(module, "has no tests") - - tests.sort() - suite = unittest.TestSuite() - for test in tests: - if len(test.examples) == 0: - continue - if not test.filename: - filename = module.__file__ - if filename[-4:] in (".pyc", ".pyo"): - filename = filename[:-1] - test.filename = filename - suite.addTest(DocTestCase(test, **options)) - - return suite - -class DocFileCase(DocTestCase): - - def id(self): - return '_'.join(self._dt_test.name.split('.')) - - def __repr__(self): - return self._dt_test.filename - __str__ = __repr__ - - def format_failure(self, err): - return ('Failed doctest test for %s\n File "%s", line 0\n\n%s' - % (self._dt_test.name, self._dt_test.filename, err) - ) - -def DocFileTest(path, module_relative=True, package=None, - globs=None, parser=DocTestParser(), **options): - if globs is None: - globs = {} - - if package and not module_relative: - raise ValueError("Package may only be specified for module-" - "relative paths.") - - # Relativize the path. - if module_relative: - package = _normalize_module(package) - path = _module_relative_path(package, path) - - # Find the file and read it. - name = os.path.basename(path) - doc = open(path).read() - - # Convert it to a test, and wrap it in a DocFileCase. - test = parser.get_doctest(doc, globs, name, path, 0) - return DocFileCase(test, **options) - -def DocFileSuite(*paths, **kw): - """A unittest suite for one or more doctest files. - - The path to each doctest file is given as a string; the - interpretation of that string depends on the keyword argument - "module_relative". - - A number of options may be provided as keyword arguments: - - module_relative - If "module_relative" is True, then the given file paths are - interpreted as os-independent module-relative paths. By - default, these paths are relative to the calling module's - directory; but if the "package" argument is specified, then - they are relative to that package. To ensure os-independence, - "filename" should use "/" characters to separate path - segments, and may not be an absolute path (i.e., it may not - begin with "/"). - - If "module_relative" is False, then the given file paths are - interpreted as os-specific paths. These paths may be absolute - or relative (to the current working directory). - - package - A Python package or the name of a Python package whose directory - should be used as the base directory for module relative paths. - If "package" is not specified, then the calling module's - directory is used as the base directory for module relative - filenames. It is an error to specify "package" if - "module_relative" is False. - - setUp - A set-up function. This is called before running the - tests in each file. The setUp function will be passed a DocTest - object. The setUp function can access the test globals as the - globs attribute of the test passed. - - tearDown - A tear-down function. This is called after running the - tests in each file. The tearDown function will be passed a DocTest - object. The tearDown function can access the test globals as the - globs attribute of the test passed. - - globs - A dictionary containing initial global variables for the tests. - - optionflags - A set of doctest option flags expressed as an integer. - - parser - A DocTestParser (or subclass) that should be used to extract - tests from the files. - """ - suite = unittest.TestSuite() - - # We do this here so that _normalize_module is called at the right - # level. If it were called in DocFileTest, then this function - # would be the caller and we might guess the package incorrectly. - if kw.get('module_relative', True): - kw['package'] = _normalize_module(kw.get('package')) - - for path in paths: - suite.addTest(DocFileTest(path, **kw)) - - return suite - -###################################################################### -## 9. Debugging Support -###################################################################### - -def script_from_examples(s): - r"""Extract script from text with examples. - - Converts text with examples to a Python script. Example input is - converted to regular code. Example output and all other words - are converted to comments: - - >>> text = ''' - ... Here are examples of simple math. - ... - ... Python has super accurate integer addition - ... - ... >>> 2 + 2 - ... 5 - ... - ... And very friendly error messages: - ... - ... >>> 1/0 - ... To Infinity - ... And - ... Beyond - ... - ... You can use logic if you want: - ... - ... >>> if 0: - ... ... blah - ... ... blah - ... ... - ... - ... Ho hum - ... ''' - - >>> print script_from_examples(text) - # Here are examples of simple math. - # - # Python has super accurate integer addition - # - 2 + 2 - # Expected: - ## 5 - # - # And very friendly error messages: - # - 1/0 - # Expected: - ## To Infinity - ## And - ## Beyond - # - # You can use logic if you want: - # - if 0: - blah - blah - # - # Ho hum - """ - output = [] - for piece in DocTestParser().parse(s): - if isinstance(piece, Example): - # Add the example's source code (strip trailing NL) - output.append(piece.source[:-1]) - # Add the expected output: - want = piece.want - if want: - output.append('# Expected:') - output += ['## '+l for l in want.split('\n')[:-1]] - else: - # Add non-example text. - output += [_comment_line(l) - for l in piece.split('\n')[:-1]] - - # Trim junk on both ends. - while output and output[-1] == '#': - output.pop() - while output and output[0] == '#': - output.pop(0) - # Combine the output, and return it. - return '\n'.join(output) - -def testsource(module, name): - """Extract the test sources from a doctest docstring as a script. - - Provide the module (or dotted name of the module) containing the - test to be debugged and the name (within the module) of the object - with the doc string with tests to be debugged. - """ - module = _normalize_module(module) - tests = DocTestFinder().find(module) - test = [t for t in tests if t.name == name] - if not test: - raise ValueError(name, "not found in tests") - test = test[0] - testsrc = script_from_examples(test.docstring) - return testsrc - -def debug_src(src, pm=False, globs=None): - """Debug a single doctest docstring, in argument `src`'""" - testsrc = script_from_examples(src) - debug_script(testsrc, pm, globs) - -def debug_script(src, pm=False, globs=None): - "Debug a test script. `src` is the script, as a string." - import pdb - - # Note that tempfile.NameTemporaryFile() cannot be used. As the - # docs say, a file so created cannot be opened by name a second time - # on modern Windows boxes, and execfile() needs to open it. - srcfilename = tempfile.mktemp(".py", "doctestdebug") - f = open(srcfilename, 'w') - f.write(src) - f.close() - - try: - if globs: - globs = globs.copy() - else: - globs = {} - - if pm: - try: - execfile(srcfilename, globs, globs) - except: - print sys.exc_info()[1] - pdb.post_mortem(sys.exc_info()[2]) - else: - # Note that %r is vital here. '%s' instead can, e.g., cause - # backslashes to get treated as metacharacters on Windows. - pdb.run("execfile(%r)" % srcfilename, globs, globs) - - finally: - os.remove(srcfilename) - -def debug(module, name, pm=False): - """Debug a single doctest docstring. - - Provide the module (or dotted name of the module) containing the - test to be debugged and the name (within the module) of the object - with the docstring with tests to be debugged. - """ - module = _normalize_module(module) - testsrc = testsource(module, name) - debug_script(testsrc, pm, module.__dict__) - -###################################################################### -## 10. Example Usage -###################################################################### -class _TestClass: - """ - A pointless class, for sanity-checking of docstring testing. - - Methods: - square() - get() - - >>> _TestClass(13).get() + _TestClass(-12).get() - 1 - >>> hex(_TestClass(13).square().get()) - '0xa9' - """ - - def __init__(self, val): - """val -> _TestClass object with associated value val. - - >>> t = _TestClass(123) - >>> print t.get() - 123 - """ - - self.val = val - - def square(self): - """square() -> square TestClass's associated value - - >>> _TestClass(13).square().get() - 169 - """ - - self.val = self.val ** 2 - return self - - def get(self): - """get() -> return TestClass's associated value. - - >>> x = _TestClass(-42) - >>> print x.get() - -42 - """ - - return self.val - -__test__ = {"_TestClass": _TestClass, - "string": r""" - Example of a string object, searched as-is. - >>> x = 1; y = 2 - >>> x + y, x * y - (3, 2) - """, - - "bool-int equivalence": r""" - In 2.2, boolean expressions displayed - 0 or 1. By default, we still accept - them. This can be disabled by passing - DONT_ACCEPT_TRUE_FOR_1 to the new - optionflags argument. - >>> 4 == 4 - 1 - >>> 4 == 4 - True - >>> 4 > 4 - 0 - >>> 4 > 4 - False - """, - - "blank lines": r""" - Blank lines can be marked with <BLANKLINE>: - >>> print 'foo\n\nbar\n' - foo - <BLANKLINE> - bar - <BLANKLINE> - """, - - "ellipsis": r""" - If the ellipsis flag is used, then '...' can be used to - elide substrings in the desired output: - >>> print range(1000) #doctest: +ELLIPSIS - [0, 1, 2, ..., 999] - """, - - "whitespace normalization": r""" - If the whitespace normalization flag is used, then - differences in whitespace are ignored. - >>> print range(30) #doctest: +NORMALIZE_WHITESPACE - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, - 27, 28, 29] - """, - } - -def _test(): - r = unittest.TextTestRunner() - r.run(DocTestSuite()) - -if __name__ == "__main__": - _test() diff --git a/paste/util/filemixin.py b/paste/util/filemixin.py index 10a9e7c..b06b039 100644 --- a/paste/util/filemixin.py +++ b/paste/util/filemixin.py @@ -50,4 +50,4 @@ class FileMixin(object): for line in lines: self.write(line) - + diff --git a/paste/util/finddata.py b/paste/util/finddata.py index af29c04..bb7c760 100644 --- a/paste/util/finddata.py +++ b/paste/util/finddata.py @@ -4,6 +4,7 @@ # you can't import this from another package, when you don't know if # that package is installed yet. +from __future__ import print_function import os import sys from fnmatch import fnmatchcase @@ -47,7 +48,7 @@ def find_package_data( Note patterns use wildcards, or can be exact paths (including leading ``./``), and all searching is case-insensitive. """ - + out = {} stack = [(convert_path(where), '', package, only_in_packages)] while stack: @@ -61,9 +62,8 @@ def find_package_data( or fn.lower() == pattern.lower()): bad_name = True if show_ignored: - print >> sys.stderr, ( - "Directory %s ignored by pattern %s" - % (fn, pattern)) + print("Directory %s ignored by pattern %s" + % (fn, pattern), file=sys.stderr) break if bad_name: continue @@ -84,9 +84,8 @@ def find_package_data( or fn.lower() == pattern.lower()): bad_name = True if show_ignored: - print >> sys.stderr, ( - "File %s ignored by pattern %s" - % (fn, pattern)) + print("File %s ignored by pattern %s" + % (fn, pattern), file=sys.stderr) break if bad_name: continue diff --git a/paste/util/findpackage.py b/paste/util/findpackage.py index 68b5e8b..9d653e5 100644 --- a/paste/util/findpackage.py +++ b/paste/util/findpackage.py @@ -23,4 +23,4 @@ def find_package(dir): raise ValueError( "%s is not under any path found in sys.path" % orig_dir) last_dir = dir - + diff --git a/paste/util/import_string.py b/paste/util/import_string.py index 3feb4dd..a10db18 100644 --- a/paste/util/import_string.py +++ b/paste/util/import_string.py @@ -47,7 +47,7 @@ def simple_import(s): try: module = import_module(name) parts = parts[1:] - except ImportError, e: + except ImportError as e: last_import_error = e break obj = module @@ -79,7 +79,7 @@ def try_import_module(module_name): """ try: return import_module(module_name) - except ImportError, e: + except ImportError as e: if not getattr(e, 'args', None): raise desc = e.args[0] diff --git a/paste/util/intset.py b/paste/util/intset.py index 3873c75..3e026e2 100644 --- a/paste/util/intset.py +++ b/paste/util/intset.py @@ -6,6 +6,7 @@ Integer set class. Copyright (C) 2006, Heiko Wundram. Released under the MIT license. """ +import six # Version information # ------------------- @@ -28,40 +29,43 @@ class _Infinity(object): self._neg = neg def __lt__(self,value): - if not isinstance(value,(int,long,_Infinity)): + if not isinstance(value, _VALID_TYPES): return NotImplemented return ( self._neg and not ( isinstance(value,_Infinity) and value._neg ) ) def __le__(self,value): - if not isinstance(value,(int,long,_Infinity)): + if not isinstance(value, _VALID_TYPES): return NotImplemented return self._neg def __gt__(self,value): - if not isinstance(value,(int,long,_Infinity)): + if not isinstance(value, _VALID_TYPES): return NotImplemented return not ( self._neg or ( isinstance(value,_Infinity) and not value._neg ) ) def __ge__(self,value): - if not isinstance(value,(int,long,_Infinity)): + if not isinstance(value, _VALID_TYPES): return NotImplemented return not self._neg def __eq__(self,value): - if not isinstance(value,(int,long,_Infinity)): + if not isinstance(value, _VALID_TYPES): return NotImplemented return isinstance(value,_Infinity) and self._neg == value._neg def __ne__(self,value): - if not isinstance(value,(int,long,_Infinity)): + if not isinstance(value, _VALID_TYPES): return NotImplemented - return not isinstance(value,_Infinity) or self._neg <> value._neg + return not isinstance(value,_Infinity) or self._neg != value._neg def __repr__(self): return "None" +_VALID_TYPES = six.integer_types + (_Infinity,) + + # Constants # --------- @@ -117,22 +121,22 @@ class IntSet(object): # Check keyword arguments. if kwargs: raise ValueError("Invalid keyword argument.") - if not ( isinstance(self._min,(int,long)) or self._min is _MININF ): + if not ( isinstance(self._min, six.integer_types) or self._min is _MININF ): raise TypeError("Invalid type of min argument.") - if not ( isinstance(self._max,(int,long)) or self._max is _MAXINF ): + if not ( isinstance(self._max, six.integer_types) or self._max is _MAXINF ): raise TypeError("Invalid type of max argument.") if ( self._min is not _MININF and self._max is not _MAXINF and self._min > self._max ): raise ValueError("Minimum is not smaller than maximum.") - if isinstance(self._max,(int,long)): + if isinstance(self._max, six.integer_types): self._max += 1 # Process arguments. for arg in args: - if isinstance(arg,(int,long)): + if isinstance(arg, six.integer_types): start, stop = arg, arg+1 elif isinstance(arg,tuple): - if len(arg) <> 2: + if len(arg) != 2: raise ValueError("Invalid tuple, must be (start,stop).") # Process argument. @@ -143,14 +147,14 @@ class IntSet(object): stop = self._max # Check arguments. - if not ( isinstance(start,(int,long)) or start is _MININF ): + if not ( isinstance(start, six.integer_types) or start is _MININF ): raise TypeError("Invalid type of tuple start.") - if not ( isinstance(stop,(int,long)) or stop is _MAXINF ): + if not ( isinstance(stop, six.integer_types) or stop is _MAXINF ): raise TypeError("Invalid type of tuple stop.") if ( start is not _MININF and stop is not _MAXINF and start > stop ): continue - if isinstance(stop,(int,long)): + if isinstance(stop, six.integer_types): stop += 1 else: raise TypeError("Invalid argument.") @@ -211,7 +215,7 @@ class IntSet(object): def __coerce__(self,other): if isinstance(other,IntSet): return self, other - elif isinstance(other,(int,long,tuple)): + elif isinstance(other, six.integer_types + (tuple,)): try: return self, self.__class__(other) except TypeError: @@ -471,10 +475,10 @@ class IntSet(object): rv = [] for start, stop in self._ranges: - if ( isinstance(start,(int,long)) and isinstance(stop,(int,long)) + if ( isinstance(start, six.integer_types) and isinstance(stop, six.integer_types) and stop-start == 1 ): rv.append("%r" % start) - elif isinstance(stop,(int,long)): + elif isinstance(stop, six.integer_types): rv.append("(%r,%r)" % (start,stop-1)) else: rv.append("(%r,%r)" % (start,stop)) @@ -489,23 +493,23 @@ if __name__ == "__main__": x = IntSet((10,20),30) y = IntSet((10,20)) z = IntSet((10,20),30,(15,19),min=0,max=40) - print x - print x&110 - print x|110 - print x^(15,25) - print x-12 - print 12 in x - print x.issubset(x) - print y.issubset(x) - print x.istruesubset(x) - print y.istruesubset(x) + print(x) + print(x&110) + print(x|110) + print(x^(15,25)) + print(x-12) + print(12 in x) + print(x.issubset(x)) + print(y.issubset(x)) + print(x.istruesubset(x)) + print(y.istruesubset(x)) for val in x: - print val - print x.inverse() - print x == z - print x == y - print x <> y - print hash(x) - print hash(z) - print len(x) - print x.len() + print(val) + print(x.inverse()) + print(x == z) + print(x == y) + print(x != y) + print(hash(x)) + print(hash(z)) + print(len(x)) + print(x.len()) diff --git a/paste/util/ip4.py b/paste/util/ip4.py index b0dfde8..9ce17b8 100644 --- a/paste/util/ip4.py +++ b/paste/util/ip4.py @@ -19,8 +19,9 @@ __date__ = "2006-01-20" # Imports # ------- -import intset +from paste.util import intset import socket +import six # IP4Range class @@ -82,19 +83,19 @@ class IP4Range(intset.IntSet): # Type 1, 2 or 3. args[i] = self._parseAddrRange(argval) elif isinstance(argval,tuple): - if len(tuple) <> 2: + if len(tuple) != 2: raise ValueError("Tuple is of invalid length.") addr1, addr2 = argval if isinstance(addr1,str): addr1 = self._parseAddrRange(addr1)[0] - elif not isinstance(addr1,(int,long)): + elif not isinstance(addr1, six.integer_types): raise TypeError("Invalid argument.") if isinstance(addr2,str): addr2 = self._parseAddrRange(addr2)[1] - elif not isinstance(addr2,(int,long)): + elif not isinstance(addr2, six.integer_types): raise TypeError("Invalid argument.") args[i] = (addr1,addr2) - elif not isinstance(argval,(int,long)): + elif not isinstance(argval, six.integer_types): raise TypeError("Invalid argument.") # Initialize the integer set. @@ -142,7 +143,7 @@ class IP4Range(intset.IntSet): while (mask&1): mask >>= 1 masklen += 1 - if remaining+masklen <> 32: + if remaining+masklen != 32: raise ValueError("Mask isn't a proper host mask.") naddr1 = naddr & (((1<<masklen)-1)<<(32-masklen)) naddr2 = naddr1 + (1<<(32-masklen)) - 1 @@ -231,7 +232,7 @@ class IP4Range(intset.IntSet): return "%s(%s)" % (self.__class__.__name__,",".join(rv)) def _parseAddr(addr,lookup=True): - if lookup and addr.translate(IP4Range._UNITYTRANS, IP4Range._IPREMOVE): + if lookup and any(ch not in IP4Range._IPREMOVE for ch in addr): try: addr = socket.gethostbyname(addr) except socket.error: @@ -260,14 +261,14 @@ if __name__ == "__main__": # Little test script. x = IP4Range("172.22.162.250/24") y = IP4Range("172.22.162.250","172.22.163.250","172.22.163.253<->255") - print x + print(x) for val in x.itermasks(): - print val + print(val) for val in y.itermasks(): - print val + print(val) for val in (x|y).itermasks(): - print val + print(val) for val in (x^y).iterranges(): - print val + print(val) for val in x: - print val + print(val) diff --git a/paste/util/killthread.py b/paste/util/killthread.py index f1fc93f..4df4f42 100644 --- a/paste/util/killthread.py +++ b/paste/util/killthread.py @@ -1,7 +1,7 @@ """ Kill a thread, from http://sebulba.wikispaces.com/recipe+thread2 """ -import types +import six try: import ctypes except ImportError: @@ -16,7 +16,7 @@ def async_raise(tid, exctype): tid is the value given by thread.get_ident() (an integer). Raise SystemExit to kill a thread.""" - if not isinstance(exctype, (types.ClassType, type)): + if not isinstance(exctype, (six.class_types, type)): raise TypeError("Only types can be raised (not instances)") if not isinstance(tid, int): raise TypeError("tid must be an integer") diff --git a/paste/util/looper.py b/paste/util/looper.py index 03a6b42..b56358a 100644 --- a/paste/util/looper.py +++ b/paste/util/looper.py @@ -7,9 +7,9 @@ These can be awkward to manage in a normal Python loop, but using the looper you can get a better sense of the context. Use like:: >>> for loop, item in looper(['a', 'b', 'c']): - ... print loop.number, item + ... print("%s %s" % (loop.number, item)) ... if not loop.last: - ... print '---' + ... print('---') 1 a --- 2 b @@ -20,12 +20,15 @@ looper you can get a better sense of the context. Use like:: __all__ = ['looper'] +import six + + class looper(object): """ Helper for looping (particularly in templates) - + Use this like:: - + for loop, item in looper(seq): if loop.first: ... @@ -56,6 +59,7 @@ class looper_iter(object): result = loop_pos(self.seq, self.pos), self.seq[self.pos] self.pos += 1 return result + __next__ = next class loop_pos(object): @@ -65,7 +69,7 @@ class loop_pos(object): def __repr__(self): return '<loop pos=%r at %r>' % ( - self.seq[pos], pos) + self.seq[self.pos], self.pos) def index(self): return self.pos @@ -137,7 +141,7 @@ class loop_pos(object): def _compare_group(self, item, other, getter): if getter is None: return item != other - elif (isinstance(getter, basestring) + elif (isinstance(getter, (six.binary_type, six.text_type)) and getter.startswith('.')): getter = getter[1:] if getter.endswith('()'): @@ -149,4 +153,4 @@ class loop_pos(object): return getter(item) != getter(other) else: return item[getter] != other[getter] - + diff --git a/paste/util/mimeparse.py b/paste/util/mimeparse.py index fe699f7..b796c8b 100644 --- a/paste/util/mimeparse.py +++ b/paste/util/mimeparse.py @@ -132,7 +132,7 @@ def best_match(supported, header): """ if not supported: return '' - parsed_header = map(parse_media_range, header.split(',')) + parsed_header = list(map(parse_media_range, header.split(','))) best_type = max([ (fitness_and_quality_parsed(mime_type, parsed_header), -n) for n, mime_type in enumerate(supported)]) @@ -154,7 +154,7 @@ def desired_matches(desired, header): >>> desired_matches(['text/html', 'application/xml'], 'application/xml,application/json') ['application/xml'] """ - parsed_ranges = map(parse_media_range, header.split(',')) + parsed_ranges = list(map(parse_media_range, header.split(','))) return [mimetype for mimetype in desired if quality_parsed(mimetype, parsed_ranges)] diff --git a/paste/util/multidict.py b/paste/util/multidict.py index d3eb1e9..701d1ac 100644 --- a/paste/util/multidict.py +++ b/paste/util/multidict.py @@ -2,8 +2,15 @@ # Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php import cgi import copy +import six import sys -from UserDict import DictMixin + +try: + # Python 3 + from collections import MutableMapping as DictMixin +except ImportError: + # Python 2 + from UserDict import DictMixin class MultiDict(DictMixin): @@ -19,15 +26,15 @@ class MultiDict(DictMixin): "MultiDict can only be called with one positional argument") if args: if hasattr(args[0], 'iteritems'): - items = list(args[0].iteritems()) + items = args[0].iteritems() elif hasattr(args[0], 'items'): items = args[0].items() else: - items = list(args[0]) - self._items = items + items = args[0] + self._items = list(items) else: self._items = [] - self._items.extend(kw.iteritems()) + self._items.extend(six.iteritems(kw)) def __getitem__(self, key): for k, v in self._items: @@ -54,7 +61,7 @@ class MultiDict(DictMixin): """ result = [] for k, v in self._items: - if key == k: + if type(key) == type(k) and key == k: result.append(v) return result @@ -110,7 +117,7 @@ class MultiDict(DictMixin): items = self._items found = False for i in range(len(items)-1, -1, -1): - if items[i][0] == key: + if type(items[i][0]) == type(key) and items[i][0] == key: del items[i] found = True if not found: @@ -118,7 +125,7 @@ class MultiDict(DictMixin): def __contains__(self, key): for k, v in self._items: - if k == key: + if type(k) == type(key) and k == key: return True return False @@ -139,10 +146,10 @@ class MultiDict(DictMixin): def pop(self, key, *args): if len(args) > 1: - raise TypeError, "pop expected at most 2 arguments, got "\ - + repr(1 + len(args)) + raise TypeError("pop expected at most 2 arguments, got " + + repr(1 + len(args))) for i in range(len(self._items)): - if self._items[i][0] == key: + if type(self._items[i][0]) == type(key) and self._items[i][0] == key: v = self._items[i][1] del self._items[i] return v @@ -226,6 +233,20 @@ class UnicodeMultiDict(DictMixin): self.encoding = encoding self.errors = errors self.decode_keys = decode_keys + if self.decode_keys: + items = self.multi._items + for index, item in enumerate(items): + key, value = item + key = self._encode_key(key) + items[index] = (key, value) + + def _encode_key(self, key): + if self.decode_keys: + try: + key = key.encode(self.encoding, self.errors) + except AttributeError: + pass + return key def _decode_key(self, key): if self.decode_keys: @@ -245,9 +266,10 @@ class UnicodeMultiDict(DictMixin): if isinstance(value, cgi.FieldStorage): # decode FieldStorage's field name and filename value = copy.copy(value) - if self.decode_keys: + if self.decode_keys and isinstance(value.name, six.binary_type): value.name = value.name.decode(self.encoding, self.errors) - value.filename = value.filename.decode(self.encoding, self.errors) + if six.PY2: + value.filename = value.filename.decode(self.encoding, self.errors) else: try: value = value.decode(self.encoding, self.errors) @@ -256,21 +278,25 @@ class UnicodeMultiDict(DictMixin): return value def __getitem__(self, key): + key = self._encode_key(key) return self._decode_value(self.multi.__getitem__(key)) def __setitem__(self, key, value): + key = self._encode_key(key) self.multi.__setitem__(key, value) def add(self, key, value): """ Add the key and value, not overwriting any previous value. """ + key = self._encode_key(key) self.multi.add(key, value) def getall(self, key): """ Return a list of all values matching the key (may be an empty list) """ + key = self._encode_key(key) return [self._decode_value(v) for v in self.multi.getall(key)] def getone(self, key): @@ -278,6 +304,7 @@ class UnicodeMultiDict(DictMixin): Get one value matching the key, raising a KeyError if multiple values were found. """ + key = self._encode_key(key) return self._decode_value(self.multi.getone(key)) def mixed(self): @@ -289,7 +316,7 @@ class UnicodeMultiDict(DictMixin): request. """ unicode_mixed = {} - for key, value in self.multi.mixed().iteritems(): + for key, value in six.iteritems(self.multi.mixed()): if isinstance(value, list): value = [self._decode_value(value) for value in value] else: @@ -303,15 +330,17 @@ class UnicodeMultiDict(DictMixin): list of values. """ unicode_dict = {} - for key, value in self.multi.dict_of_lists().iteritems(): + for key, value in six.iteritems(self.multi.dict_of_lists()): value = [self._decode_value(value) for value in value] unicode_dict[self._decode_key(key)] = value return unicode_dict def __delitem__(self, key): + key = self._encode_key(key) self.multi.__delitem__(key) def __contains__(self, key): + key = self._encode_key(key) return self.multi.__contains__(key) has_key = __contains__ @@ -320,12 +349,15 @@ class UnicodeMultiDict(DictMixin): self.multi.clear() def copy(self): - return UnicodeMultiDict(self.multi.copy(), self.encoding, self.errors) + return UnicodeMultiDict(self.multi.copy(), self.encoding, self.errors, + decode_keys=self.decode_keys) def setdefault(self, key, default=None): + key = self._encode_key(key) return self._decode_value(self.multi.setdefault(key, default)) def pop(self, key, *args): + key = self._encode_key(key) return self._decode_value(self.multi.pop(key, *args)) def popitem(self): @@ -354,10 +386,10 @@ class UnicodeMultiDict(DictMixin): def items(self): return [(self._decode_key(k), self._decode_value(v)) for \ - k, v in self.multi.iteritems()] + k, v in six.iteritems(self.multi)] def iteritems(self): - for k, v in self.multi.iteritems(): + for k, v in six.iteritems(self.multi): yield (self._decode_key(k), self._decode_value(v)) def values(self): diff --git a/paste/util/quoting.py b/paste/util/quoting.py index 6184752..df0d9da 100644 --- a/paste/util/quoting.py +++ b/paste/util/quoting.py @@ -2,9 +2,11 @@ # Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php import cgi -import htmlentitydefs -import urllib +import six import re +from six.moves import html_entities +from six.moves.urllib.parse import quote, unquote + __all__ = ['html_quote', 'html_unquote', 'url_quote', 'url_unquote', 'strip_html'] @@ -15,31 +17,28 @@ def html_quote(v, encoding=None): r""" Quote the value (turned to a string) as HTML. This quotes <, >, and quotes: - - >>> html_quote(1) - '1' - >>> html_quote(None) - '' - >>> html_quote('<hey!>') - '<hey!>' - >>> html_quote(u'\u1029') - '\xe1\x80\xa9' """ encoding = encoding or default_encoding if v is None: return '' - elif isinstance(v, str): + elif isinstance(v, six.binary_type): return cgi.escape(v, 1) - elif isinstance(v, unicode): - return cgi.escape(v.encode(encoding), 1) + elif isinstance(v, six.text_type): + if six.PY3: + return cgi.escape(v, 1) + else: + return cgi.escape(v.encode(encoding), 1) else: - return cgi.escape(unicode(v).encode(encoding), 1) + if six.PY3: + return cgi.escape(six.text_type(v), 1) + else: + return cgi.escape(six.text_type(v).encode(encoding), 1) _unquote_re = re.compile(r'&([a-zA-Z]+);') -def _entity_subber(match, name2c=htmlentitydefs.name2codepoint): +def _entity_subber(match, name2c=html_entities.name2codepoint): code = name2c.get(match.group(1)) if code: - return unichr(code) + return six.unichr(code) else: return match.group(0) @@ -47,20 +46,8 @@ def html_unquote(s, encoding=None): r""" Decode the value. - >>> html_unquote('<hey you>') - u'<hey\xa0you>' - >>> html_unquote('') - u'' - >>> html_unquote('&blahblah;') - u'&blahblah;' - >>> html_unquote('\xe1\x80\xa9') - u'\u1029' """ - if isinstance(s, str): - if s == '': - # workaround re.sub('', '', u'') returning '' < 2.5.2 - # instead of u'' >= 2.5.2 - return u'' + if isinstance(s, six.binary_type): s = s.decode(encoding or default_encoding) return _unquote_re.sub(_entity_subber, s) @@ -85,13 +72,13 @@ def comment_quote(s): """ comment = str(s) #comment = _bad_chars_re.sub('', comment) - #print 'in ', repr(str(s)) - #print 'out', repr(comment) + #print('in ', repr(str(s))) + #print('out', repr(comment)) comment = _comment_quote_re.sub('->', comment) return comment -url_quote = urllib.quote -url_unquote = urllib.unquote +url_quote = quote +url_unquote = unquote if __name__ == '__main__': import doctest diff --git a/paste/util/scgiserver.py b/paste/util/scgiserver.py index d20c952..1c86c86 100644 --- a/paste/util/scgiserver.py +++ b/paste/util/scgiserver.py @@ -25,6 +25,7 @@ Kudos to the WSGI folk for writing a nice PEP & the Quixote folk for writing a nice extensible SCGI server for Python! """ +import six import sys import time from scgi import scgi_server @@ -41,7 +42,7 @@ class SWAP(scgi_server.SCGIHandler): """ app_obj = None prefix = None - + def __init__(self, *args, **kwargs): assert self.app_obj, "must set app_obj" assert self.prefix is not None, "must set prefix" @@ -84,13 +85,13 @@ class SWAP(scgi_server.SCGIHandler): chunks = [] def write(data): chunks.append(data) - + def start_response(status, response_headers, exc_info=None): if exc_info: try: if headers_sent: # Re-raise original exception if headers sent - raise exc_info[0], exc_info[1], exc_info[2] + six.reraise(exc_info[0], exc_info[1], exc_info[2]) finally: exc_info = None # avoid dangling circular ref elif headers_set: @@ -105,7 +106,7 @@ class SWAP(scgi_server.SCGIHandler): try: for data in result: chunks.append(data) - + # Before the first output, send the stored headers if not headers_set: # Error -- the app never called start_response @@ -114,7 +115,7 @@ class SWAP(scgi_server.SCGIHandler): chunks = ["XXX start_response never called"] else: status, response_headers = headers_sent[:] = headers_set - + output.write('Status: %s\r\n' % status) for header in response_headers: output.write('%s: %s\r\n' % header) @@ -131,7 +132,7 @@ class SWAP(scgi_server.SCGIHandler): input.close() output.close() conn.close() - except IOError, err: + except IOError as err: debug("IOError while closing connection ignored: %s" % err) diff --git a/paste/util/string24.py b/paste/util/string24.py deleted file mode 100644 index 7c0e001..0000000 --- a/paste/util/string24.py +++ /dev/null @@ -1,531 +0,0 @@ -"""A collection of string operations (most are no longer used). - -Warning: most of the code you see here isn't normally used nowadays. -Beginning with Python 1.6, many of these functions are implemented as -methods on the standard string object. They used to be implemented by -a built-in module called strop, but strop is now obsolete itself. - -Public module variables: - -whitespace -- a string containing all characters considered whitespace -lowercase -- a string containing all characters considered lowercase letters -uppercase -- a string containing all characters considered uppercase letters -letters -- a string containing all characters considered letters -digits -- a string containing all characters considered decimal digits -hexdigits -- a string containing all characters considered hexadecimal digits -octdigits -- a string containing all characters considered octal digits -punctuation -- a string containing all characters considered punctuation -printable -- a string containing all characters considered printable - -""" - -# Some strings for ctype-style character classification -whitespace = ' \t\n\r\v\f' -lowercase = 'abcdefghijklmnopqrstuvwxyz' -uppercase = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' -letters = lowercase + uppercase -ascii_lowercase = lowercase -ascii_uppercase = uppercase -ascii_letters = ascii_lowercase + ascii_uppercase -digits = '0123456789' -hexdigits = digits + 'abcdef' + 'ABCDEF' -octdigits = '01234567' -punctuation = """!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~""" -printable = digits + letters + punctuation + whitespace - -# Case conversion helpers -# Use str to convert Unicode literal in case of -U -# Note that Cookie.py bogusly uses _idmap :( -l = map(chr, xrange(256)) -_idmap = str('').join(l) -del l - -# Functions which aren't available as string methods. - -# Capitalize the words in a string, e.g. " aBc dEf " -> "Abc Def". -# See also regsub.capwords(). -def capwords(s, sep=None): - """capwords(s, [sep]) -> string - - Split the argument into words using split, capitalize each - word using capitalize, and join the capitalized words using - join. Note that this replaces runs of whitespace characters by - a single space. - - """ - return (sep or ' ').join([x.capitalize() for x in s.split(sep)]) - - -# Construct a translation string -_idmapL = None -def maketrans(fromstr, tostr): - """maketrans(frm, to) -> string - - Return a translation table (a string of 256 bytes long) - suitable for use in string.translate. The strings frm and to - must be of the same length. - - """ - if len(fromstr) != len(tostr): - raise ValueError, "maketrans arguments must have same length" - global _idmapL - if not _idmapL: - _idmapL = map(None, _idmap) - L = _idmapL[:] - fromstr = map(ord, fromstr) - for i in range(len(fromstr)): - L[fromstr[i]] = tostr[i] - return ''.join(L) - - - -#################################################################### -import re as _re - -class _multimap: - """Helper class for combining multiple mappings. - - Used by .{safe_,}substitute() to combine the mapping and keyword - arguments. - """ - def __init__(self, primary, secondary): - self._primary = primary - self._secondary = secondary - - def __getitem__(self, key): - try: - return self._primary[key] - except KeyError: - return self._secondary[key] - - -class _TemplateMetaclass(type): - pattern = r""" - %(delim)s(?: - (?P<escaped>%(delim)s) | # Escape sequence of two delimiters - (?P<named>%(id)s) | # delimiter and a Python identifier - {(?P<braced>%(id)s)} | # delimiter and a braced identifier - (?P<invalid>) # Other ill-formed delimiter exprs - ) - """ - - def __init__(cls, name, bases, dct): - super(_TemplateMetaclass, cls).__init__(name, bases, dct) - if 'pattern' in dct: - pattern = cls.pattern - else: - pattern = _TemplateMetaclass.pattern % { - 'delim' : _re.escape(cls.delimiter), - 'id' : cls.idpattern, - } - cls.pattern = _re.compile(pattern, _re.IGNORECASE | _re.VERBOSE) - - -class Template: - """A string class for supporting $-substitutions.""" - __metaclass__ = _TemplateMetaclass - - delimiter = '$' - idpattern = r'[_a-z][_a-z0-9]*' - - def __init__(self, template): - self.template = template - - # Search for $$, $identifier, ${identifier}, and any bare $'s - - def _invalid(self, mo): - i = mo.start('invalid') - lines = self.template[:i].splitlines(True) - if not lines: - colno = 1 - lineno = 1 - else: - colno = i - len(''.join(lines[:-1])) - lineno = len(lines) - raise ValueError('Invalid placeholder in string: line %d, col %d' % - (lineno, colno)) - - def substitute(self, *args, **kws): - if len(args) > 1: - raise TypeError('Too many positional arguments') - if not args: - mapping = kws - elif kws: - mapping = _multimap(kws, args[0]) - else: - mapping = args[0] - # Helper function for .sub() - def convert(mo): - # Check the most common path first. - named = mo.group('named') or mo.group('braced') - if named is not None: - val = mapping[named] - # We use this idiom instead of str() because the latter will - # fail if val is a Unicode containing non-ASCII characters. - return '%s' % val - if mo.group('escaped') is not None: - return self.delimiter - if mo.group('invalid') is not None: - self._invalid(mo) - raise ValueError('Unrecognized named group in pattern', - self.pattern) - return self.pattern.sub(convert, self.template) - - def safe_substitute(self, *args, **kws): - if len(args) > 1: - raise TypeError('Too many positional arguments') - if not args: - mapping = kws - elif kws: - mapping = _multimap(kws, args[0]) - else: - mapping = args[0] - # Helper function for .sub() - def convert(mo): - named = mo.group('named') - if named is not None: - try: - # We use this idiom instead of str() because the latter - # will fail if val is a Unicode containing non-ASCII - return '%s' % mapping[named] - except KeyError: - return self.delimiter + named - braced = mo.group('braced') - if braced is not None: - try: - return '%s' % mapping[braced] - except KeyError: - return self.delimiter + '{' + braced + '}' - if mo.group('escaped') is not None: - return self.delimiter - if mo.group('invalid') is not None: - return self.delimiter - raise ValueError('Unrecognized named group in pattern', - self.pattern) - return self.pattern.sub(convert, self.template) - - - -#################################################################### -# NOTE: Everything below here is deprecated. Use string methods instead. -# This stuff will go away in Python 3.0. - -# Backward compatible names for exceptions -index_error = ValueError -atoi_error = ValueError -atof_error = ValueError -atol_error = ValueError - -# convert UPPER CASE letters to lower case -def lower(s): - """lower(s) -> string - - Return a copy of the string s converted to lowercase. - - """ - return s.lower() - -# Convert lower case letters to UPPER CASE -def upper(s): - """upper(s) -> string - - Return a copy of the string s converted to uppercase. - - """ - return s.upper() - -# Swap lower case letters and UPPER CASE -def swapcase(s): - """swapcase(s) -> string - - Return a copy of the string s with upper case characters - converted to lowercase and vice versa. - - """ - return s.swapcase() - -# Strip leading and trailing tabs and spaces -def strip(s, chars=None): - """strip(s [,chars]) -> string - - Return a copy of the string s with leading and trailing - whitespace removed. - If chars is given and not None, remove characters in chars instead. - If chars is unicode, S will be converted to unicode before stripping. - - """ - return s.strip(chars) - -# Strip leading tabs and spaces -def lstrip(s, chars=None): - """lstrip(s [,chars]) -> string - - Return a copy of the string s with leading whitespace removed. - If chars is given and not None, remove characters in chars instead. - - """ - return s.lstrip(chars) - -# Strip trailing tabs and spaces -def rstrip(s, chars=None): - """rstrip(s [,chars]) -> string - - Return a copy of the string s with trailing whitespace removed. - If chars is given and not None, remove characters in chars instead. - - """ - return s.rstrip(chars) - - -# Split a string into a list of space/tab-separated words -def split(s, sep=None, maxsplit=-1): - """split(s [,sep [,maxsplit]]) -> list of strings - - Return a list of the words in the string s, using sep as the - delimiter string. If maxsplit is given, splits at no more than - maxsplit places (resulting in at most maxsplit+1 words). If sep - is not specified or is None, any whitespace string is a separator. - - (split and splitfields are synonymous) - - """ - return s.split(sep, maxsplit) -splitfields = split - -# Split a string into a list of space/tab-separated words -def rsplit(s, sep=None, maxsplit=-1): - """rsplit(s [,sep [,maxsplit]]) -> list of strings - - Return a list of the words in the string s, using sep as the - delimiter string, starting at the end of the string and working - to the front. If maxsplit is given, at most maxsplit splits are - done. If sep is not specified or is None, any whitespace string - is a separator. - """ - return s.rsplit(sep, maxsplit) - -# Join fields with optional separator -def join(words, sep = ' '): - """join(list [,sep]) -> string - - Return a string composed of the words in list, with - intervening occurrences of sep. The default separator is a - single space. - - (joinfields and join are synonymous) - - """ - return sep.join(words) -joinfields = join - -# Find substring, raise exception if not found -def index(s, *args): - """index(s, sub [,start [,end]]) -> int - - Like find but raises ValueError when the substring is not found. - - """ - return s.index(*args) - -# Find last substring, raise exception if not found -def rindex(s, *args): - """rindex(s, sub [,start [,end]]) -> int - - Like rfind but raises ValueError when the substring is not found. - - """ - return s.rindex(*args) - -# Count non-overlapping occurrences of substring -def count(s, *args): - """count(s, sub[, start[,end]]) -> int - - Return the number of occurrences of substring sub in string - s[start:end]. Optional arguments start and end are - interpreted as in slice notation. - - """ - return s.count(*args) - -# Find substring, return -1 if not found -def find(s, *args): - """find(s, sub [,start [,end]]) -> in - - Return the lowest index in s where substring sub is found, - such that sub is contained within s[start,end]. Optional - arguments start and end are interpreted as in slice notation. - - Return -1 on failure. - - """ - return s.find(*args) - -# Find last substring, return -1 if not found -def rfind(s, *args): - """rfind(s, sub [,start [,end]]) -> int - - Return the highest index in s where substring sub is found, - such that sub is contained within s[start,end]. Optional - arguments start and end are interpreted as in slice notation. - - Return -1 on failure. - - """ - return s.rfind(*args) - -# for a bit of speed -_float = float -_int = int -_long = long - -# Convert string to float -def atof(s): - """atof(s) -> float - - Return the floating point number represented by the string s. - - """ - return _float(s) - - -# Convert string to integer -def atoi(s , base=10): - """atoi(s [,base]) -> int - - Return the integer represented by the string s in the given - base, which defaults to 10. The string s must consist of one - or more digits, possibly preceded by a sign. If base is 0, it - is chosen from the leading characters of s, 0 for octal, 0x or - 0X for hexadecimal. If base is 16, a preceding 0x or 0X is - accepted. - - """ - return _int(s, base) - - -# Convert string to long integer -def atol(s, base=10): - """atol(s [,base]) -> long - - Return the long integer represented by the string s in the - given base, which defaults to 10. The string s must consist - of one or more digits, possibly preceded by a sign. If base - is 0, it is chosen from the leading characters of s, 0 for - octal, 0x or 0X for hexadecimal. If base is 16, a preceding - 0x or 0X is accepted. A trailing L or l is not accepted, - unless base is 0. - - """ - return _long(s, base) - - -# Left-justify a string -def ljust(s, width, *args): - """ljust(s, width[, fillchar]) -> string - - Return a left-justified version of s, in a field of the - specified width, padded with spaces as needed. The string is - never truncated. If specified the fillchar is used instead of spaces. - - """ - return s.ljust(width, *args) - -# Right-justify a string -def rjust(s, width, *args): - """rjust(s, width[, fillchar]) -> string - - Return a right-justified version of s, in a field of the - specified width, padded with spaces as needed. The string is - never truncated. If specified the fillchar is used instead of spaces. - - """ - return s.rjust(width, *args) - -# Center a string -def center(s, width, *args): - """center(s, width[, fillchar]) -> string - - Return a center version of s, in a field of the specified - width. padded with spaces as needed. The string is never - truncated. If specified the fillchar is used instead of spaces. - - """ - return s.center(width, *args) - -# Zero-fill a number, e.g., (12, 3) --> '012' and (-3, 3) --> '-03' -# Decadent feature: the argument may be a string or a number -# (Use of this is deprecated; it should be a string as with ljust c.s.) -def zfill(x, width): - """zfill(x, width) -> string - - Pad a numeric string x with zeros on the left, to fill a field - of the specified width. The string x is never truncated. - - """ - if not isinstance(x, basestring): - x = repr(x) - return x.zfill(width) - -# Expand tabs in a string. -# Doesn't take non-printing chars into account, but does understand \n. -def expandtabs(s, tabsize=8): - """expandtabs(s [,tabsize]) -> string - - Return a copy of the string s with all tab characters replaced - by the appropriate number of spaces, depending on the current - column, and the tabsize (default 8). - - """ - return s.expandtabs(tabsize) - -# Character translation through look-up table. -def translate(s, table, deletions=""): - """translate(s,table [,deletions]) -> string - - Return a copy of the string s, where all characters occurring - in the optional argument deletions are removed, and the - remaining characters have been mapped through the given - translation table, which must be a string of length 256. The - deletions argument is not allowed for Unicode strings. - - """ - if deletions: - return s.translate(table, deletions) - else: - # Add s[:0] so that if s is Unicode and table is an 8-bit string, - # table is converted to Unicode. This means that table *cannot* - # be a dictionary -- for that feature, use u.translate() directly. - return s.translate(table + s[:0]) - -# Capitalize a string, e.g. "aBc dEf" -> "Abc def". -def capitalize(s): - """capitalize(s) -> string - - Return a copy of the string s with only its first character - capitalized. - - """ - return s.capitalize() - -# Substring replacement (global) -def replace(s, old, new, maxsplit=-1): - """replace (str, old, new[, maxsplit]) -> string - - Return a copy of string str with all occurrences of substring - old replaced by new. If the optional argument maxsplit is - given, only the first maxsplit occurrences are replaced. - - """ - return s.replace(old, new, maxsplit) - - -# Try importing optional built-in module "strop" -- if it exists, -# it redefines some string operations that are 100-1000 times faster. -# It also defines values for whitespace, lowercase and uppercase -# that match <ctype.h>'s definitions. - -try: - from strop import maketrans, lowercase, uppercase, whitespace - letters = lowercase + uppercase -except ImportError: - pass # Use the original versions diff --git a/paste/util/subprocess24.py b/paste/util/subprocess24.py deleted file mode 100644 index 57ec119..0000000 --- a/paste/util/subprocess24.py +++ /dev/null @@ -1,1152 +0,0 @@ -# subprocess - Subprocesses with accessible I/O streams -# -# For more information about this module, see PEP 324. -# -# This module should remain compatible with Python 2.2, see PEP 291. -# -# Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se> -# -# Licensed to PSF under a Contributor Agreement. -# See http://www.python.org/2.4/license for licensing details. - -r"""subprocess - Subprocesses with accessible I/O streams - -This module allows you to spawn processes, connect to their -input/output/error pipes, and obtain their return codes. This module -intends to replace several other, older modules and functions, like: - -os.system -os.spawn* -os.popen* -popen2.* -commands.* - -Information about how the subprocess module can be used to replace these -modules and functions can be found below. - - - -Using the subprocess module -=========================== -This module defines one class called Popen: - -class Popen(args, bufsize=0, executable=None, - stdin=None, stdout=None, stderr=None, - preexec_fn=None, close_fds=False, shell=False, - cwd=None, env=None, universal_newlines=False, - startupinfo=None, creationflags=0): - - -Arguments are: - -args should be a string, or a sequence of program arguments. The -program to execute is normally the first item in the args sequence or -string, but can be explicitly set by using the executable argument. - -On UNIX, with shell=False (default): In this case, the Popen class -uses os.execvp() to execute the child program. args should normally -be a sequence. A string will be treated as a sequence with the string -as the only item (the program to execute). - -On UNIX, with shell=True: If args is a string, it specifies the -command string to execute through the shell. If args is a sequence, -the first item specifies the command string, and any additional items -will be treated as additional shell arguments. - -On Windows: the Popen class uses CreateProcess() to execute the child -program, which operates on strings. If args is a sequence, it will be -converted to a string using the list2cmdline method. Please note that -not all MS Windows applications interpret the command line the same -way: The list2cmdline is designed for applications using the same -rules as the MS C runtime. - -bufsize, if given, has the same meaning as the corresponding argument -to the built-in open() function: 0 means unbuffered, 1 means line -buffered, any other positive value means use a buffer of -(approximately) that size. A negative bufsize means to use the system -default, which usually means fully buffered. The default value for -bufsize is 0 (unbuffered). - -stdin, stdout and stderr specify the executed programs' standard -input, standard output and standard error file handles, respectively. -Valid values are PIPE, an existing file descriptor (a positive -integer), an existing file object, and None. PIPE indicates that a -new pipe to the child should be created. With None, no redirection -will occur; the child's file handles will be inherited from the -parent. Additionally, stderr can be STDOUT, which indicates that the -stderr data from the applications should be captured into the same -file handle as for stdout. - -If preexec_fn is set to a callable object, this object will be called -in the child process just before the child is executed. - -If close_fds is true, all file descriptors except 0, 1 and 2 will be -closed before the child process is executed. - -if shell is true, the specified command will be executed through the -shell. - -If cwd is not None, the current directory will be changed to cwd -before the child is executed. - -If env is not None, it defines the environment variables for the new -process. - -If universal_newlines is true, the file objects stdout and stderr are -opened as a text files, but lines may be terminated by any of '\n', -the Unix end-of-line convention, '\r', the Macintosh convention or -'\r\n', the Windows convention. All of these external representations -are seen as '\n' by the Python program. Note: This feature is only -available if Python is built with universal newline support (the -default). Also, the newlines attribute of the file objects stdout, -stdin and stderr are not updated by the communicate() method. - -The startupinfo and creationflags, if given, will be passed to the -underlying CreateProcess() function. They can specify things such as -appearance of the main window and priority for the new process. -(Windows only) - - -This module also defines two shortcut functions: - -call(*args, **kwargs): - Run command with arguments. Wait for command to complete, then - return the returncode attribute. The arguments are the same as for - the Popen constructor. Example: - - retcode = call(["ls", "-l"]) - - -Exceptions ----------- -Exceptions raised in the child process, before the new program has -started to execute, will be re-raised in the parent. Additionally, -the exception object will have one extra attribute called -'child_traceback', which is a string containing traceback information -from the childs point of view. - -The most common exception raised is OSError. This occurs, for -example, when trying to execute a non-existent file. Applications -should prepare for OSErrors. - -A ValueError will be raised if Popen is called with invalid arguments. - - -Security --------- -Unlike some other popen functions, this implementation will never call -/bin/sh implicitly. This means that all characters, including shell -metacharacters, can safely be passed to child processes. - - -Popen objects -============= -Instances of the Popen class have the following methods: - -poll() - Check if child process has terminated. Returns returncode - attribute. - -wait() - Wait for child process to terminate. Returns returncode attribute. - -communicate(input=None) - Interact with process: Send data to stdin. Read data from stdout - and stderr, until end-of-file is reached. Wait for process to - terminate. The optional stdin argument should be a string to be - sent to the child process, or None, if no data should be sent to - the child. - - communicate() returns a tuple (stdout, stderr). - - Note: The data read is buffered in memory, so do not use this - method if the data size is large or unlimited. - -The following attributes are also available: - -stdin - If the stdin argument is PIPE, this attribute is a file object - that provides input to the child process. Otherwise, it is None. - -stdout - If the stdout argument is PIPE, this attribute is a file object - that provides output from the child process. Otherwise, it is - None. - -stderr - If the stderr argument is PIPE, this attribute is file object that - provides error output from the child process. Otherwise, it is - None. - -pid - The process ID of the child process. - -returncode - The child return code. A None value indicates that the process - hasn't terminated yet. A negative value -N indicates that the - child was terminated by signal N (UNIX only). - - -Replacing older functions with the subprocess module -==================================================== -In this section, "a ==> b" means that b can be used as a replacement -for a. - -Note: All functions in this section fail (more or less) silently if -the executed program cannot be found; this module raises an OSError -exception. - -In the following examples, we assume that the subprocess module is -imported with "from subprocess import *". - - -Replacing /bin/sh shell backquote ---------------------------------- -output=`mycmd myarg` -==> -output = Popen(["mycmd", "myarg"], stdout=PIPE).communicate()[0] - - -Replacing shell pipe line -------------------------- -output=`dmesg | grep hda` -==> -p1 = Popen(["dmesg"], stdout=PIPE) -p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) -output = p2.communicate()[0] - - -Replacing os.system() ---------------------- -sts = os.system("mycmd" + " myarg") -==> -p = Popen("mycmd" + " myarg", shell=True) -sts = os.waitpid(p.pid, 0) - -Note: - -* Calling the program through the shell is usually not required. - -* It's easier to look at the returncode attribute than the - exitstatus. - -A more real-world example would look like this: - -try: - retcode = call("mycmd" + " myarg", shell=True) - if retcode < 0: - print >>sys.stderr, "Child was terminated by signal", -retcode - else: - print >>sys.stderr, "Child returned", retcode -except OSError, e: - print >>sys.stderr, "Execution failed:", e - - -Replacing os.spawn* -------------------- -P_NOWAIT example: - -pid = os.spawnlp(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg") -==> -pid = Popen(["/bin/mycmd", "myarg"]).pid - - -P_WAIT example: - -retcode = os.spawnlp(os.P_WAIT, "/bin/mycmd", "mycmd", "myarg") -==> -retcode = call(["/bin/mycmd", "myarg"]) - - -Vector example: - -os.spawnvp(os.P_NOWAIT, path, args) -==> -Popen([path] + args[1:]) - - -Environment example: - -os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env) -==> -Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"}) - - -Replacing os.popen* -------------------- -pipe = os.popen(cmd, mode='r', bufsize) -==> -pipe = Popen(cmd, shell=True, bufsize=bufsize, stdout=PIPE).stdout - -pipe = os.popen(cmd, mode='w', bufsize) -==> -pipe = Popen(cmd, shell=True, bufsize=bufsize, stdin=PIPE).stdin - - -(child_stdin, child_stdout) = os.popen2(cmd, mode, bufsize) -==> -p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) -(child_stdin, child_stdout) = (p.stdin, p.stdout) - - -(child_stdin, - child_stdout, - child_stderr) = os.popen3(cmd, mode, bufsize) -==> -p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) -(child_stdin, - child_stdout, - child_stderr) = (p.stdin, p.stdout, p.stderr) - - -(child_stdin, child_stdout_and_stderr) = os.popen4(cmd, mode, bufsize) -==> -p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) -(child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout) - - -Replacing popen2.* ------------------- -Note: If the cmd argument to popen2 functions is a string, the command -is executed through /bin/sh. If it is a list, the command is directly -executed. - -(child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode) -==> -p = Popen(["somestring"], shell=True, bufsize=bufsize - stdin=PIPE, stdout=PIPE, close_fds=True) -(child_stdout, child_stdin) = (p.stdout, p.stdin) - - -(child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, mode) -==> -p = Popen(["mycmd", "myarg"], bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) -(child_stdout, child_stdin) = (p.stdout, p.stdin) - -The popen2.Popen3 and popen3.Popen4 basically works as subprocess.Popen, -except that: - -* subprocess.Popen raises an exception if the execution fails -* the capturestderr argument is replaced with the stderr argument. -* stdin=PIPE and stdout=PIPE must be specified. -* popen2 closes all filedescriptors by default, but you have to specify - close_fds=True with subprocess.Popen. - - -""" - -import sys -mswindows = (sys.platform == "win32") - -import os -import types -import traceback - -if mswindows: - import threading - import msvcrt - ## @@: Changed in Paste - ## Since this module is only used on pre-python-2.4 systems, they probably - ## don't have _subprocess installed, but hopefully have the win32 stuff - ## installed. - if 1: # <-- change this to use pywin32 instead of the _subprocess driver - import pywintypes - from win32api import GetStdHandle, STD_INPUT_HANDLE, \ - STD_OUTPUT_HANDLE, STD_ERROR_HANDLE - from win32api import GetCurrentProcess, DuplicateHandle, \ - GetModuleFileName, GetVersion - from win32con import DUPLICATE_SAME_ACCESS, SW_HIDE - from win32pipe import CreatePipe - from win32process import CreateProcess, STARTUPINFO, \ - GetExitCodeProcess, STARTF_USESTDHANDLES, \ - STARTF_USESHOWWINDOW, CREATE_NEW_CONSOLE - from win32event import WaitForSingleObject, INFINITE, WAIT_OBJECT_0 - else: - from _subprocess import * - class STARTUPINFO: - dwFlags = 0 - hStdInput = None - hStdOutput = None - hStdError = None - class pywintypes: - error = IOError -else: - import select - import errno - import fcntl - import pickle - -__all__ = ["Popen", "PIPE", "STDOUT", "call"] - -try: - MAXFD = os.sysconf("SC_OPEN_MAX") -except: - MAXFD = 256 - -# True/False does not exist on 2.2.0 -try: - False -except NameError: - False = 0 - True = 1 - -_active = [] - -def _cleanup(): - for inst in _active[:]: - inst.poll() - -PIPE = -1 -STDOUT = -2 - - -def call(*args, **kwargs): - """Run command with arguments. Wait for command to complete, then - return the returncode attribute. - - The arguments are the same as for the Popen constructor. Example: - - retcode = call(["ls", "-l"]) - """ - return Popen(*args, **kwargs).wait() - - -def list2cmdline(seq): - """ - Translate a sequence of arguments into a command line - string, using the same rules as the MS C runtime: - - 1) Arguments are delimited by white space, which is either a - space or a tab. - - 2) A string surrounded by double quotation marks is - interpreted as a single argument, regardless of white space - contained within. A quoted string can be embedded in an - argument. - - 3) A double quotation mark preceded by a backslash is - interpreted as a literal double quotation mark. - - 4) Backslashes are interpreted literally, unless they - immediately precede a double quotation mark. - - 5) If backslashes immediately precede a double quotation mark, - every pair of backslashes is interpreted as a literal - backslash. If the number of backslashes is odd, the last - backslash escapes the next double quotation mark as - described in rule 3. - """ - - # See - # http://msdn.microsoft.com/library/en-us/vccelng/htm/progs_12.asp - result = [] - needquote = False - for arg in seq: - bs_buf = [] - - # Add a space to separate this argument from the others - if result: - result.append(' ') - - needquote = (" " in arg) or ("\t" in arg) - if needquote: - result.append('"') - - for c in arg: - if c == '\\': - # Don't know if we need to double yet. - bs_buf.append(c) - elif c == '"': - # Double backspaces. - result.append('\\' * len(bs_buf)*2) - bs_buf = [] - result.append('\\"') - else: - # Normal char - if bs_buf: - result.extend(bs_buf) - bs_buf = [] - result.append(c) - - # Add remaining backspaces, if any. - if bs_buf: - result.extend(bs_buf) - - if needquote: - result.extend(bs_buf) - result.append('"') - - return ''.join(result) - - -class Popen(object): - def __init__(self, args, bufsize=0, executable=None, - stdin=None, stdout=None, stderr=None, - preexec_fn=None, close_fds=False, shell=False, - cwd=None, env=None, universal_newlines=False, - startupinfo=None, creationflags=0): - """Create new Popen instance.""" - _cleanup() - - if not isinstance(bufsize, (int, long)): - raise TypeError("bufsize must be an integer") - - if mswindows: - if preexec_fn is not None: - raise ValueError("preexec_fn is not supported on Windows " - "platforms") - if close_fds: - raise ValueError("close_fds is not supported on Windows " - "platforms") - else: - # POSIX - if startupinfo is not None: - raise ValueError("startupinfo is only supported on Windows " - "platforms") - if creationflags != 0: - raise ValueError("creationflags is only supported on Windows " - "platforms") - - self.stdin = None - self.stdout = None - self.stderr = None - self.pid = None - self.returncode = None - self.universal_newlines = universal_newlines - - # Input and output objects. The general principle is like - # this: - # - # Parent Child - # ------ ----- - # p2cwrite ---stdin---> p2cread - # c2pread <--stdout--- c2pwrite - # errread <--stderr--- errwrite - # - # On POSIX, the child objects are file descriptors. On - # Windows, these are Windows file handles. The parent objects - # are file descriptors on both platforms. The parent objects - # are None when not using PIPEs. The child objects are None - # when not redirecting. - - (p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) = self._get_handles(stdin, stdout, stderr) - - self._execute_child(args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, - startupinfo, creationflags, shell, - p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) - - if p2cwrite: - self.stdin = os.fdopen(p2cwrite, 'wb', bufsize) - if c2pread: - if universal_newlines: - self.stdout = os.fdopen(c2pread, 'rU', bufsize) - else: - self.stdout = os.fdopen(c2pread, 'rb', bufsize) - if errread: - if universal_newlines: - self.stderr = os.fdopen(errread, 'rU', bufsize) - else: - self.stderr = os.fdopen(errread, 'rb', bufsize) - - _active.append(self) - - - def _translate_newlines(self, data): - data = data.replace("\r\n", "\n") - data = data.replace("\r", "\n") - return data - - - if mswindows: - # - # Windows methods - # - def _get_handles(self, stdin, stdout, stderr): - """Construct and return tupel with IO objects: - p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite - """ - if stdin == None and stdout == None and stderr == None: - return (None, None, None, None, None, None) - - p2cread, p2cwrite = None, None - c2pread, c2pwrite = None, None - errread, errwrite = None, None - - if stdin == None: - p2cread = GetStdHandle(STD_INPUT_HANDLE) - elif stdin == PIPE: - p2cread, p2cwrite = CreatePipe(None, 0) - # Detach and turn into fd - p2cwrite = p2cwrite.Detach() - p2cwrite = msvcrt.open_osfhandle(p2cwrite, 0) - elif type(stdin) == types.IntType: - p2cread = msvcrt.get_osfhandle(stdin) - else: - # Assuming file-like object - p2cread = msvcrt.get_osfhandle(stdin.fileno()) - p2cread = self._make_inheritable(p2cread) - - if stdout == None: - c2pwrite = GetStdHandle(STD_OUTPUT_HANDLE) - elif stdout == PIPE: - c2pread, c2pwrite = CreatePipe(None, 0) - # Detach and turn into fd - c2pread = c2pread.Detach() - c2pread = msvcrt.open_osfhandle(c2pread, 0) - elif type(stdout) == types.IntType: - c2pwrite = msvcrt.get_osfhandle(stdout) - else: - # Assuming file-like object - c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) - c2pwrite = self._make_inheritable(c2pwrite) - - if stderr == None: - errwrite = GetStdHandle(STD_ERROR_HANDLE) - elif stderr == PIPE: - errread, errwrite = CreatePipe(None, 0) - # Detach and turn into fd - errread = errread.Detach() - errread = msvcrt.open_osfhandle(errread, 0) - elif stderr == STDOUT: - errwrite = c2pwrite - elif type(stderr) == types.IntType: - errwrite = msvcrt.get_osfhandle(stderr) - else: - # Assuming file-like object - errwrite = msvcrt.get_osfhandle(stderr.fileno()) - errwrite = self._make_inheritable(errwrite) - - return (p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) - - - def _make_inheritable(self, handle): - """Return a duplicate of handle, which is inheritable""" - return DuplicateHandle(GetCurrentProcess(), handle, - GetCurrentProcess(), 0, 1, - DUPLICATE_SAME_ACCESS) - - - def _find_w9xpopen(self): - """Find and return absolut path to w9xpopen.exe""" - w9xpopen = os.path.join(os.path.dirname(GetModuleFileName(0)), - "w9xpopen.exe") - if not os.path.exists(w9xpopen): - # Eeek - file-not-found - possibly an embedding - # situation - see if we can locate it in sys.exec_prefix - w9xpopen = os.path.join(os.path.dirname(sys.exec_prefix), - "w9xpopen.exe") - if not os.path.exists(w9xpopen): - raise RuntimeError("Cannot locate w9xpopen.exe, which is " - "needed for Popen to work with your " - "shell or platform.") - return w9xpopen - - - def _execute_child(self, args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, - startupinfo, creationflags, shell, - p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite): - """Execute program (MS Windows version)""" - - if not isinstance(args, types.StringTypes): - args = list2cmdline(args) - - # Process startup details - default_startupinfo = STARTUPINFO() - if startupinfo == None: - startupinfo = default_startupinfo - if not None in (p2cread, c2pwrite, errwrite): - startupinfo.dwFlags |= STARTF_USESTDHANDLES - startupinfo.hStdInput = p2cread - startupinfo.hStdOutput = c2pwrite - startupinfo.hStdError = errwrite - - if shell: - default_startupinfo.dwFlags |= STARTF_USESHOWWINDOW - default_startupinfo.wShowWindow = SW_HIDE - comspec = os.environ.get("COMSPEC", "cmd.exe") - args = comspec + " /c " + args - if (GetVersion() >= 0x80000000L or - os.path.basename(comspec).lower() == "command.com"): - # Win9x, or using command.com on NT. We need to - # use the w9xpopen intermediate program. For more - # information, see KB Q150956 - # (http://web.archive.org/web/20011105084002/http://support.microsoft.com/support/kb/articles/Q150/9/56.asp) - w9xpopen = self._find_w9xpopen() - args = '"%s" %s' % (w9xpopen, args) - # Not passing CREATE_NEW_CONSOLE has been known to - # cause random failures on win9x. Specifically a - # dialog: "Your program accessed mem currently in - # use at xxx" and a hopeful warning about the - # stability of your system. Cost is Ctrl+C wont - # kill children. - creationflags |= CREATE_NEW_CONSOLE - - # Start the process - try: - hp, ht, pid, tid = CreateProcess(executable, args, - # no special security - None, None, - # must inherit handles to pass std - # handles - 1, - creationflags, - env, - cwd, - startupinfo) - except pywintypes.error, e: - # Translate pywintypes.error to WindowsError, which is - # a subclass of OSError. FIXME: We should really - # translate errno using _sys_errlist (or simliar), but - # how can this be done from Python? - raise WindowsError(*e.args) - - # Retain the process handle, but close the thread handle - self._handle = hp - self.pid = pid - ht.Close() - - # Child is launched. Close the parent's copy of those pipe - # handles that only the child should have open. You need - # to make sure that no handles to the write end of the - # output pipe are maintained in this process or else the - # pipe will not close when the child process exits and the - # ReadFile will hang. - if p2cread != None: - p2cread.Close() - if c2pwrite != None: - c2pwrite.Close() - if errwrite != None: - errwrite.Close() - - - def poll(self): - """Check if child process has terminated. Returns returncode - attribute.""" - if self.returncode == None: - if WaitForSingleObject(self._handle, 0) == WAIT_OBJECT_0: - self.returncode = GetExitCodeProcess(self._handle) - _active.remove(self) - return self.returncode - - - def wait(self): - """Wait for child process to terminate. Returns returncode - attribute.""" - if self.returncode == None: - obj = WaitForSingleObject(self._handle, INFINITE) - self.returncode = GetExitCodeProcess(self._handle) - _active.remove(self) - return self.returncode - - - def _readerthread(self, fh, buffer): - buffer.append(fh.read()) - - - def communicate(self, input=None): - """Interact with process: Send data to stdin. Read data from - stdout and stderr, until end-of-file is reached. Wait for - process to terminate. The optional input argument should be a - string to be sent to the child process, or None, if no data - should be sent to the child. - - communicate() returns a tuple (stdout, stderr).""" - stdout = None # Return - stderr = None # Return - - if self.stdout: - stdout = [] - stdout_thread = threading.Thread(target=self._readerthread, - args=(self.stdout, stdout)) - stdout_thread.setDaemon(True) - stdout_thread.start() - if self.stderr: - stderr = [] - stderr_thread = threading.Thread(target=self._readerthread, - args=(self.stderr, stderr)) - stderr_thread.setDaemon(True) - stderr_thread.start() - - if self.stdin: - if input != None: - self.stdin.write(input) - self.stdin.close() - - if self.stdout: - stdout_thread.join() - if self.stderr: - stderr_thread.join() - - # All data exchanged. Translate lists into strings. - if stdout != None: - stdout = stdout[0] - if stderr != None: - stderr = stderr[0] - - # Translate newlines, if requested. We cannot let the file - # object do the translation: It is based on stdio, which is - # impossible to combine with select (unless forcing no - # buffering). - if self.universal_newlines and hasattr(open, 'newlines'): - if stdout: - stdout = self._translate_newlines(stdout) - if stderr: - stderr = self._translate_newlines(stderr) - - self.wait() - return (stdout, stderr) - - else: - # - # POSIX methods - # - def _get_handles(self, stdin, stdout, stderr): - """Construct and return tupel with IO objects: - p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite - """ - p2cread, p2cwrite = None, None - c2pread, c2pwrite = None, None - errread, errwrite = None, None - - if stdin == None: - pass - elif stdin == PIPE: - p2cread, p2cwrite = os.pipe() - elif type(stdin) == types.IntType: - p2cread = stdin - else: - # Assuming file-like object - p2cread = stdin.fileno() - - if stdout == None: - pass - elif stdout == PIPE: - c2pread, c2pwrite = os.pipe() - elif type(stdout) == types.IntType: - c2pwrite = stdout - else: - # Assuming file-like object - c2pwrite = stdout.fileno() - - if stderr == None: - pass - elif stderr == PIPE: - errread, errwrite = os.pipe() - elif stderr == STDOUT: - errwrite = c2pwrite - elif type(stderr) == types.IntType: - errwrite = stderr - else: - # Assuming file-like object - errwrite = stderr.fileno() - - return (p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite) - - - def _set_cloexec_flag(self, fd): - try: - cloexec_flag = fcntl.FD_CLOEXEC - except AttributeError: - cloexec_flag = 1 - - old = fcntl.fcntl(fd, fcntl.F_GETFD) - fcntl.fcntl(fd, fcntl.F_SETFD, old | cloexec_flag) - - - def _close_fds(self, but): - for i in range(3, MAXFD): - if i == but: - continue - try: - os.close(i) - except: - pass - - - def _execute_child(self, args, executable, preexec_fn, close_fds, - cwd, env, universal_newlines, - startupinfo, creationflags, shell, - p2cread, p2cwrite, - c2pread, c2pwrite, - errread, errwrite): - """Execute program (POSIX version)""" - - if isinstance(args, types.StringTypes): - args = [args] - - if shell: - args = ["/bin/sh", "-c"] + args - - if executable == None: - executable = args[0] - - # For transferring possible exec failure from child to parent - # The first char specifies the exception type: 0 means - # OSError, 1 means some other error. - errpipe_read, errpipe_write = os.pipe() - self._set_cloexec_flag(errpipe_write) - - self.pid = os.fork() - if self.pid == 0: - # Child - try: - # Close parent's pipe ends - if p2cwrite: - os.close(p2cwrite) - if c2pread: - os.close(c2pread) - if errread: - os.close(errread) - os.close(errpipe_read) - - # Dup fds for child - if p2cread: - os.dup2(p2cread, 0) - if c2pwrite: - os.dup2(c2pwrite, 1) - if errwrite: - os.dup2(errwrite, 2) - - # Close pipe fds. Make sure we doesn't close the same - # fd more than once. - if p2cread: - os.close(p2cread) - if c2pwrite and c2pwrite not in (p2cread,): - os.close(c2pwrite) - if errwrite and errwrite not in (p2cread, c2pwrite): - os.close(errwrite) - - # Close all other fds, if asked for - if close_fds: - self._close_fds(but=errpipe_write) - - if cwd != None: - os.chdir(cwd) - - if preexec_fn: - apply(preexec_fn) - - if env == None: - os.execvp(executable, args) - else: - os.execvpe(executable, args, env) - - except: - exc_type, exc_value, tb = sys.exc_info() - # Save the traceback and attach it to the exception object - exc_lines = traceback.format_exception(exc_type, - exc_value, - tb) - exc_value.child_traceback = ''.join(exc_lines) - os.write(errpipe_write, pickle.dumps(exc_value)) - - # This exitcode won't be reported to applications, so it - # really doesn't matter what we return. - os._exit(255) - - # Parent - os.close(errpipe_write) - if p2cread and p2cwrite: - os.close(p2cread) - if c2pwrite and c2pread: - os.close(c2pwrite) - if errwrite and errread: - os.close(errwrite) - - # Wait for exec to fail or succeed; possibly raising exception - data = os.read(errpipe_read, 1048576) # Exceptions limited to 1 MB - os.close(errpipe_read) - if data != "": - os.waitpid(self.pid, 0) - child_exception = pickle.loads(data) - raise child_exception - - - def _handle_exitstatus(self, sts): - if os.WIFSIGNALED(sts): - self.returncode = -os.WTERMSIG(sts) - elif os.WIFEXITED(sts): - self.returncode = os.WEXITSTATUS(sts) - else: - # Should never happen - raise RuntimeError("Unknown child exit status!") - - _active.remove(self) - - - def poll(self): - """Check if child process has terminated. Returns returncode - attribute.""" - if self.returncode == None: - try: - pid, sts = os.waitpid(self.pid, os.WNOHANG) - if pid == self.pid: - self._handle_exitstatus(sts) - except os.error: - pass - return self.returncode - - - def wait(self): - """Wait for child process to terminate. Returns returncode - attribute.""" - if self.returncode == None: - pid, sts = os.waitpid(self.pid, 0) - self._handle_exitstatus(sts) - return self.returncode - - - def communicate(self, input=None): - """Interact with process: Send data to stdin. Read data from - stdout and stderr, until end-of-file is reached. Wait for - process to terminate. The optional input argument should be a - string to be sent to the child process, or None, if no data - should be sent to the child. - - communicate() returns a tuple (stdout, stderr).""" - read_set = [] - write_set = [] - stdout = None # Return - stderr = None # Return - - if self.stdin: - # Flush stdio buffer. This might block, if the user has - # been writing to .stdin in an uncontrolled fashion. - self.stdin.flush() - if input: - write_set.append(self.stdin) - else: - self.stdin.close() - if self.stdout: - read_set.append(self.stdout) - stdout = [] - if self.stderr: - read_set.append(self.stderr) - stderr = [] - - while read_set or write_set: - rlist, wlist, xlist = select.select(read_set, write_set, []) - - if self.stdin in wlist: - # When select has indicated that the file is writable, - # we can write up to PIPE_BUF bytes without risk - # blocking. POSIX defines PIPE_BUF >= 512 - bytes_written = os.write(self.stdin.fileno(), input[:512]) - input = input[bytes_written:] - if not input: - self.stdin.close() - write_set.remove(self.stdin) - - if self.stdout in rlist: - data = os.read(self.stdout.fileno(), 1024) - if data == "": - self.stdout.close() - read_set.remove(self.stdout) - stdout.append(data) - - if self.stderr in rlist: - data = os.read(self.stderr.fileno(), 1024) - if data == "": - self.stderr.close() - read_set.remove(self.stderr) - stderr.append(data) - - # All data exchanged. Translate lists into strings. - if stdout != None: - stdout = ''.join(stdout) - if stderr != None: - stderr = ''.join(stderr) - - # Translate newlines, if requested. We cannot let the file - # object do the translation: It is based on stdio, which is - # impossible to combine with select (unless forcing no - # buffering). - if self.universal_newlines and hasattr(open, 'newlines'): - if stdout: - stdout = self._translate_newlines(stdout) - if stderr: - stderr = self._translate_newlines(stderr) - - self.wait() - return (stdout, stderr) - - -def _demo_posix(): - # - # Example 1: Simple redirection: Get process list - # - plist = Popen(["ps"], stdout=PIPE).communicate()[0] - print "Process list:" - print plist - - # - # Example 2: Change uid before executing child - # - if os.getuid() == 0: - p = Popen(["id"], preexec_fn=lambda: os.setuid(100)) - p.wait() - - # - # Example 3: Connecting several subprocesses - # - print "Looking for 'hda'..." - p1 = Popen(["dmesg"], stdout=PIPE) - p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) - print repr(p2.communicate()[0]) - - # - # Example 4: Catch execution error - # - print - print "Trying a weird file..." - try: - print Popen(["/this/path/does/not/exist"]).communicate() - except OSError, e: - if e.errno == errno.ENOENT: - print "The file didn't exist. I thought so..." - print "Child traceback:" - print e.child_traceback - else: - print "Error", e.errno - else: - print >>sys.stderr, "Gosh. No error." - - -def _demo_windows(): - # - # Example 1: Connecting several subprocesses - # - print "Looking for 'PROMPT' in set output..." - p1 = Popen("set", stdout=PIPE, shell=True) - p2 = Popen('find "PROMPT"', stdin=p1.stdout, stdout=PIPE) - print repr(p2.communicate()[0]) - - # - # Example 2: Simple execution of program - # - print "Executing calc..." - p = Popen("calc") - p.wait() - - -if __name__ == "__main__": - if mswindows: - _demo_windows() - else: - _demo_posix() diff --git a/paste/util/template.py b/paste/util/template.py index 1e42b5a..f0826af 100644 --- a/paste/util/template.py +++ b/paste/util/template.py @@ -31,9 +31,10 @@ If there are syntax errors ``TemplateError`` will be raised. """ import re +import six import sys import cgi -import urllib +from six.moves.urllib.parse import quote from paste.util.looper import looper __all__ = ['TemplateError', 'Template', 'sub', 'HTMLTemplate', @@ -77,7 +78,7 @@ class Template(object): def __init__(self, content, name=None, namespace=None): self.content = content - self._unicode = isinstance(content, unicode) + self._unicode = isinstance(content, six.text_type) self.name = name self._parsed = parse(content, name=name) if namespace is None: @@ -123,7 +124,7 @@ class Template(object): def _interpret_codes(self, codes, ns, out): __traceback_hide__ = True for item in codes: - if isinstance(item, basestring): + if isinstance(item, six.string_types): out.append(item) else: self._interpret_code(item, ns, out) @@ -184,7 +185,7 @@ class Template(object): __traceback_hide__ = True # @@: if/else/else gets through for part in parts: - assert not isinstance(part, basestring) + assert not isinstance(part, six.string_types) name, pos = part[0], part[1] if name == 'else': result = True @@ -207,17 +208,17 @@ class Template(object): else: arg0 = str(e) e.args = (self._add_line_info(arg0, pos),) - raise exc_info[0], e, exc_info[2] + six.reraise(exc_info[0], e, exc_info[2]) def _exec(self, code, ns, pos): __traceback_hide__ = True try: - exec code in ns + six.exec_(code, ns) except: exc_info = sys.exc_info() e = exc_info[1] e.args = (self._add_line_info(e.args[0], pos),) - raise exc_info[0], e, exc_info[2] + six.reraise(exc_info[0], e, exc_info[2]) def _repr(self, value, pos): __traceback_hide__ = True @@ -226,7 +227,7 @@ class Template(object): return '' if self._unicode: try: - value = unicode(value) + value = six.text_type(value) except UnicodeDecodeError: value = str(value) else: @@ -235,22 +236,22 @@ class Template(object): exc_info = sys.exc_info() e = exc_info[1] e.args = (self._add_line_info(e.args[0], pos),) - raise exc_info[0], e, exc_info[2] + six.reraise(exc_info[0], e, exc_info[2]) else: - if self._unicode and isinstance(value, str): + if self._unicode and isinstance(value, six.binary_type): if not self.decode_encoding: raise UnicodeDecodeError( 'Cannot decode str value %r into unicode ' '(no default_encoding provided)' % value) value = value.decode(self.default_encoding) - elif not self._unicode and isinstance(value, unicode): + elif not self._unicode and isinstance(value, six.text_type): if not self.decode_encoding: raise UnicodeEncodeError( 'Cannot encode unicode value %r into str ' '(no default_encoding provided)' % value) value = value.encode(self.default_encoding) return value - + def _add_line_info(self, msg, pos): msg = "%s at line %s column %s" % ( @@ -263,7 +264,6 @@ def sub(content, **kw): name = kw.get('__name') tmpl = Template(content, name=name) return tmpl.substitute(kw) - return result def paste_script_template_renderer(content, vars, filename=None): tmpl = Template(content, name=filename) @@ -317,7 +317,7 @@ class html(object): def html_quote(value): if value is None: return '' - if not isinstance(value, basestring): + if not isinstance(value, six.string_types): if hasattr(value, '__unicode__'): value = unicode(value) else: @@ -328,14 +328,14 @@ def html_quote(value): return value def url(v): - if not isinstance(v, basestring): + if not isinstance(v, six.string_types): if hasattr(v, '__unicode__'): v = unicode(v) else: v = str(v) if isinstance(v, unicode): v = v.encode('utf8') - return urllib.quote(v) + return quote(v) def attr(**kw): kw = kw.items() @@ -369,7 +369,6 @@ def sub_html(content, **kw): name = kw.get('__name') tmpl = HTMLTemplate(content, name=name) return tmpl.substitute(kw) - return result ############################################################ @@ -449,7 +448,7 @@ def trim_lex(tokens): """ for i in range(len(tokens)): current = tokens[i] - if isinstance(tokens[i], basestring): + if isinstance(tokens[i], six.string_types): # we don't trim this continue item = current[0] @@ -463,8 +462,8 @@ def trim_lex(tokens): next = '' else: next = tokens[i+1] - if (not isinstance(next, basestring) - or not isinstance(prev, basestring)): + if (not isinstance(next, six.string_types) + or not isinstance(prev, six.string_types)): continue if ((not prev or trail_whitespace_re.search(prev)) and (not next or lead_whitespace_re.search(next))): @@ -478,7 +477,7 @@ def trim_lex(tokens): next = next[m.end():] tokens[i+1] = next return tokens - + def find_position(string, index): """Given a string and index, return (line, column)""" @@ -505,7 +504,7 @@ def parse(s, name=None): [('cond', (1, 3), ('if', (1, 3), 'x', ['a']), ('elif', (1, 12), 'y', ['b']), ('else', (1, 23), None, ['c']))] Some exceptions:: - + >>> parse('{{continue}}') Traceback (most recent call last): ... @@ -543,7 +542,7 @@ def parse(s, name=None): return result def parse_expr(tokens, name, context=()): - if isinstance(tokens[0], basestring): + if isinstance(tokens[0], six.string_types): return tokens[0], tokens[1:] expr, pos = tokens[0] expr = expr.strip() @@ -626,7 +625,7 @@ def parse_one_cond(tokens, name, context): return part, tokens next, tokens = parse_expr(tokens, name, context) content.append(next) - + def parse_for(tokens, name, context): first, pos = tokens[0] tokens = tokens[1:] @@ -714,8 +713,8 @@ def fill_command(args=None): help="Put the environment in as top-level variables") options, args = parser.parse_args(args) if len(args) < 1: - print 'You must give a template filename' - print dir(parser) + print('You must give a template filename') + print(dir(parser)) assert 0 template_name = args[0] args = args[1:] @@ -724,7 +723,7 @@ def fill_command(args=None): vars.update(os.environ) for value in args: if '=' not in value: - print 'Bad argument: %r' % value + print('Bad argument: %r' % value) sys.exit(2) name, value = value.split('=', 1) if name.startswith('py:'): @@ -754,5 +753,5 @@ def fill_command(args=None): if __name__ == '__main__': from paste.util.template import fill_command fill_command() - - + + diff --git a/paste/util/threadedprint.py b/paste/util/threadedprint.py index ae7dc0d..820311e 100644 --- a/paste/util/threadedprint.py +++ b/paste/util/threadedprint.py @@ -114,7 +114,7 @@ class PrintCatcher(filemixin.FileMixin): self._default.read(*args) else: catchers[name].read(*args) - + def _writedefault(self, name, v): self._default.write(v) @@ -237,8 +237,8 @@ def install_stdin(**kw): register_stdin = _stdincatcher.register deregister_stdin = _stdincatcher.deregister -def uninstall(): - global _stdincatcher, _oldstin, register_stdin, deregister_stdin +def uninstall_stdin(): + global _stdincatcher, _oldstdin, register_stdin, deregister_stdin if _stdincatcher: sys.stdin = _oldstdin _stdincatcher = _oldstdin = None diff --git a/paste/wsgilib.py b/paste/wsgilib.py index 67ced97..98299e2 100644 --- a/paste/wsgilib.py +++ b/paste/wsgilib.py @@ -5,6 +5,8 @@ A module of many disparate routines. """ +from __future__ import print_function + # functions which moved to paste.request and paste.response # Deprecated around 15 Dec 2005 from paste.request import get_cookies, parse_querystring, parse_formvars @@ -13,10 +15,10 @@ from paste.response import HeaderDict, has_header, header_value, remove_header from paste.response import error_body_response, error_response, error_response_app from traceback import print_exception -import urllib -from cStringIO import StringIO +import six import sys -from urlparse import urlsplit +from six.moves import cStringIO as StringIO +from six.moves.urllib.parse import unquote, urlsplit import warnings __all__ = ['add_close', 'add_start_close', 'capture_output', 'catch_errors', @@ -53,10 +55,9 @@ class add_close(object): def __del__(self): if not self._closed: # We can't raise an error or anything at this stage - print >> sys.stderr, ( - "Error: app_iter.close() was not called when finishing " + print("Error: app_iter.close() was not called when finishing " "WSGI request. finalization function %s not called" - % self.close_func) + % self.close_func, file=sys.stderr) class add_start_close(object): """ @@ -80,7 +81,8 @@ class add_start_close(object): if self.first: self.start_func() self.first = False - return self.app_iter.next() + return next(self.app_iter) + __next__ = next def close(self): self._closed = True @@ -92,10 +94,9 @@ class add_start_close(object): def __del__(self): if not self._closed: # We can't raise an error or anything at this stage - print >> sys.stderr, ( - "Error: app_iter.close() was not called when finishing " + print("Error: app_iter.close() was not called when finishing " "WSGI request. finalization function %s not called" - % self.close_func) + % self.close_func, file=sys.stderr) class chained_app_iters(object): @@ -132,15 +133,14 @@ class chained_app_iters(object): except: got_exc = sys.exc_info() if got_exc: - raise got_exc[0], got_exc[1], got_exc[2] + six.reraise(got_exc[0], got_exc[1], got_exc[2]) def __del__(self): if not self._closed: # We can't raise an error or anything at this stage - print >> sys.stderr, ( - "Error: app_iter.close() was not called when finishing " + print("Error: app_iter.close() was not called when finishing " "WSGI request. finalization function %s not called" - % self.close_func) + % self.close_func, file=sys.stderr) class encode_unicode_app_iter(object): """ @@ -158,10 +158,11 @@ class encode_unicode_app_iter(object): return self def next(self): - content = self.app_iter.next() - if isinstance(content, unicode): + content = next(self.app_iter) + if isinstance(content, six.text_type): content = content.encode(self.encoding, self.errors) return content + __next__ = next def close(self): if hasattr(self.app_iterable, 'close'): @@ -283,19 +284,19 @@ def raw_interactive(application, path='', raise_on_wsgi_error=False, if raise_on_wsgi_error: errors = ErrorRaiser() else: - errors = StringIO() + errors = six.BytesIO() basic_environ = { # mandatory CGI variables 'REQUEST_METHOD': 'GET', # always mandatory 'SCRIPT_NAME': '', # may be empty if app is at the root 'PATH_INFO': '', # may be empty if at root of app 'SERVER_NAME': 'localhost', # always mandatory - 'SERVER_PORT': '80', # always mandatory + 'SERVER_PORT': '80', # always mandatory 'SERVER_PROTOCOL': 'HTTP/1.0', # mandatory wsgi variables 'wsgi.version': (1, 0), 'wsgi.url_scheme': 'http', - 'wsgi.input': StringIO(''), + 'wsgi.input': six.BytesIO(), 'wsgi.errors': errors, 'wsgi.multithread': False, 'wsgi.multiprocess': False, @@ -303,7 +304,7 @@ def raw_interactive(application, path='', raise_on_wsgi_error=False, } if path: (_, _, path_info, query, fragment) = urlsplit(str(path)) - path_info = urllib.unquote(path_info) + path_info = unquote(path_info) # urlsplit returns unicode so coerce it back to str path_info, query = str(path_info), str(query) basic_environ['PATH_INFO'] = path_info @@ -316,8 +317,8 @@ def raw_interactive(application, path='', raise_on_wsgi_error=False, and 'HTTP_HOST' not in basic_environ): basic_environ['HTTP_HOST'] = basic_environ['SERVER_NAME'] istream = basic_environ['wsgi.input'] - if isinstance(istream, str): - basic_environ['wsgi.input'] = StringIO(istream) + if isinstance(istream, bytes): + basic_environ['wsgi.input'] = six.BytesIO(istream) basic_environ['CONTENT_LENGTH'] = len(istream) data = {} output = [] @@ -328,7 +329,7 @@ def raw_interactive(application, path='', raise_on_wsgi_error=False, try: if headers_sent: # Re-raise original exception only if headers sent - raise exc_info[0], exc_info[1], exc_info[2] + six.reraise(exc_info[0], exc_info[1], exc_info[2]) finally: # avoid dangling circular reference exc_info = None @@ -344,15 +345,15 @@ def raw_interactive(application, path='', raise_on_wsgi_error=False, try: try: for s in app_iter: - if not isinstance(s, str): + if not isinstance(s, six.binary_type): raise ValueError( - "The app_iter response can only contain str (not " + "The app_iter response can only contain bytes (not " "unicode); got: %r" % s) headers_sent.append(True) if not headers_set: raise AssertionError("Content sent w/o headers!") output.append(s) - except TypeError, e: + except TypeError as e: # Typically "iteration over non-sequence", so we want # to give better debugging information... e.args = ((e.args[0] + ' iterable: %r' % app_iter),) + e.args[1:] @@ -360,7 +361,7 @@ def raw_interactive(application, path='', raise_on_wsgi_error=False, finally: if hasattr(app_iter, 'close'): app_iter.close() - return (data['status'], data['headers'], ''.join(output), + return (data['status'], data['headers'], b''.join(output), errors.getvalue()) class ErrorRaiser(object): @@ -406,7 +407,7 @@ def dump_environ(environ, start_response): variables out as a plain text response. """ output = [] - keys = environ.keys() + keys = list(environ.keys()) keys.sort() for k in keys: v = str(environ[k]).replace("\n","\n ") @@ -417,6 +418,8 @@ def dump_environ(environ, start_response): output.append(environ['wsgi.input'].read(int(content_length))) output.append("\n") output = "".join(output) + if six.PY3: + output = output.encode('utf8') headers = [('Content-Type', 'text/plain'), ('Content-Length', str(len(output)))] start_response("200 OK", headers) @@ -436,7 +439,7 @@ def capture_output(environ, start_response, application): Sends status and header, but *not* body. Returns (status, headers, body). Typically this is used like: - + .. code-block:: python def dehtmlifying_middleware(application): @@ -487,7 +490,7 @@ def intercept_output(environ, application, conditional=None, ``capture_output``) Typically this is used like: - + .. code-block:: python def dehtmlifying_middleware(application): @@ -509,7 +512,7 @@ def intercept_output(environ, application, conditional=None, ``start_response`` will be called and ``(None, None, app_iter)`` will be returned. You must detect that in your code and return the app_iter, like: - + .. code-block:: python def dehtmlifying_middleware(application): @@ -594,4 +597,4 @@ for _name in __all__: if __name__ == '__main__': import doctest doctest.testmod() - + diff --git a/paste/wsgiwrappers.py b/paste/wsgiwrappers.py index 64f20cf..7b8f6de 100644 --- a/paste/wsgiwrappers.py +++ b/paste/wsgiwrappers.py @@ -8,7 +8,14 @@ to deal with an incoming request and sending a response. import re import warnings from pprint import pformat -from Cookie import SimpleCookie +try: + # Python 3 + from http.cookies import SimpleCookie +except ImportError: + # Python 2 + from Cookie import SimpleCookie +import six + from paste.request import EnvironHeaders, get_cookie_dict, \ parse_dict_querystring, parse_formvars from paste.util.multidict import MultiDict, UnicodeMultiDict @@ -78,18 +85,18 @@ class WSGIRequest(object): The class variable ``defaults`` specifies default values for ``charset``, ``errors``, and ``langauge``. These can be overridden for the current request via the registry. - + The ``language`` default value is considered the fallback during i18n translations to ensure in odd cases that mixed languages don't occur should the ``language`` file contain the string but not another language in the accepted languages list. The ``language`` value only applies when getting a list of accepted languages from the HTTP Accept header. - + This behavior is duplicated from Aquarium, and may seem strange but is - very useful. Normally, everything in the code is in "en-us". However, + very useful. Normally, everything in the code is in "en-us". However, the "en-us" translation catalog is usually empty. If the user requests ``["en-us", "zh-cn"]`` and a translation isn't found for a string in - "en-us", you don't want gettext to fallback to "zh-cn". You want it to + "en-us", you don't want gettext to fallback to "zh-cn". You want it to just use the string itself. Hence, if a string isn't found in the ``language`` catalog, the string in the source code will be used. @@ -106,7 +113,7 @@ class WSGIRequest(object): self.environ = environ # This isn't "state" really, since the object is derivative: self.headers = EnvironHeaders(environ) - + defaults = self.defaults._current_obj() self.charset = defaults.get('charset') if self.charset: @@ -118,7 +125,7 @@ class WSGIRequest(object): self.errors = defaults.get('errors', 'strict') self.decode_param_names = defaults.get('decode_param_names', False) self._languages = None - + body = environ_getter('wsgi.input') scheme = environ_getter('wsgi.url_scheme') method = environ_getter('REQUEST_METHOD') @@ -137,12 +144,12 @@ class WSGIRequest(object): else: return {} urlvars = property(urlvars, doc=urlvars.__doc__) - + def is_xhr(self): """Returns a boolean if X-Requested-With is present and a XMLHttpRequest""" return self.environ.get('HTTP_X_REQUESTED_WITH', '') == 'XMLHttpRequest' is_xhr = property(is_xhr, doc=is_xhr.__doc__) - + def host(self): """Host name provided in HTTP_HOST, with fall-back to SERVER_NAME""" return self.environ.get('HTTP_HOST', self.environ.get('SERVER_NAME')) @@ -150,7 +157,7 @@ class WSGIRequest(object): def languages(self): """Return a list of preferred languages, most preferred first. - + The list may be empty. """ if self._languages is not None: @@ -167,7 +174,7 @@ class WSGIRequest(object): self._languages = langs return self._languages languages = property(languages, doc=languages.__doc__) - + def _GET(self): return parse_dict_querystring(self.environ) @@ -247,7 +254,7 @@ class WSGIRequest(object): """Dictionary of cookies keyed by cookie name. Just a plain dictionary, may be empty but not None. - + """ return get_cookie_dict(self.environ) cookies = property(cookies, doc=cookies.__doc__) @@ -264,7 +271,7 @@ class WSGIRequest(object): def match_accept(self, mimetypes): """Return a list of specified mime-types that the browser's HTTP Accept header allows in the order provided.""" - return desired_matches(mimetypes, + return desired_matches(mimetypes, self.environ.get('HTTP_ACCEPT', '*/*')) def __repr__(self): @@ -294,10 +301,10 @@ class WSGIResponse(object): """ defaults = StackedObjectProxy( - default=dict(content_type='text/html', charset='utf-8', + default=dict(content_type='text/html', charset='utf-8', errors='strict', headers={'Cache-Control':'no-cache'}) ) - def __init__(self, content='', mimetype=None, code=200): + def __init__(self, content=b'', mimetype=None, code=200): self._iter = None self._is_str_iter = True @@ -329,14 +336,14 @@ class WSGIResponse(object): return '\n'.join(['%s: %s' % (key, value) for key, value in self.headers.headeritems()]) \ + '\n\n' + content - + def __call__(self, environ, start_response): """Convenience call to return output and set status information - + Conforms to the WSGI interface for calling purposes only. - + Example usage: - + .. code-block:: python def wsgi_app(environ, start_response): @@ -344,7 +351,7 @@ class WSGIResponse(object): response.write("Hello world") response.headers['Content-Type'] = 'latin1' return response(environ, start_response) - + """ status_text = STATUS_CODE_TEXT[self.status_code] status = '%s %s' % (self.status_code, status_text) @@ -358,7 +365,7 @@ class WSGIResponse(object): elif is_file: return iter(lambda: self.content.read(), '') return self.get_content() - + def determine_charset(self): """ Determine the encoding as specified by the Content-Type's charset @@ -367,7 +374,7 @@ class WSGIResponse(object): charset_match = _CHARSET_RE.search(self.headers.get('Content-Type', '')) if charset_match: return charset_match.group(1) - + def has_header(self, header): """ Case-insensitive check for a header @@ -403,7 +410,7 @@ class WSGIResponse(object): self.cookies[key]['max-age'] = 0 def _set_content(self, content): - if hasattr(content, '__iter__'): + if not isinstance(content, (six.binary_type, six.text_type)): self._iter = content if isinstance(content, list): self._is_str_iter = True @@ -428,7 +435,7 @@ class WSGIResponse(object): return encode_unicode_app_iter(self.content, charset, self.errors) else: return self.content - + def wsgi_response(self): """ Return this WSGIResponse as a tuple of WSGI formatted data, including: @@ -440,13 +447,13 @@ class WSGIResponse(object): for c in self.cookies.values(): response_headers.append(('Set-Cookie', c.output(header=''))) return status, response_headers, self.get_content() - + # The remaining methods partially implement the file-like object interface. # See http://docs.python.org/lib/bltin-file-objects.html def write(self, content): if not self._is_str_iter: - raise IOError, "This %s instance's content is not writable: (content " \ - 'is an iterator)' % self.__class__.__name__ + raise IOError("This %s instance's content is not writable: (content " + 'is an iterator)' % self.__class__.__name__) self.content.append(content) def flush(self): @@ -454,8 +461,8 @@ class WSGIResponse(object): def tell(self): if not self._is_str_iter: - raise IOError, 'This %s instance cannot tell its position: (content ' \ - 'is an iterator)' % self.__class__.__name__ + raise IOError('This %s instance cannot tell its position: (content ' + 'is an iterator)' % self.__class__.__name__) return sum([len(chunk) for chunk in self._iter]) ######################################## |