Commited all py papers into Google code

1 files changed, 783 insertions, 0 deletions

diff --git a/pypers/oxford/magic.txt b/pypers/oxford/magic.txt
new file mode 100755
index 0000000..9437ac5
--- /dev/null
+++ b/pypers/oxford/magic.txt
@@ -0,0 +1,783 @@
+Lecture 3: Magic (i.e. decorators and metaclasses)
+================================================================
+
+Part I: decorators
++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+Decorators are just sugar: their functionality was already in the language
+
+>>> def s(): pass
+>>> s = staticmethod(s)
+
+>>> @staticmethod
+... def s(): pass
+...
+
+However sugar *does* matter.
+
+A typical decorator: traced
+-----------------------------
+::
+
+ #<traced.py>
+
+ def traced(func):
+     def tracedfunc(*args, **kw):
+         print "calling %s.%s" % (func.__module__, func.__name__)
+         return func(*args, **kw)
+     tracedfunc.__name__ = func.__name__
+     return tracedfunc
+
+ @traced
+ def f(): pass
+
+ #</traced.py>
+
+>>> from traced import f
+>>> f()
+calling traced.f
+
+A decorator factory: Timed
+------------------------------------------
+
+::
+
+ #<timed.py>
+
+ import sys, time
+
+ class Timed(object):
+     """Decorator factory: each decorator object wraps a function and 
+     executes it many times (default 100 times).
+     The average time spent in one iteration, expressed in milliseconds, 
+     is stored in the attributes wrappedfunc.time and wrappedfunc.clocktime,
+     and displayed into a log file which defaults to stdout.
+     """
+     def __init__(self, repeat=100, logfile=sys.stdout):
+         self.repeat = repeat
+         self.logfile = logfile
+     def __call__(self, func):
+         def wrappedfunc(*args, **kw):
+             fullname = "%s.%s ..." % (func.__module__, func.func_name)
+             print >> self.logfile, 'Executing %s' % fullname.ljust(30),
+             time1 = time.time()
+             clocktime1 = time.clock()
+             for i in xrange(self.repeat):
+                 res = func(*args,**kw) # executes func self.repeat times
+             time2 = time.time()
+             clocktime2 = time.clock()
+             wrappedfunc.time = 1000*(time2-time1)/self.repeat
+             wrappedfunc.clocktime = 1000*(clocktime2 - clocktime1)/self.repeat
+             print >> self.logfile, \
+                   'Real time: %s ms;' % self.rounding(wrappedfunc.time),
+             print >> self.logfile, \
+                   'Clock time: %s ms' % self.rounding(wrappedfunc.clocktime)
+             return res
+         wrappedfunc.func_name = func.func_name
+         wrappedfunc.__module__ = func.__module__
+         return wrappedfunc
+     @staticmethod
+     def rounding(float_):
+         "Three digits rounding for small numbers, 1 digit rounding otherwise."
+         if float_ < 10.:
+             return "%5.3f" % float_
+         else:
+             return "%5.1f" % float_
+    
+ #</timed.py>
+
+>>> from timed import Timed
+>>> from random import sample
+>>> example_ls = sample(xrange(1000000), 1000)
+>>> @Timed()
+... def list_sort(ls):
+...     ls.sort()
+... 
+>>> list_sort(example_ls) #doctest: +ELLIPSIS
+Executing __main__.list_sort ... Real time: 0... ms; Clock time: 0... ms
+
+
+A powerful decorator pattern
+--------------------------------
+::
+
+ #<traced_function2.py>
+
+ from decorators import decorator
+
+ def trace(f, *args, **kw):
+     print "calling %s with args %s, %s" % (f.func_name, args, kw) 
+     return f(*args, **kw)
+
+ traced_function = decorator(trace)
+
+ @traced_function
+ def f1(x):
+     pass
+
+ @traced_function
+ def f2(x, y):
+     pass
+
+ #</traced_function2.py>
+
+>>> from traced_function2 import traced_function, f1, f2
+>>> f1(1)
+calling f1 with args (1,), {}
+>>> f2(1,2)
+calling f2 with args (1, 2), {}
+
+works with pydoc::
+
+ $ pydoc2.4 traced_function2.f2  
+ Help on function f1 in traced_function2:
+
+ traced_function2.f1 = f1(x)
+
+ $ pydoc2.4 traced_function2.f2  
+ Help on function f2 in traced_function2:
+
+ traced_function2.f2 = f2(x, y)
+
+Here is the source code::
+
+ #<decorators.py>
+ 
+ import inspect, itertools
+
+ def getinfo(func):
+     """Return an info dictionary containing:
+     - name (the name of the function : str)
+     - argnames (the names of the arguments : list)
+     - defarg (the values of the default arguments : list)
+     - fullsign (the full signature : str)
+     - shortsign (the short signature : str)
+     - arg0 ... argn (shortcuts for the names of the arguments)
+ 
+     >> def f(self, x=1, y=2, *args, **kw): pass
+ 
+     >> info = getinfo(f)
+ 
+     >> info["name"]
+     'f'
+     >> info["argnames"]
+     ['self', 'x', 'y', 'args', 'kw']
+     
+     >> info["defarg"]
+     (1, 2)
+ 
+     >> info["shortsign"]
+     'self, x, y, *args, **kw'
+     
+     >> info["fullsign"]
+     'self, x=defarg[0], y=defarg[1], *args, **kw'
+ 
+     >> info["arg0"], info["arg1"], info["arg2"], info["arg3"], info["arg4"]
+     ('self', 'x', 'y', 'args', 'kw')
+     """
+     assert inspect.ismethod(func) or inspect.isfunction(func)
+     regargs, varargs, varkwargs, defaults = inspect.getargspec(func)
+     argnames = list(regargs)
+     if varargs: argnames.append(varargs)
+     if varkwargs: argnames.append(varkwargs)
+     counter = itertools.count()
+     fullsign = inspect.formatargspec(
+         regargs, varargs, varkwargs, defaults,
+         formatvalue=lambda value: "=defarg[%i]" % counter.next())[1:-1]
+     shortsign = inspect.formatargspec(
+         regargs, varargs, varkwargs, defaults,
+         formatvalue=lambda value: "")[1:-1]
+     dic = dict(("arg%s" % n, name) for n, name in enumerate(argnames))
+     dic.update(name=func.__name__, argnames=argnames, shortsign=shortsign,
+         fullsign = fullsign, defarg = func.func_defaults or ())
+     return dic
+ 
+ def _contains_reserved_names(dic): # helper
+     return "_call_" in dic or "_func_" in dic
+ 
+ def _decorate(func, caller):
+     """Takes a function and a caller and returns the function
+     decorated with that caller. The decorated function is obtained
+     by evaluating a lambda function with the correct signature.
+     """
+     infodict = getinfo(func)
+     assert not _contains_reserved_names(infodict["argnames"]), \
+            "You cannot use _call_ or _func_ as argument names!"
+     execdict=dict(_func_=func, _call_=caller, defarg=func.func_defaults or ())
+     if func.__name__ == "<lambda>":
+         lambda_src = "lambda %(fullsign)s: _call_(_func_, %(shortsign)s)" \
+                      % infodict
+         dec_func = eval(lambda_src, execdict)
+     else:
+         func_src = """def %(name)s(%(fullsign)s):
+         return _call_(_func_, %(shortsign)s)""" % infodict
+         exec func_src in execdict 
+         dec_func = execdict[func.__name__]
+     dec_func.__doc__ = func.__doc__
+     dec_func.__dict__ = func.__dict__
+     return dec_func
+ 
+ class decorator(object):
+     """General purpose decorator factory: takes a caller function as
+ input and returns a decorator. A caller function is any function like this::
+ 
+     def caller(func, *args, **kw):
+         # do something
+         return func(*args, **kw)
+     
+ Here is an example of usage:
+ 
+     >> @decorator
+     .. def chatty(f, *args, **kw):
+     ..     print "Calling %r" % f.__name__
+     ..     return f(*args, **kw)
+     
+     >> @chatty
+     .. def f(): pass
+     ..
+     >> f()
+     Calling 'f'
+     """
+     def __init__(self, caller):
+         self.caller = caller
+     def __call__(self, func):
+         return _decorate(func, self.caller)
+ 
+ 
+ #</decorators.py>
+
+The possibilities of this pattern are endless.
+
+A deferred decorator
+-----------------------
+
+You want to execute a procedure only after a certain time delay (for instance
+for use within an asyncronous Web framework)::
+
+
+ #<deferred.py>
+ "Deferring the execution of a procedure (function returning None)"
+
+ import threading
+ from decorators import decorator
+
+ def deferred(nsec):
+     def call_later(func, *args, **kw):
+         return threading.Timer(nsec, func, args, kw).start()
+     return decorator(call_later)
+
+ @deferred(2)
+ def hello():
+     print "hello"
+
+ if __name__ == "__main__":
+     hello()    
+     print "Calling hello() ..."
+    
+
+ #</deferred.py>
+
+ $ python deferred.py
+
+Show an example of an experimental decorator based web framework
+(doctester_frontend).
+ 
+Part II: metaclasses
+++++++++++++++++++++++++++++++++++++++++++++++++++
+
+Metaclasses are there! Consider this example from a recent post on c.l.py::
+
+ #<BaseClass.py>
+
+ class BaseClass(object):
+    "Do something"
+
+ #</BaseClass.py>
+
+>>> import BaseClass # instead of 'from BaseClass import BaseClass'
+>>> class C(BaseClass): pass
+...
+Traceback (most recent call last):
+  ...
+TypeError: Error when calling the metaclass bases
+    module.__init__() takes at most 2 arguments (3 given)
+
+The reason for the error is that class ``C(BaseClass): pass`` is
+actually calling the ``type`` metaclass with three arguments::
+
+  C = type("C", (BaseClass,), {})
+
+``type.__new__`` tries to use ``type(BaseClass)`` as metaclass,
+but since BaseClass here is a module,  and ``ModuleType`` is not
+a metaclass, it cannot work. The error message reflects a conflict with 
+the signature of ModuleType which requires two parameters and not three.
+
+So even if you don't use them, you may want to know they exist.
+
+Rejuvenating old-style classes
+--------------------------------------------
+
+>>> class Old: pass
+>>> print type(Old)
+<type 'classobj'>
+
+>>> __metaclass__ = type # to rejuvenate class
+>>> class NotOld: pass
+...
+>>> print NotOld.__class__
+<type 'type'>
+
+A typical metaclass example: MetaTracer
+----------------------------------------
+
+::
+
+ #<metatracer.py>
+
+ import inspect
+ from decorators import decorator
+
+ @decorator
+ def traced(meth, *args, **kw):
+     cls = meth.__cls__
+     modname = meth.__module__ or cls.__module__
+     print "calling %s.%s.%s" % (modname, cls.__name__, meth.__name__)
+     return meth(*args, **kw)
+
+ class MetaTracer(type):            
+     def __init__(cls, name, bases, dic):
+         super(MetaTracer, cls).__init__(name, bases, dic)
+         for k, v in dic.iteritems():
+             if inspect.isfunction(v):
+                 v.__cls__ = cls # so we know in which class v was defined
+                 setattr(cls, k, traced(v))
+
+ #</metatracer.py>
+
+Usage: exploring classes in the standard library
+
+::
+
+ #<dictmixin.py>
+
+ from metatracer import MetaTracer
+ from UserDict import DictMixin
+
+ class TracedDM(DictMixin, object):
+     __metaclass__ = MetaTracer
+     def __getitem__(self, item):
+         return item
+     def keys(self): 
+         return [1,2,3]
+
+ #</dictmixin.py>
+
+>>> from dictmixin import TracedDM
+>>> print TracedDM()
+calling dictmixin.TracedDM.keys
+calling dictmixin.TracedDM.__getitem__
+calling dictmixin.TracedDM.__getitem__
+calling dictmixin.TracedDM.__getitem__
+{1: 1, 2: 2, 3: 3}
+
+Real life example: check overriding
+-------------------------------------
+::
+
+ #<check_overriding.py>
+
+ class Base(object):
+     a = 0
+
+ class CheckOverriding(type):
+     "Prints a message if we are overriding a name."
+     def __new__(mcl, name, bases, dic):
+         for name, val in dic.iteritems():
+             if name.startswith("__") and name.endswith("__"): 
+                 continue # ignore special names
+             a_base_has_name = True in (hasattr(base, name) for base in bases)
+             if a_base_has_name:
+                 print "AlreadyDefinedNameWarning: " + name
+         return super(CheckOverriding, mcl).__new__(mcl, name, bases, dic)
+
+ class MyClass(Base):
+     __metaclass__ = CheckOverriding
+     a = 1
+
+ class ChildClass(MyClass):
+     a = 2
+
+#</check_overriding.py>
+
+>>> import check_overriding
+AlreadyDefinedNameWarning: a
+AlreadyDefinedNameWarning: a
+
+LogFile
+---------------------------------------------
+::
+
+ #<logfile.py>
+
+ import subprocess
+
+ def memoize(func):
+     memoize_dic = {}
+     def wrapped_func(*args):
+         if args in memoize_dic:
+             return memoize_dic[args]
+         else:
+             result = func(*args)
+             memoize_dic[args] = result
+             return result
+     wrapped_func.__name__ = func.__name__
+     wrapped_func.__doc__ = func.__doc__
+     wrapped_func.__dict__ = func.__dict__
+     return wrapped_func
+
+ class Memoize(type): # Singleton is a special case of Memoize
+     @memoize
+     def __call__(cls, *args):
+         return super(Memoize, cls).__call__(*args)
+
+ class LogFile(file):
+     """Open a file for append."""
+     __metaclass__ = Memoize
+     def __init__(self, name = "/tmp/err.log"):
+         self.viewer_cmd = 'xterm -e less'.split()
+         super(LogFile, self).__init__(name, "a")
+
+     def display(self, *ls):
+         "Use 'less' to display the log file in a separate xterm."
+         print >> self, "\n".join(map(str, ls)); self.flush()
+         subprocess.call(self.viewer_cmd + [self.name])
+
+     def reset(self):
+         "Erase the log file."
+         print >> file(self.name, "w")
+
+ if __name__ == "__main__": # test
+     print >> LogFile(), "hello"
+     print >> LogFile(), "world"
+     LogFile().display()
+
+ #</logfile.py>
+
+ $ python logfile.py
+
+Cooperative hierarchies
+------------------------------
+
+::
+
+ #<cooperative_init.py>
+ 
+ """Given a hierarchy, makes __init__ cooperative.
+ The only change needed is to add a line
+ 
+    __metaclass__ = CooperativeInit
+ 
+ to the base class of your hierarchy."""
+ 
+ from decorators import decorator 
+
+ class CooperativeInit(type):
+     def __init__(cls, name, bases, dic):
+
+         @decorator
+         def make_cooperative(__init__, self, *args, **kw):
+             super(cls, self).__init__(*args, **kw)
+             __init__(self, *args, **kw)
+
+         __init__ = dic.get("__init__")
+         if __init__:
+             cls.__init__ = make_cooperative(__init__)
+
+ class Base:
+     __metaclass__ = CooperativeInit
+     def __init__(self):
+         print "B.__init__"
+
+ class C1(Base):
+     def __init__(self):
+         print "C1.__init__"
+
+ class C2(Base):
+    def __init__(self):
+        print "C2.__init__"
+
+ class D(C1, C2):
+     def __init__(self):
+         print "D.__init__"
+
+ #</cooperative_init.py>
+
+>>> from cooperative_init import D
+>>> d = D()
+B.__init__
+C2.__init__
+C1.__init__
+D.__init__
+
+Metaclass-enhanced modules
+----------------------------------------------------------------
+
+::
+
+ #<import_with_metaclass.py>
+ """
+ ``import_with_metaclass(metaclass, modulepath)`` generates
+ a new module from and old module, by enhancing all of its classes.
+ This is not perfect, but it should give you a start."""
+ 
+ import os, sys, inspect, types
+
+ def import_with_metaclass(metaclass, modulepath):
+     modname = os.path.basename(modulepath)[:-3] # simplistic
+     mod = types.ModuleType(modname)
+     locs = dict(
+         __module__ = modname,
+         __metaclass__ = metaclass,
+         object = metaclass("object", (), {}))
+     execfile(modulepath, locs)
+     for k, v in locs.iteritems():
+         if inspect.isclass(v): # otherwise it would be "__builtin__"
+             v.__module__ = "__dynamic__"
+         setattr(mod, k, v)
+     return mod
+  
+#</import_with_metaclass.py>
+
+>>> from import_with_metaclass import import_with_metaclass
+>>> from metatracer import MetaTracer
+>>> traced_optparse = import_with_metaclass(MetaTracer, 
+...     "/usr/lib/python2.4/optparse.py")
+>>> op = traced_optparse.OptionParser()
+calling __dynamic__.OptionParser.__init__
+calling __dynamic__.OptionContainer.__init__
+calling __dynamic__.OptionParser._create_option_list
+calling __dynamic__.OptionContainer._create_option_mappings
+calling __dynamic__.OptionContainer.set_conflict_handler
+calling __dynamic__.OptionContainer.set_description
+calling __dynamic__.OptionParser.set_usage
+calling __dynamic__.IndentedHelpFormatter.__init__
+calling __dynamic__.HelpFormatter.__init__
+calling __dynamic__.HelpFormatter.set_parser
+calling __dynamic__.OptionParser._populate_option_list
+calling __dynamic__.OptionParser._add_help_option
+calling __dynamic__.OptionContainer.add_option
+calling __dynamic__.Option.__init__
+calling __dynamic__.Option._check_opt_strings
+calling __dynamic__.Option._set_opt_strings
+calling __dynamic__.Option._set_attrs
+calling __dynamic__.OptionContainer._check_conflict
+calling __dynamic__.OptionParser._init_parsing_state
+
+traced_optparse is a dynamically generated module not leaving in the
+file system.
+
+Magic properties
+--------------------
+::
+
+ #<magicprop.py>
+
+ class MagicProperties(type):
+     def __init__(cls, name, bases, dic):
+         prop_names = set(name[3:] for name in dic
+                          if name.startswith("get")
+                          or name.startswith("set"))
+         for name in prop_names:
+             getter = getattr(cls, "get" + name, None)
+             setter = getattr(cls, "set" + name, None)
+             setattr(cls, name, property(getter, setter))
+
+ class Base(object):
+     __metaclass__ = MagicProperties
+     def getx(self):
+         return self._x
+     def setx(self, value):
+         self._x = value
+
+ class Child(Base):
+     def getx(self):
+         print "getting x"
+         return super(Child, self).getx() 
+     def setx(self, value):
+         print "setting x"
+         super(Child, self).setx(value) 
+
+ #</magicprop.py>
+
+>>> from magicprop import Child
+>>> c = Child()
+>>> c.x = 1
+setting x
+>>> print c.x
+getting x
+1
+
+Hack: evil properties
+------------------------------------
+
+::
+
+ #<evilprop.py>
+
+ def convert2property(name, bases, d):
+     return property(d.get('get'), d.get('set'),
+                     d.get('del'),d.get('__doc__'))
+
+ class C(object):
+     class x:
+         """An evil test property"""
+         __metaclass__ = convert2property
+         def get(self):
+             print 'Getting %s' % self._x
+             return self._x
+         def set(self, value):
+             self._x = value
+             print 'Setting to', value
+
+ #</evilprop.py>
+
+>>> from evilprop import C
+>>> c = C()
+>>> c.x = 5
+Setting to 5
+>>> c.x
+Getting 5
+5
+>>> print C.x.__doc__
+An evil test property
+
+Why I suggest *not* to use metaclasses in production code
+---------------------------------------------------------
+
+ + there are very few good use case for metaclasses in production code
+   (i.e. 99% of time you don't need them)
+
+ + they put a cognitive burden on the developer;
+
+ + a design without metaclasses is less magic and likely more robust;
+
+ + a design with metaclasses makes it difficult to use other metaclasses 
+   for debugging.
+
+As far as I know, string.Template is the only metaclass-enhanced class
+in the standard library; the metaclass is used to give the possibility to
+change the defaults::
+
+    delimiter = '$'
+    idpattern = r'[_a-z][_a-z0-9]*'
+
+in subclasses of Template.
+
+>>> from string import Template
+>>> from metatracer import MetaTracer
+>>> class TracedTemplate(Template):
+...     __metaclass__ = MetaTracer
+...
+Traceback (most recent call last):
+  ...
+TypeError: Error when calling the metaclass bases
+    metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases
+
+Solution: use a consistent metaclass
+
+>>> class GoodMeta(MetaTracer, type(Template)): pass
+...
+>>> class TracedTemplate(Template):
+...     __metaclass__ = GoodMeta
+
+
+Is there an automatic way of solving the conflict?
+---------------------------------------------------------------------
+
+Yes, but you really need to be a metaclass wizard.
+
+http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/204197
+
+>>> from noconflict import classmaker
+>>> class TracedTemplate(Template):
+...     __metaclass__ = classmaker((MetaTracer,))
+>>> print type(TracedTemplate)
+<class 'noconflict._MetaTracer_TemplateMetaclass'>
+
+::
+
+ #<noconflict.py>
+
+ import inspect, types, __builtin__
+ from skip_redundant import skip_redundant
+
+ memoized_metaclasses_map = {}
+
+ # utility function
+ def remove_redundant(metaclasses):
+    skipset = set([types.ClassType])
+    for meta in metaclasses: # determines the metaclasses to be skipped
+        skipset.update(inspect.getmro(meta)[1:])
+    return tuple(skip_redundant(metaclasses, skipset))
+
+ ##################################################################
+ ## now the core of the module: two mutually recursive functions ##
+ ##################################################################
+
+ def get_noconflict_metaclass(bases, left_metas, right_metas):
+     """Not intended to be used outside of this module, unless you know
+     what you are doing."""
+     # make tuple of needed metaclasses in specified priority order
+     metas = left_metas + tuple(map(type, bases)) + right_metas
+     needed_metas = remove_redundant(metas)
+
+     # return existing confict-solving meta, if any
+     if needed_metas in memoized_metaclasses_map:
+       return memoized_metaclasses_map[needed_metas]
+     # nope: compute, memoize and return needed conflict-solving meta
+     elif not needed_metas:         # wee, a trivial case, happy us
+         meta = type
+     elif len(needed_metas) == 1: # another trivial case
+        meta = needed_metas[0]
+     # check for recursion, can happen i.e. for Zope ExtensionClasses
+     elif needed_metas == bases: 
+         raise TypeError("Incompatible root metatypes", needed_metas)
+     else: # gotta work ...
+         metaname = '_' + ''.join([m.__name__ for m in needed_metas])
+         meta = classmaker()(metaname, needed_metas, {})
+     memoized_metaclasses_map[needed_metas] = meta
+     return meta
+
+ def classmaker(left_metas=(), right_metas=()):
+    def make_class(name, bases, adict):
+        metaclass = get_noconflict_metaclass(bases, left_metas, right_metas)
+        return metaclass(name, bases, adict)
+    return make_class
+
+ #################################################################
+ ## and now a conflict-safe replacement for 'type'              ## 
+ #################################################################
+ 
+ __type__=__builtin__.type # the aboriginal 'type'
+ # left available in case you decide to rebind __builtin__.type
+
+ class safetype(__type__):
+     # this is REALLY DEEP MAGIC
+     """Overrides the ``__new__`` method of the ``type`` metaclass, making the
+     generation of classes conflict-proof."""
+     def __new__(mcl, *args):
+         nargs = len(args)
+         if nargs == 1: # works as __builtin__.type
+             return __type__(args[0]) 
+         elif nargs == 3: # creates the class using the appropriate metaclass
+             n, b, d = args # name, bases and dictionary
+             meta = get_noconflict_metaclass(b, (mcl,), ()) 
+             if meta is mcl: # meta is trivial, dispatch to the default __new__
+                 return super(safetype, mcl).__new__(mcl, n, b, d)
+             else: # non-trivial metaclass, dispatch to the right __new__
+                 # (it will take a second round) # print mcl, meta
+                 return super(mcl, meta).__new__(meta, n, b, d)
+         else:
+             raise TypeError('%s() takes 1 or 3 arguments' % mcl.__name__)
+
+ #</noconflict.py>