diff options
| author | R. Tyler Ballance <tyler@slide.com> | 2009-04-16 22:32:41 -0700 |
|---|---|---|
| committer | R. Tyler Ballance <tyler@slide.com> | 2009-04-16 22:32:41 -0700 |
| commit | 45f570f03abfbe929a14a7c23e2aa2a39ce152c5 (patch) | |
| tree | 992e83fd05db4ab0231ec1a2257ba45b744376c5 | |
| parent | 2fc2c598e3ab45631739fe0f582fbfc18573b417 (diff) | |
| parent | 1c42c232c419001116a099e4a49032231fff38fe (diff) | |
| download | python-cheetah-45f570f03abfbe929a14a7c23e2aa2a39ce152c5.tar.gz | |
Merge branch 'next' of git@github.com:rtyler/cheetahv2.1.1
68 files changed, 6285 insertions, 1475 deletions
@@ -1,3 +1,6 @@ *.pyc *.swp *.so +www/*.html +build +*.py.bak @@ -1,16 +1,2 @@ -Known Bugs in Cheetah --------------------------- - - See the file CHANGES for a list of bugs that have been resolved. - - Developers: if a bug was significant and affected a released version of - Cheetah, be sure to note its fix in the CHANGES file! --------------------------- - -Dict method bug ---------------- -Do not use placeholder names identical to Python dict methods, or Cheetah will -return the wrong value. The most notorious names are $update, $keys, $values, -$items. A complete list is at http://docs.python.org/lib/typesmapping.html. -Note that future versions of Python may add dict methods. This bug is -difficult to fix given Cheetah's NameMapper, so it will remain around for a -long time. (MO 2006/02/11) +Please see http://bugs.communitycheetah.org @@ -1,6 +1,17 @@ -Please initial your changes (there's a key at bottom) and add a date for each -release -================================================================================ + +2.1.1 (April 16, 2009) + - Support __eq__() and __ne__() the way you might expect in src/Tools/RecursiveNull (patch suggested by Peter Warasin <peter@endian.com>) + - Applied patch to avoid hitting the filesystem to get the file modification time everytime a #include directive is processed (Jean-Baptiste Quenot <jbq@caraldi.com>) + - Applied patch to fix some annoying cases when Cheetah writes to stderr instead of propagating the exception (Jean-Baptiste Quenot <jbq@caraldi.com>) + - Added KDE editor support + - Applied patch to correct importHook behavior on Python 2.6 (reported/patched by Toshio Ernie Kuratomi <a.badger@gmail.com>) + - Correct unicode issue when calling/embedding unicode templates inside of other templtes (testcase Tests.Unicode.JPQ_UTF8_Test3. reported by Jean-Baptiste Quenot <jbq@caraldi.com>) + - Added --shbang option (e.g. "cheetah compile --shbang '#!/usr/bin/python2.6' ") + - Removed dependency on optik OptionParser in favor of builtin Python optparse module + - Introduction of the #transform directive for whole-document filtering + - Introduction of Cheetah.contrib.markdown and Cheetah.Filters.Markdown for outputting a markdown processed template (meant for #transform) + - Cheetah.Filters.CodeHighlighter, pygments-based code highlighting filter for use with #transform + - Addition of "useLegacyImportMode" compiler setting (defaulted to True) to allow for older (read: broken) import behavior 2.1.0.1 (March 27, 2009) - Fix inline import issue introduced in v2.1.0 diff --git a/SetupConfig.py b/SetupConfig.py index bea5a94..1aa8aa7 100644 --- a/SetupConfig.py +++ b/SetupConfig.py @@ -1,5 +1,5 @@ #-------Main Package Settings-----------# -name = "Cheetah Community Edition" +name = 'Cheetah' from src.Version import Version as version maintainer = "R. Tyler Ballance" author = "Tavis Rudd" @@ -11,7 +11,8 @@ packages = ['Cheetah', 'Cheetah.Tests', 'Cheetah.Tools', 'Cheetah.Utils', - 'Cheetah.Utils.optik', + 'Cheetah.contrib', + 'Cheetah.contrib.markdown', ] classifiers = [line.strip() for line in '''\ #Development Status :: 4 - Beta @@ -1,16 +1,9 @@ -Cheetah TODO list ------------------ -* If you are working on a task please put your initials at the end of the - description -* When a task is completed please remember to note it in the CHANGES file -* Unresolved bugs are listed in the BUGS file. Resolved bugs are be listed - in the CHANGES file if the bug is considered significant enough and it - affected a released version of Cheetah. - -Required for Cheetah 2.0 -======================== -- Replace Optik with Python's optparse. Optik license has been removed from - Users' Guide. +NOTE: Please see http://bugs.communitycheetah.org + for future feature requests/bugs/TODO + + +=============================================================================== +=============================================================================== Desired for Cheetah 2.0 ======================= @@ -38,7 +31,6 @@ TODO Items (many are just ideas. This is not an official roadmap!) leak from one fill to the next. - CheetahWrapper stuff: (MO) - * "cheetah compile --shbang '#!/usr/bin/python2.2'" * "cheetah preview [options] [FILES]" print template-specific portion of main method(s) to stdout, with line numbers based on the .py template module. Make a Template method to do the same thing, a la .generatedModuleCode(). diff --git a/contrib/editors/kde-cheetah.xml b/contrib/editors/kde-cheetah.xml new file mode 100644 index 0000000..2d3179d --- /dev/null +++ b/contrib/editors/kde-cheetah.xml @@ -0,0 +1,339 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE language SYSTEM "language.dtd" +[ + <!ENTITY name "[A-Za-z_:][\w.:_-]*"> + <!ENTITY ident "[A-Za-z_][A-Za-z_0-9]*"> + <!ENTITY entref "&(#[0-9]+|#[xX][0-9A-Fa-f]+|&name;);"> + <!ENTITY block "block|def"> + <!ENTITY idcmd "attr|del|encoding|errorCatcher|extends|implements|import|include(\s+raw)?|raise|shBang"> +]> +<!-- + This file is part of KDE's kate project. + + copyright : (C) 2008 by David Zaslavsky + email : dzaslavs@ellipsix.net + + This file is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. +--> +<!-- + You'll find the "Writing a Kate Highlighting XML File HOWTO" at + http://kate.kde.org/doc/hlhowto.php + This is a template for the XML format used for syntax highlight descriptions + for the Kate text editor (http://kate.kde.org), which is part of the KDE + desktop environment (http://www.kde.org). + + Use it as the base for your own syntax files. + + Look at language.dtd for some documentation of the allowed elements and their attributes. + There is also a description of how to validate your syntax file. + + You'll find the "Writing a Kate Highlighting XML File HOWTO" at + http://kate.kde.org/doc/hlhowto.php + --> + +<language version="1.00" kateversion="2.4" name="Cheetah" section="Markup" extensions="*.tmpl" mimetype="text/plain" author="David Zaslavsky" licence="GPL"> +<highlighting> + <list name="specialvars"> + <item>None</item> + <item>False</item> + <item>True</item> + </list> + <contexts> + <context name="Start" attribute="Normal Text" lineEndContext="#stay"> + <DetectSpaces/> + <IncludeRules context="FindCheetah"/> + <IncludeRules context="FindHTML"/> + </context> + <context name="FindCheetah" attribute="Normal Text" lineEndContext="#stay"> + <RegExpr attribute="Comment" String="##.*$" context="#stay"/> + <Detect2Chars char="#" char1="*" attribute="Comment" context="LongComment" beginRegion="MLComment"/> + <RegExpr String="#raw\s*($|#)" attribute="Command" context="Raw"/> + <RegExpr String="#from\s+&ident;\s+import\s+&ident;\s*($|#)" attribute="Command" context="#stay"/> + <RegExpr String="#block\s+&ident;\s*($|#)" attribute="Command" context="#stay" beginRegion="block"/> + <RegExpr String="#def\s+&ident;\s*($|#)" attribute="Command" context="#stay" beginRegion="def"/> + <RegExpr String="#block\s+&ident;\s*\(" attribute="Command" context="Def Expression" beginRegion="block"/> + <RegExpr String="#def\s+&ident;\s*\(" attribute="Command" context="Def Expression" beginRegion="def"/> + <StringDetect String="#for" attribute="Command" context="For Expression" beginRegion="for"/> + <StringDetect String="#if" attribute="Command" context="If Expression" beginRegion="if"/> + <RegExpr String="#(else\s+if|elif)" attribute="Command" context="Expression" endRegion="if" beginRegion="if"/> + <StringDetect String="#repeat" attribute="Command" context="Expression" beginRegion="repeat"/> + <StringDetect String="#unless" attribute="Command" context="Expression" beginRegion="unless"/> + <StringDetect String="#while" attribute="Command" context="Expression" beginRegion="while"/> + <RegExpr String="#end block(\s+&ident;)?\s*($|#)" attribute="Command" context="#stay" endRegion="block"/> + <RegExpr String="#end def(\s+&ident;)?\s*($|#)" attribute="Command" context="#stay" endRegion="def"/> + <RegExpr String="#end for\s*($|#)" attribute="Command" context="#stay" endRegion="for"/> + <RegExpr String="#end if\s*($|#)" attribute="Command" context="#stay" endRegion="if"/> + <RegExpr String="#end repeat\s*($|#)" attribute="Command" context="#stay" endRegion="repeat"/> + <RegExpr String="#end unless\s*($|#)" attribute="Command" context="#stay" endRegion="unless"/> + <RegExpr String="#end while\s*($|#)" attribute="Command" context="#stay" endRegion="while"/> + <!-- short-form block directives --> + <RegExpr String="#(block|def) &ident;:" attribute="Command" context="Short Block Body"/> + <!-- no-argument directives --> + <RegExpr String="#(break|breakpoint|cache|compiler-settings|continue|finally|else|indent|pass|raw|slurp|stop|try)\s*($|#)" attribute="Command" context="#stay"/> + <!-- one-argument directives --> + <RegExpr String="#(attr|del|encoding|errorCatcher|extends|implements|import|include(\s+raw)?|shBang)\s+&ident;\s*($|#)" attribute="Command" context="#stay"/> + <!-- expression-argument directives --> + <RegExpr String="#(assert|echo|except|filter|raise|set|silent)" attribute="Command" context="Expression"/> + <StringDetect String="<%" attribute="Element" context="Python"/> + <StringDetect String="<%=" attribute="Element" context="Python"/> + <DetectChar char="$" attribute="Placeholder" context="Placeholder"/> + </context> + <context name="FindHTML" attribute="Normal Text" lineEndContext="#stay"> + <DetectSpaces/> + <DetectIdentifier/> + <StringDetect attribute="Comment" context="HTMLComment" String="<!--" beginRegion="HTMLComment" /> + <StringDetect attribute="CDATA" context="CDATA" String="<![CDATA[" beginRegion="cdata" /> + <RegExpr attribute="Doctype" context="Doctype" String="<!DOCTYPE\s+" beginRegion="doctype" /> + <RegExpr attribute="Processing Instruction" context="PI" String="<\?[\w:-]*" beginRegion="pi" /> + <RegExpr attribute="Element" context="CSS" String="<style\b" insensitive="TRUE" beginRegion="style" /> + <RegExpr attribute="Element" context="JS" String="<script\b" insensitive="TRUE" beginRegion="script" /> + <RegExpr attribute="Element" context="El Open" String="<pre\b" insensitive="TRUE" beginRegion="pre" /> + <RegExpr attribute="Element" context="El Open" String="<div\b" insensitive="TRUE" beginRegion="div" /> + <RegExpr attribute="Element" context="El Open" String="<table\b" insensitive="TRUE" beginRegion="table" /> + <RegExpr attribute="Element" context="El Open" String="<&name;" /> + <RegExpr attribute="Element" context="El Close" String="</pre\b" insensitive="TRUE" endRegion="pre" /> + <RegExpr attribute="Element" context="El Close" String="</div\b" insensitive="TRUE" endRegion="div" /> + <RegExpr attribute="Element" context="El Close" String="</table\b" insensitive="TRUE" endRegion="table" /> + <RegExpr attribute="Element" context="El Close" String="</&name;" /> + <IncludeRules context="FindEntityRefs" /> + </context> + <context name="Directive" attribute="Command" lineEndContext="#pop"> + <DetectChar char="#" attribute="Command" context="#pop"/> + </context> + <context name="Expression" attribute="Command" lineEndContext="#pop"> + <DetectChar char="#" attribute="Command" context="#pop"/> + <DetectChar char="$" attribute="Placeholder" context="Placeholder"/> + <RangeDetect char="'" char1="'" attribute="String" context="#stay"/> + <RangeDetect char=""" char1=""" attribute="String" context="#stay"/> + <keyword String="specialvars" attribute="Special Variable" context="#stay"/> + </context> + <context name="Def Expression" attribute="Command" lineEndContext="#pop"> + <DetectChar char=")" attribute="Command" context="#stay"/> + <IncludeRules context="Expression"/> + </context> + <context name="For Expression" attribute="Command" lineEndContext="#pop"> + <StringDetect String="in" attribute="Command" context="#stay"/> + <IncludeRules context="Expression"/> + </context> + <context name="If Expression" attribute="Command" lineEndContext="#pop"> + <StringDetect String="then" attribute="Command" context="Then Clause" endRegion="if"/> + <IncludeRules context="Expression"/> + </context> + <context name="Then Clause" attribute="Normal Text" lineEndContext="#pop#pop"> + <DetectChar char="#" attribute="Command" context="#pop#pop"/> + <StringDetect String="else" attribute="Command" context="#stay"/> + <keyword String="specialvars" attribute="Special Variable" context="#stay"/> + <DetectChar char="$" attribute="Placeholder" context="Placeholder"/> + <IncludeRules context="FindHTML"/> + </context> + <context name="Short Block Body" attribute="Normal Text" lineEndContext="#pop#pop"> + <DetectChar char="#" attribute="Command" context="#pop#pop"/> + <DetectChar char="$" attribute="Placeholder" context="Placeholder"/> + <IncludeRules context="FindHTML"/> + </context> + <context name="Raw" attribute="Command" lineEndContext="Raw content"> + <DetectChar char="#" attribute="Command" context="Raw content"/> + </context> + <context name="Raw content" attribute="Normal Text" lineEndContext="#stay"> + <StringDetect String="#end raw" attribute="Command" context="#pop#pop#pop"/> + </context> + <context name="Placeholder" attribute="Placeholder" lineEndContext="#pop"> + <RegExpr String="([[({])" attribute="Placeholder" context="Placeholder Brackets"/> + <RegExpr String="[A-Za-z_][A-Za-z_0-9]*" attribute="Placeholder" context="#stay"/> + <DetectChar char="." attribute="Placeholder" context="#stay"/> + <DetectSpaces attribute="Normal Text" context="#pop"/> + <RegExpr String="." context="#pop" lookAhead="true"/> + </context> + <context name="Placeholder Brackets" attribute="Placeholder" lineEndContext="#pop"> + <RegExpr String="([[({])" attribute="Placeholder" context="Placeholder Brackets"/> + <DetectChar char="]" attribute="Placeholder" context="#pop"/> + <DetectChar char=")" attribute="Placeholder" context="#pop"/> + <DetectChar char="}" attribute="Placeholder" context="#pop"/> + <RangeDetect char="'" char1="'" attribute="String" context="#stay"/> + <RangeDetect char=""" char1=""" attribute="String" context="#stay"/> + <keyword String="specialvars" attribute="Special Variable" context="#stay"/> + <DetectChar char="#" attribute="Error" context="#stay"/> + </context> + <context name="ShortComment" attribute="Comment" lineEndContext="#pop"> + <DetectSpaces/> + <IncludeRules context="##Alerts" /> + </context> + <context name="LongComment" attribute="Comment" lineEndContext="#stay"> + <DetectSpaces/> + <Detect2Chars char="*" char1="#" attribute="Comment" context="#pop" endRegion="MLComment"/> + <IncludeRules context="##Alerts" /> + </context> + <context name="HTMLComment" attribute="Comment" lineEndContext="#stay"> + <DetectSpaces/> + <IncludeRules context="FindCheetah"/> + <IncludeRules context="##Alerts"/> + <DetectIdentifier/> + <StringDetect attribute="Comment" context="#pop" String="-->" endRegion="HTMLComment" /> + <RegExpr attribute="Error" context="#stay" String="-(-(?!->))+" /> + </context> + <context name="Python" attribute="Normal Text" lineEndContext="#stay"> + <Detect2Chars char="%" char1=">" attribute="Normal Text" context="#pop"/> + <IncludeRules context="##Python" includeAttrib="true"/> + </context> + <context name="FindEntityRefs" attribute="Normal Text" lineEndContext="#stay"> + <IncludeRules context="FindCheetah"/> + <RegExpr attribute="EntityRef" context="#stay" String="&entref;" /> + <AnyChar attribute="Error" context="#stay" String="&<" /> + </context> + <context name="FindPEntityRefs" attribute="Normal Text" lineEndContext="#stay"> + <IncludeRules context="FindCheetah"/> + <RegExpr attribute="EntityRef" context="#stay" String="&entref;" /> + <RegExpr attribute="PEntityRef" context="#stay" String="%&name;;" /> + <AnyChar attribute="Error" context="#stay" String="&%" /> + </context> + <context name="FindAttributes" attribute="Normal Text" lineEndContext="#stay"> + <IncludeRules context="FindCheetah"/> + <RegExpr attribute="Attribute" context="#stay" String="&name;" column="0"/> + <RegExpr attribute="Attribute" context="#stay" String="\s+&name;" /> + <DetectChar attribute="Attribute" context="Value" char="=" /> + </context> + <context name="CDATA" attribute="Normal Text" lineEndContext="#stay"> + <DetectSpaces/> + <DetectIdentifier/> + <IncludeRules context="FindCheetah"/> + <StringDetect attribute="CDATA" context="#pop" String="]]>" endRegion="cdata" /> + <StringDetect attribute="EntityRef" context="#stay" String="]]&gt;" /> + </context> + <context name="PI" attribute="Normal Text" lineEndContext="#stay"> + <IncludeRules context="FindCheetah"/> + <Detect2Chars attribute="Processing Instruction" context="#pop" char="?" char1=">" endRegion="pi" /> + </context> + <context name="Doctype" attribute="Normal Text" lineEndContext="#stay"> + <IncludeRules context="FindCheetah"/> + <DetectChar attribute="Doctype" context="#pop" char=">" endRegion="doctype" /> + <DetectChar attribute="Doctype" context="Doctype Internal Subset" char="[" beginRegion="int_subset" /> + </context> + <context name="Doctype Internal Subset" attribute="Normal Text" lineEndContext="#stay"> + <IncludeRules context="FindCheetah"/> + <DetectChar attribute="Doctype" context="#pop" char="]" endRegion="int_subset" /> + <StringDetect attribute="Comment" context="LongComment" String="<!--" beginRegion="MLComment" /> + <RegExpr attribute="Processing Instruction" context="PI" String="<\?[\w:-]*" beginRegion="pi" /> + <IncludeRules context="FindPEntityRefs" /> + </context> + <context name="Doctype Markupdecl" attribute="Normal Text" lineEndContext="#stay"> + <IncludeRules context="FindCheetah"/> + <DetectChar attribute="Doctype" context="#pop" char=">" /> + <DetectChar attribute="Value" context="Doctype Markupdecl DQ" char=""" /> + <DetectChar attribute="Value" context="Doctype Markupdecl SQ" char="'" /> + </context> + <context name="Doctype Markupdecl DQ" attribute="Value" lineEndContext="#stay"> + <IncludeRules context="FindCheetah"/> + <DetectChar attribute="Value" context="#pop" char=""" /> + <IncludeRules context="FindPEntityRefs" /> + </context> + <context name="Doctype Markupdecl SQ" attribute="Value" lineEndContext="#stay"> + <IncludeRules context="FindCheetah"/> + <DetectChar attribute="Value" context="#pop" char="'" /> + <IncludeRules context="FindPEntityRefs" /> + </context> + <context name="El Open" attribute="Normal Text" lineEndContext="#stay"> + <IncludeRules context="FindCheetah"/> + <Detect2Chars attribute="Element" context="#pop" char="/" char1=">" /> + <DetectChar attribute="Element" context="#pop" char=">" /> + <IncludeRules context="FindAttributes" /> + <RegExpr attribute="Error" context="#stay" String="\S" /> + </context> + <context name="El Close" attribute="Normal Text" lineEndContext="#stay"> + <IncludeRules context="FindCheetah"/> + <DetectChar attribute="Element" context="#pop" char=">" /> + <RegExpr attribute="Error" context="#stay" String="\S" /> + </context> + <context name="El Close 2" attribute="Normal Text" lineEndContext="#stay"> + <IncludeRules context="FindCheetah"/> + <DetectChar attribute="Element" context="#pop#pop#pop" char=">" /> + <RegExpr attribute="Error" context="#stay" String="\S" /> + </context> + <context name="El Close 3" attribute="Normal Text" lineEndContext="#stay"> + <IncludeRules context="FindCheetah"/> + <DetectChar attribute="Element" context="#pop#pop#pop#pop" char=">" /> + <RegExpr attribute="Error" context="#stay" String="\S" /> + </context> + <context name="CSS" attribute="Normal Text" lineEndContext="#stay"> + <IncludeRules context="FindCheetah"/> + <Detect2Chars attribute="Element" context="#pop" char="/" char1=">" endRegion="style" /> + <DetectChar attribute="Element" context="CSS content" char=">" /> + <IncludeRules context="FindAttributes" /> + <RegExpr attribute="Error" context="#stay" String="\S" /> + </context> + <context name="CSS content" attribute="Normal Text" lineEndContext="#stay"> + <IncludeRules context="FindCheetah"/> + <RegExpr attribute="Element" context="El Close 2" String="</style\b" insensitive="TRUE" endRegion="style" /> + <IncludeRules context="##CSS" includeAttrib="true"/> + </context> + <context name="JS" attribute="Normal Text" lineEndContext="#stay"> + <IncludeRules context="FindCheetah"/> + <Detect2Chars attribute="Element" context="#pop" char="/" char1=">" endRegion="script" /> + <DetectChar attribute="Element" context="JS content" char=">" /> + <IncludeRules context="FindAttributes" /> + <RegExpr attribute="Error" context="#stay" String="\S" /> + </context> + <context name="JS content" attribute="Normal Text" lineEndContext="#stay"> + <IncludeRules context="FindCheetah"/> + <RegExpr attribute="Element" context="El Close 2" String="</script\b" insensitive="TRUE" endRegion="script" /> + <RegExpr attribute="Comment" context="JS comment close" String="//(?=.*</script\b)" insensitive="TRUE" /> + <IncludeRules context="##JavaScript" includeAttrib="true"/> + </context> + <context name="JS comment close" attribute="Comment" lineEndContext="#pop"> + <IncludeRules context="FindCheetah"/> + <RegExpr attribute="Element" context="El Close 3" String="</script\b" insensitive="TRUE" endRegion="script" /> + <IncludeRules context="##Alerts" /> + </context> + <context name="Value" attribute="Normal Text" lineEndContext="#stay" fallthrough="true" fallthroughContext="Value NQ"> + <IncludeRules context="FindCheetah"/> + <DetectChar attribute="Value" context="Value DQ" char=""" /> + <DetectChar attribute="Value" context="Value SQ" char="'" /> + <DetectSpaces /> + </context> + <context name="Value NQ" attribute="Normal Text" lineEndContext="#pop#pop" fallthrough="true" fallthroughContext="#pop#pop"> + <IncludeRules context="FindCheetah"/> + <IncludeRules context="FindEntityRefs" /> + <RegExpr attribute="Value" context="#stay" String="/(?!>)" /> + <RegExpr attribute="Value" context="#stay" String="[^/><"'\s]" /> + </context> + <context name="Value DQ" attribute="Value" lineEndContext="#stay"> + <IncludeRules context="FindCheetah"/> + <DetectChar attribute="Value" context="#pop#pop" char=""" /> + <IncludeRules context="FindEntityRefs" /> + </context> + <context name="Value SQ" attribute="Value" lineEndContext="#stay"> + <IncludeRules context="FindCheetah"/> + <DetectChar attribute="Value" context="#pop#pop" char="'" /> + <IncludeRules context="FindEntityRefs" /> + </context> + </contexts> + <itemDatas> + <itemData name="Normal Text" defStyleNum="dsNormal"/> + <itemData name="Command" defStyleNum="dsFunction"/> + <itemData name="Comment" defStyleNum="dsComment"/> + <itemData name="String" defStyleNum="dsString"/> + <itemData name="Special Variable" defStyleNum="dsOthers"/> + <itemData name="Placeholder" defStyleNum="dsKeyword" color="#5555ff" selColor="#ffffff" bold="0" italic="0" /> + <itemData name="CDATA" defStyleNum="dsBaseN" bold="1" /> + <itemData name="Processing Instruction" defStyleNum="dsKeyword" /> + <itemData name="Doctype" defStyleNum="dsDataType" bold="1" /> + <itemData name="Element" defStyleNum="dsKeyword" /> + <itemData name="Attribute" defStyleNum="dsOthers" /> + <itemData name="Value" defStyleNum="dsString" color="#a00" /> + <itemData name="EntityRef" defStyleNum="dsDecVal" /> + <itemData name="PEntityRef" defStyleNum="dsDecVal" /> + <itemData name="Error" defStyleNum="dsError" /> + </itemDatas> +</highlighting> +<general> + <keywords casesensitive="1" additionalDeliminator="#"/> + <comments> + <comment name="singleLine" start="##"/> + <comment name="multiLine" start="#*" end="*#" region="MLComment"/> + <comment name="multiLine" start="<!--" end="-->" region="HTMLComment"/> + </comments> +</general> +</language> +<!-- kate: space-indent on; indent-width 4; replace-tabs on; indent-mode xml; --> diff --git a/docs/devel_guide_src/.cvsignore b/docs/devel_guide_src/.cvsignore deleted file mode 100644 index 19a84ff..0000000 --- a/docs/devel_guide_src/.cvsignore +++ /dev/null @@ -1,4 +0,0 @@ -.cvsignore -*.aux -*.l2h -devel_guide diff --git a/docs/users_guide_2_src/.cvsignore b/docs/users_guide_2_src/.cvsignore deleted file mode 100644 index 2d19fc7..0000000 --- a/docs/users_guide_2_src/.cvsignore +++ /dev/null @@ -1 +0,0 @@ -*.html diff --git a/docs/users_guide_src/.cvsignore b/docs/users_guide_src/.cvsignore deleted file mode 100644 index 2e19d98..0000000 --- a/docs/users_guide_src/.cvsignore +++ /dev/null @@ -1,4 +0,0 @@ -.cvsignore -*.aux -*.l2h -users_guide diff --git a/src/.cvsignore b/src/.cvsignore deleted file mode 100644 index d5bd613..0000000 --- a/src/.cvsignore +++ /dev/null @@ -1,2 +0,0 @@ -.cvsignore -*.pyc diff --git a/src/CheetahWrapper.py b/src/CheetahWrapper.py index 5e3d6dd..5a37f5e 100755 --- a/src/CheetahWrapper.py +++ b/src/CheetahWrapper.py @@ -18,11 +18,11 @@ __revision__ = "$Revision: 1.26 $"[11:-2] import getopt, glob, os, pprint, re, shutil, sys import cPickle as pickle +from optparse import OptionParser from Cheetah.Version import Version from Cheetah.Template import Template, DEFAULT_COMPILER_SETTINGS from Cheetah.Utils.Misc import mkdirsWithPyInitFiles -from Cheetah.Utils.optik import OptionParser optionDashesRE = re.compile( R"^-{1,2}" ) moduleNameRE = re.compile( R"^[a-zA-Z_][a-zA-Z_0-9]*$" ) @@ -53,18 +53,6 @@ class Bundle: return "<Bundle %r>" % self.__dict__ -class MyOptionParser(OptionParser): - standard_option_list = [] # We use commands for Optik's standard options. - - def error(self, msg): - """Print our usage+error page.""" - usage(HELP_PAGE2, msg) - - def print_usage(self, file=None): - """Our usage+error page already has this.""" - pass - - ################################################## ## USAGE FUNCTION & MESSAGES @@ -108,36 +96,10 @@ If FILES is a single "-", read standard input and write standard output. Run "cheetah options" for the list of valid options. """ -HELP_PAGE2 = """\ -OPTIONS FOR "compile" AND "fill": ---------------------------------- - --idir DIR, --odir DIR : input/output directories (default: current dir) - --iext EXT, --oext EXT : input/output filename extensions - (default for compile: tmpl/py, fill: tmpl/html) - -R : recurse subdirectories looking for input files - --debug : print lots of diagnostic output to standard error - --env : put the environment in the searchList - --flat : no destination subdirectories - --nobackup : don't make backups - --pickle FILE : unpickle FILE and put that object in the searchList - --stdout, -p : output to standard output (pipe) - --settings : a string representing the compiler settings to use - e.g. --settings='useNameMapper=False,useFilters=False' - This string is eval'd in Python so it should contain - valid Python syntax. - --templateAPIClass : a string representing a subclass of - Cheetah.Template:Template to use for compilation - - --parallel : compile or fill templates in parallel, e.g. - --parallel 4 - -Run "cheetah help" for the main help screen. -""" - ################################################## ## CheetahWrapper CLASS -class CheetahWrapper: +class CheetahWrapper(object): MAKE_BACKUPS = True BACKUP_SUFFIX = ".bak" _templateClass = None @@ -150,6 +112,7 @@ class CheetahWrapper: self.pathArgs = None self.sourceFiles = [] self.searchList = [] + self.parser = None ################################################## ## MAIN ROUTINE @@ -192,24 +155,25 @@ class CheetahWrapper: C, D, W = self.chatter, self.debug, self.warn self.isCompile = isCompile = self.command[0] == 'c' defaultOext = isCompile and ".py" or ".html" - parser = MyOptionParser() - pao = parser.add_option - pao("--idir", action="store", dest="idir", default="") - pao("--odir", action="store", dest="odir", default="") - pao("--iext", action="store", dest="iext", default=".tmpl") - pao("--oext", action="store", dest="oext", default=defaultOext) - pao("-R", action="store_true", dest="recurse", default=False) - pao("--stdout", "-p", action="store_true", dest="stdout", default=False) - pao("--debug", action="store_true", dest="debug", default=False) - pao("--env", action="store_true", dest="env", default=False) - pao("--pickle", action="store", dest="pickle", default="") - pao("--flat", action="store_true", dest="flat", default=False) - pao("--nobackup", action="store_true", dest="nobackup", default=False) - pao("--settings", action="store", dest="compilerSettingsString", default=None) - pao("--templateAPIClass", action="store", dest="templateClassName", default=None) - pao("--parallel", action="store", type="int", dest="parallel", default=1) - - self.opts, self.pathArgs = opts, files = parser.parse_args(args) + self.parser = OptionParser() + pao = self.parser.add_option + pao("--idir", action="store", dest="idir", default='', help='Input directory (defaults to current directory)') + pao("--odir", action="store", dest="odir", default="", help='Output directory (defaults to current directory)') + pao("--iext", action="store", dest="iext", default=".tmpl", help='File input extension (defaults: compile: .tmpl, fill: .tmpl)') + pao("--oext", action="store", dest="oext", default=defaultOext, help='File output extension (defaults: compile: .py, fill: .html)') + pao("-R", action="store_true", dest="recurse", default=False, help='Recurse through subdirectories looking for input files') + pao("--stdout", "-p", action="store_true", dest="stdout", default=False, help='Verbosely print informational messages to stdout') + pao("--debug", action="store_true", dest="debug", default=False, help='Print diagnostic/debug information to stderr') + pao("--env", action="store_true", dest="env", default=False, help='Pass the environment into the search list') + pao("--pickle", action="store", dest="pickle", default="", help='Unpickle FILE and pass it through in the search list') + pao("--flat", action="store_true", dest="flat", default=False, help='Do not build destination subdirectories') + pao("--nobackup", action="store_true", dest="nobackup", default=False, help='Do not make backup files when generating new ones') + pao("--settings", action="store", dest="compilerSettingsString", default=None, help='String of compiler settings to pass through, e.g. --settings="useNameMapper=False,useFilters=False"') + pao("--templateAPIClass", action="store", dest="templateClassName", default=None, help='Name of a subclass of Cheetah.Template.Template to use for compilation, e.g. MyTemplateClass') + pao("--parallel", action="store", type="int", dest="parallel", default=1, help='Compile/fill templates in parallel, e.g. --parallel=4') + pao('--shbang', dest='shbang', default='#!/usr/bin/env python', help='Specify the shbang to place at the top of compiled templates, e.g. --shbang="#!/usr/bin/python2.6"') + + self.opts, self.pathArgs = opts, files = self.parser.parse_args(args) D("""\ cheetah compile %s Options are @@ -252,7 +216,7 @@ Files are %s""", args, pprint.pformat(vars(opts)), files) usage(HELP_PAGE1, "", sys.stdout) def options(self): - usage(HELP_PAGE2, "", sys.stdout) + return self.parser.print_help() def test(self): # @@MO: Ugly kludge. @@ -604,6 +568,7 @@ be named according to the same rules as Python modules.""" % tup) pysrc = TemplateClass.compile(file=src, returnAClass=False, moduleName=basename, className=basename, + commandlineopts=self.opts, compilerSettings=compilerSettings) output = pysrc else: diff --git a/src/Compiler.py b/src/Compiler.py index a4f311c..71eeeb9 100644 --- a/src/Compiler.py +++ b/src/Compiler.py @@ -63,6 +63,7 @@ DEFAULT_COMPILER_SETTINGS = { 'alwaysFilterNone':True, # filter out None, before the filter is called 'useFilters':True, # use str instead if =False 'includeRawExprInFilterArgs':True, + 'useLegacyImportMode' : True, #'lookForTransactionAttr':False, 'autoAssignDummyTransactionToSelf':False, @@ -973,9 +974,16 @@ class MethodCompiler(GenUtils): def nextFilterRegionID(self): return self.nextCacheID() + + def setTransform(self, transformer, isKlass): + self.addChunk('trans = TransformerTransaction()') + self.addChunk('trans._response = trans.response()') + self.addChunk('trans._response._filter = %s' % transformer) + self.addChunk('write = trans._response.write') def setFilter(self, theFilter, isKlass): - class FilterDetails: pass + class FilterDetails: + pass filterDetails = FilterDetails() filterDetails.ID = ID = self.nextFilterRegionID() filterDetails.theFilter = theFilter @@ -1566,8 +1574,8 @@ class ModuleCompiler(SettingsManager, GenUtils): self._fileDirName, self._fileBaseName = os.path.split(self._filePath) self._fileBaseNameRoot, self._fileBaseNameExt = os.path.splitext(self._fileBaseName) - if not isinstance(source, (str,unicode)): - source = str(source) + if not isinstance(source, basestring): + source = unicode(source) # by converting to string here we allow objects such as other Templates # to be passed in @@ -1638,7 +1646,7 @@ class ModuleCompiler(SettingsManager, GenUtils): "from Cheetah.Version import MinCompatibleVersion as RequiredCheetahVersion", "from Cheetah.Version import MinCompatibleVersionTuple as RequiredCheetahVersionTuple", "from Cheetah.Template import Template", - "from Cheetah.DummyTransaction import DummyTransaction", + "from Cheetah.DummyTransaction import *", "from Cheetah.NameMapper import NotFound, valueForName, valueFromSearchList, valueFromFrameOrSearchList", "from Cheetah.CacheRegion import CacheRegion", "import Cheetah.Filters as Filters", @@ -1712,9 +1720,10 @@ class ModuleCompiler(SettingsManager, GenUtils): return self._importedVarNames def addImportedVarNames(self, varNames, raw_statement=None): + settings = self.settings() if not varNames: return - if self._methodBodyChunks and raw_statement: + if self._methodBodyChunks and raw_statement and not settings.get('useLegacyImportMode'): self.addChunk(raw_statement) else: self._importedVarNames.extend(varNames) @@ -1836,7 +1845,11 @@ class ModuleCompiler(SettingsManager, GenUtils): self._specialVars[name] = contents.strip() def addImportStatement(self, impStatement): - self._importStatements.append(impStatement) + settings = self.settings() + if not self._methodBodyChunks or settings.get('useLegacyImportMode'): + # In the case where we are importing inline in the middle of a source block + # we don't want to inadvertantly import the module at the top of the file either + self._importStatements.append(impStatement) #@@TR 2005-01-01: there's almost certainly a cleaner way to do this! importVarNames = impStatement[impStatement.find('import') + len('import'):].split(',') diff --git a/src/DummyTransaction.py b/src/DummyTransaction.py index 8ebe33d..9273c8c 100644 --- a/src/DummyTransaction.py +++ b/src/DummyTransaction.py @@ -15,6 +15,8 @@ Last Revision Date: $Date: 2005/11/13 01:12:13 $ __author__ = "Tavis Rudd <tavis@damnsimple.com>" __revision__ = "$Revision: 1.13 $"[11:-2] +import types + def flush(): pass @@ -56,3 +58,41 @@ class DummyTransaction: def response(resp=DummyResponse()): return resp self.response = response + +class TransformerResponse(object): + def __init__(self, *args, **kwargs): + self._output = [] + self._filter = None + + def write(self, value): + self._output.append(value) + + def flush(self): + pass + + def writeln(self, line): + self.write(line) + self.write('\n') + + def writelines(self, *lines): + [self.writeln(line) for line in lines] + + def getvalue(self, **kwargs): + output = kwargs.get('outputChunks') or self._output + rc = ''.join(output) + if self._filter: + _filter = self._filter + if isinstance(_filter, types.TypeType): + _filter = _filter() + return _filter.filter(rc) + return rc + + +class TransformerTransaction(object): + def __init__(self, *args, **kwargs): + self._response = None + def response(self): + if self._response: + return self._response + return TransformerResponse() + diff --git a/src/Filters.py b/src/Filters.py index 6467245..041c2a5 100644 --- a/src/Filters.py +++ b/src/Filters.py @@ -1,27 +1,17 @@ #!/usr/bin/env python -# $Id: Filters.py,v 1.33 2007/12/29 23:08:18 tavis_rudd Exp $ -"""Filters for the #filter directive; output filters Cheetah's $placeholders . - -Meta-Data -================================================================================ -Author: Tavis Rudd <tavis@damnsimple.com> -Version: $Revision: 1.33 $ -Start Date: 2001/08/01 -Last Revision Date: $Date: 2007/12/29 23:08:18 $ -""" -__author__ = "Tavis Rudd <tavis@damnsimple.com>" -__revision__ = "$Revision: 1.33 $"[11:-2] +''' + Filters for the #filter directive as well as #transform + + #filter results in output filters Cheetah's $placeholders . + #transform results in a filter on the entirety of the output +''' +import sys +import Cheetah.contrib # Additional entities WebSafe knows how to transform. No need to include # '<', '>' or '&' since those will have been done already. webSafeEntities = {' ': ' ', '"': '"'} -class Error(Exception): - pass - -################################################## -## BASE CLASS - class Filter(object): """A baseclass for the Cheetah Filters.""" @@ -49,13 +39,14 @@ class Filter(object): elif val is None: filtered = '' else: - filtered = str(val) + try: + filtered = str(val) + except UnicodeEncodeError: + filtered = unicode(val) return filtered RawOrEncodedUnicode = Filter -################################################## -## ENHANCED FILTERS class EncodeUnicode(Filter): def filter(self, val, @@ -73,18 +64,93 @@ class EncodeUnicode(Filter): >>> print t """ if isinstance(val, unicode): - filtered = val.encode(encoding) - elif val is None: - filtered = '' - else: - filtered = str(val) - return filtered + return val.encode(encoding) + if val is None: + return '' + return str(val) + + +class Markdown(EncodeUnicode): + ''' + Markdown will change regular strings to Markdown + (http://daringfireball.net/projects/markdown/) + + Such that: + My Header + ========= + Becaomes: + <h1>My Header</h1> + + and so on. + + Markdown is meant to be used with the #transform + tag, as it's usefulness with #filter is marginal at + best + ''' + def filter(self, value, **kwargs): + # This is a bit of a hack to allow outright embedding of the markdown module + try: + markdown_path = '/'.join(Cheetah.contrib.__file__.split('/')[:-1]) + sys.path.append(markdown_path) + from Cheetah.contrib import markdown + sys.path.pop() + except: + print '>>> Exception raised importing the "markdown" module' + print '>>> Are you sure you have the ElementTree module installed?' + print ' http://effbot.org/downloads/#elementtree' + raise + + encoded = super(Markdown, self).filter(value, **kwargs) + return markdown.markdown(encoded) + +class CodeHighlighter(EncodeUnicode): + ''' + The CodeHighlighter filter depends on the "pygments" module which you can + download and install from: http://pygments.org + + What the CodeHighlighter assumes the string that it's receiving is source + code and uses pygments.lexers.guess_lexer() to try to guess which parser + to use when highlighting it. + + CodeHighlighter will return the HTML and CSS to render the code block, syntax + highlighted, in a browser + + NOTE: I had an issue installing pygments on Linux/amd64/Python 2.6 dealing with + importing of pygments.lexers, I was able to correct the failure by adding: + raise ImportError + to line 39 of pygments/plugin.py (since importing pkg_resources was causing issues) + ''' + def filter(self, source, **kwargs): + encoded = super(CodeHighlighter, self).filter(source, **kwargs) + try: + from pygments import highlight + from pygments import lexers + from pygments import formatters + except ImportError, ex: + print '<%s> - Failed to import pygments! (%s)' % (self.__class__.__name__, ex) + print '-- You may need to install it from: http://pygments.org' + return encoded + + lexer = None + try: + lexer = lexers.guess_lexer(source) + except lexers.ClassNotFound: + lexer = lexers.PythonLexer() + + formatter = formatters.HtmlFormatter(cssclass='code_highlighter') + encoded = highlight(encoded, lexer, formatter) + css = formatter.get_style_defs('.code_highlighter') + return '''<style type="text/css"><!-- + %(css)s + --></style>%(source)s''' % {'css' : css, 'source' : encoded} + + class MaxLen(Filter): def filter(self, val, **kw): """Replace None with '' and cut off at maxlen.""" - output = super(MaxLen, self).filter(val, **kw) + output = super(MaxLen, self).filter(val, **kw) if kw.has_key('maxlen') and len(output) > kw['maxlen']: return output[:kw['maxlen']] return output @@ -93,7 +159,7 @@ class WebSafe(Filter): """Escape HTML entities in $placeholders. """ def filter(self, val, **kw): - s = super(WebSafe, self).filter(val, **kw) + s = super(WebSafe, self).filter(val, **kw) # These substitutions are copied from cgi.escape(). s = s.replace("&", "&") # Must be done first! s = s.replace("<", "<") @@ -127,7 +193,7 @@ class Strip(Filter): with the proposed #sed directive (which has not been ratified yet.) """ def filter(self, val, **kw): - s = super(Strip, self).filter(val, **kw) + s = super(Strip, self).filter(val, **kw) result = [] start = 0 # The current line will be s[start:end]. while 1: # Loop through each line. @@ -150,7 +216,7 @@ class StripSqueeze(Filter): input is joined into one ling line with NO trailing newline. """ def filter(self, val, **kw): - s = super(StripSqueeze, self).filter(val, **kw) + s = super(StripSqueeze, self).filter(val, **kw) s = s.split() return " ".join(s) diff --git a/src/ImportManager.py b/src/ImportManager.py index eaf398a..af9d8fc 100755 --- a/src/ImportManager.py +++ b/src/ImportManager.py @@ -407,7 +407,12 @@ class ImportManager: __builtin__.__import__ = self.importHook __builtin__.reload = self.reloadHook - def importHook(self, name, globals=None, locals=None, fromlist=None): + def importHook(self, name, globals=None, locals=None, fromlist=None, level=-1): + ''' + NOTE: Currently importHook will accept the keyword-argument "level" + but it will *NOT* use it (currently). Details about the "level" keyword + argument can be found here: http://www.python.org/doc/2.5.2/lib/built-in-funcs.html + ''' # first see if we could be importing a relative name #print "importHook(%s, %s, locals, %s)" % (name, globals['__name__'], fromlist) _sys_modules_get = sys.modules.get diff --git a/src/Parser.py b/src/Parser.py index 4e9e4db..0b8097f 100644 --- a/src/Parser.py +++ b/src/Parser.py @@ -172,6 +172,7 @@ directiveNamesAndParsers = { 'filter': 'eatFilter', 'echo': None, 'silent': None, + 'transform' : 'eatTransform', 'call': 'eatCall', 'arg': 'eatCallArg', @@ -2473,6 +2474,30 @@ class _HighLevelParser(_LowLevelParser): self.pushToOpenDirectivesStack("filter") self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) self._compiler.setFilter(theFilter, isKlass) + + def eatTransform(self): + isLineClearToStartToken = self.isLineClearToStartToken() + endOfFirstLinePos = self.findEOL() + + self.getDirectiveStartToken() + self.advance(len('transform')) + self.getWhiteSpace() + startPos = self.pos() + if self.matchCheetahVarStart(): + isKlass = True + transformer = self.getExpression(pyTokensToBreakAt=[':']) + else: + isKlass = False + transformer = self.getIdentifier() + self.getWhiteSpace() + transformer = self._applyExpressionFilters(transformer, 'transform', startPos=startPos) + + if self.peek()==':': + self.advance() + self.getWhiteSpace() + self._eatRestOfDirectiveTag(isLineClearToStartToken, endOfFirstLinePos) + self._compiler.setTransform(transformer, isKlass) + def eatErrorCatcher(self): isLineClearToStartToken = self.isLineClearToStartToken() diff --git a/src/Template.py b/src/Template.py index 9d4114c..f691d88 100644 --- a/src/Template.py +++ b/src/Template.py @@ -71,8 +71,15 @@ from Cheetah.Utils.WebInputMixin import _Converter, _lookup, NonNumericInputErro from Cheetah.Unspecified import Unspecified -class Error(Exception): pass -class PreprocessError(Error): pass +# Decide whether to use the file modification time in file's cache key +__checkFileMtime = True +def checkFileMtime(value): + globals()['__checkFileMtime'] = value + +class Error(Exception): + pass +class PreprocessError(Error): + pass def hashList(l): hashedList = [] @@ -339,7 +346,7 @@ class Template(Servlet): preprocessors=Unspecified, cacheModuleFilesForTracebacks=Unspecified, cacheDirForModuleFiles=Unspecified, - + commandlineopts=None, keepRefToGeneratedCode=Unspecified, ): @@ -617,7 +624,6 @@ class Template(Servlet): vt(compilerSettings, 'compilerSettings', [D], 'dictionary') compilerClass = valOrDefault(compilerClass, klass._getCompilerClass(source, file)) - preprocessors = valOrDefault(preprocessors, klass._CHEETAH_preprocessors) keepRefToGeneratedCode = valOrDefault( @@ -679,7 +685,7 @@ class Template(Servlet): cacheHash = None cacheItem = None - if source or isinstance(file, (str, unicode)): + if source or isinstance(file, basestring): compilerSettingsHash = None if compilerSettings: compilerSettingsHash = hashDict(compilerSettings) @@ -690,7 +696,9 @@ class Template(Servlet): fileHash = None if file: - fileHash = str(hash(file))+str(os.path.getmtime(file)) + fileHash = str(hash(file)) + if globals()['__checkFileMtime']: + fileHash += str(os.path.getmtime(file)) try: # @@TR: find some way to create a cacheHash that is consistent @@ -723,6 +731,8 @@ class Template(Servlet): baseclassName=baseclassName, mainMethodName=mainMethodName, settings=(compilerSettings or {})) + if commandlineopts: + compiler.setShBang(commandlineopts.shbang) compiler.compile() generatedModuleCode = compiler.getModuleCode() @@ -744,12 +754,8 @@ class Template(Servlet): __file__ = os.path.join(cacheDirForModuleFiles, __file__) # @@TR: might want to assert that it doesn't already exist - try: - open(__file__, 'w').write(generatedModuleCode) - # @@TR: should probably restrict the perms, etc. - except OSError: - # @@ TR: should this optionally raise? - traceback.print_exc(file=sys.stderr) + open(__file__, 'w').write(generatedModuleCode) + # @@TR: should probably restrict the perms, etc. mod = new.module(str(uniqueModuleName)) if moduleGlobals: @@ -771,7 +777,6 @@ class Template(Servlet): parseError = genParserErrorFromPythonException( source, file, generatedModuleCode, exception=e) except: - traceback.print_exc() updateLinecache(__file__, generatedModuleCode) e.generatedModuleCode = generatedModuleCode raise e @@ -979,10 +984,12 @@ class Template(Servlet): mainMethNameAttr = '_mainCheetahMethod_for_'+concreteTemplateClass.__name__ mainMethName = getattr(concreteTemplateClass,mainMethNameAttr, None) if mainMethName: - def __str__(self): return getattr(self, mainMethName)() + def __str__(self): + return getattr(self, mainMethName)() elif (hasattr(concreteTemplateClass, 'respond') and concreteTemplateClass.respond!=Servlet.respond): - def __str__(self): return self.respond() + def __str__(self): + return self.respond() else: def __str__(self): if hasattr(self, mainMethNameAttr): @@ -1413,10 +1420,9 @@ class Template(Servlet): self._CHEETAH__searchList.append(self) else: # create our own searchList - self._CHEETAH__searchList = [self._CHEETAH__globalSetVars] + self._CHEETAH__searchList = [self._CHEETAH__globalSetVars, self] if searchList is not None: self._CHEETAH__searchList.extend(list(searchList)) - self._CHEETAH__searchList.append( self ) self._CHEETAH__cheetahIncludes = {} self._CHEETAH__cacheRegions = {} self._CHEETAH__indenter = Indenter() diff --git a/src/Templates/.cvsignore b/src/Templates/.cvsignore deleted file mode 100644 index 38792fd..0000000 --- a/src/Templates/.cvsignore +++ /dev/null @@ -1,3 +0,0 @@ -.cvsignore -*.pyc -*.py_bak diff --git a/src/Tests/.cvsignore b/src/Tests/.cvsignore deleted file mode 100644 index d5bd613..0000000 --- a/src/Tests/.cvsignore +++ /dev/null @@ -1,2 +0,0 @@ -.cvsignore -*.pyc diff --git a/src/Tests/CheetahWrapper.py b/src/Tests/CheetahWrapper.py index 0abce36..7607918 100644 --- a/src/Tests/CheetahWrapper.py +++ b/src/Tests/CheetahWrapper.py @@ -29,8 +29,8 @@ import commands, os, shutil, sys, tempfile import unittest_local_copy as unittest import re # Used by listTests. +from optparse import OptionParser from Cheetah.CheetahWrapper import CheetahWrapper # Used by NoBackup. -from Cheetah.Utils.optik import OptionParser # Used by main. ################################################## ## CONSTANTS & GLOBALS ## diff --git a/src/Tests/FileRefresh.py b/src/Tests/FileRefresh.py deleted file mode 100644 index 4beb3e7..0000000 --- a/src/Tests/FileRefresh.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python -# $Id: FileRefresh.py,v 1.6 2002/10/01 17:52:03 tavis_rudd Exp $ -"""Tests to make sure that the file-update-monitoring code is working properly - -THIS TEST MODULE IS JUST A SHELL AT THE MOMENT. Feel like filling it in?? - -Meta-Data -================================================================================ -Author: Tavis Rudd <tavis@damnsimple.com>, -Version: $Revision: 1.6 $ -Start Date: 2001/10/01 -Last Revision Date: $Date: 2002/10/01 17:52:03 $ -""" -__author__ = "Tavis Rudd <tavis@damnsimple.com>" -__revision__ = "$Revision: 1.6 $"[11:-2] - - -################################################## -## DEPENDENCIES ## - -import sys -import types -import os -import os.path - - -import unittest_local_copy as unittest -from Cheetah.Template import Template - -################################################## -## CONSTANTS & GLOBALS ## - -try: - True,False -except NameError: - True, False = (1==1),(1==0) - -################################################## -## TEST DATA FOR USE IN THE TEMPLATES ## - -################################################## -## TEST BASE CLASSES - -class TemplateTest(unittest.TestCase): - pass - -################################################## -## TEST CASE CLASSES - - -################################################## -## if run from the command line ## - -if __name__ == '__main__': - unittest.main() diff --git a/src/Tests/Filters.py b/src/Tests/Filters.py new file mode 100644 index 0000000..04afdd5 --- /dev/null +++ b/src/Tests/Filters.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +import sys + +import Cheetah.Template +import Cheetah.Filters + +import unittest_local_copy as unittest + +class BasicMarkdownFilterTest(unittest.TestCase): + ''' + Test that our markdown filter works + ''' + def test_BasicHeader(self): + template = ''' +#from Cheetah.Filters import Markdown +#transform Markdown +$foo + +Header +====== + ''' + expected = '''<p>bar</p> +<h1>Header</h1>''' + try: + template = Cheetah.Template.Template(template, searchList=[{'foo' : 'bar'}]) + template = str(template) + assert template == expected + except Exception, ex: + if ex.__class__.__name__ == 'MarkdownException' and sys.version_info[0] == 2 and sys.version_info[1] < 5: + print '>>> NOTE: Support for the Markdown filter will be broken for you. Markdown says: %s' % ex + return + raise + + +class BasicCodeHighlighterFilterTest(unittest.TestCase): + ''' + Test that our code highlighter filter works + ''' + def test_Python(self): + template = ''' +#from Cheetah.Filters import CodeHighlighter +#transform CodeHighlighter + +def foo(self): + return '$foo' + ''' + template = Cheetah.Template.Template(template, searchList=[{'foo' : 'bar'}]) + template = str(template) + assert template, (template, 'We should have some content here...') + + def test_Html(self): + template = ''' +#from Cheetah.Filters import CodeHighlighter +#transform CodeHighlighter + +<html><head></head><body>$foo</body></html> + ''' + template = Cheetah.Template.Template(template, searchList=[{'foo' : 'bar'}]) + template = str(template) + assert template, (template, 'We should have some content here...') + + +if __name__ == '__main__': + unittest.main() diff --git a/src/Tests/Regressions.py b/src/Tests/Regressions.py index 70af7f9..80c1336 100644 --- a/src/Tests/Regressions.py +++ b/src/Tests/Regressions.py @@ -2,8 +2,9 @@ import Cheetah.NameMapper import Cheetah.Template +import pdb -import unittest +import unittest_local_copy as unittest # This is just stupid class GetAttrException(Exception): pass @@ -36,19 +37,19 @@ class GetAttrTest(unittest.TestCase): template = Cheetah.Template.Template.compile(template, compilerSettings={}, keepRefToGeneratedCode=True) template = template(searchList=[{'obj' : CustomGetAttrClass()}]) - assert template, 'We should have a vallid template object by now' + assert template, 'We should have a valid template object by now' self.failUnlessRaises(GetAttrException, template.raiseme) -class InlineFromImportTest(unittest.TestCase): - ''' - Verify that a bug introduced in v2.1.0 where an inline: - #from module import class - would result in the following code being generated: - improt class - ''' - def runTest(self): +class InlineImportTest(unittest.TestCase): + def test_FromFooImportThing(self): + ''' + Verify that a bug introduced in v2.1.0 where an inline: + #from module import class + would result in the following code being generated: + import class + ''' template = ''' #def myfunction() #if True @@ -58,7 +59,7 @@ class InlineFromImportTest(unittest.TestCase): #end if #end def ''' - template = Cheetah.Template.Template.compile(template, compilerSettings={}, keepRefToGeneratedCode=True) + template = Cheetah.Template.Template.compile(template, compilerSettings={'useLegacyImportMode' : False}, keepRefToGeneratedCode=True) template = template(searchList=[{}]) assert template, 'We should have a valid template object by now' @@ -66,6 +67,49 @@ class InlineFromImportTest(unittest.TestCase): rc = template.myfunction() assert rc == 17, (template, 'Didn\'t get a proper return value') + def test_ImportFailModule(self): + template = ''' + #try + #import invalidmodule + #except + #set invalidmodule = dict(FOO='BAR!') + #end try + + $invalidmodule.FOO + ''' + template = Cheetah.Template.Template.compile(template, compilerSettings={'useLegacyImportMode' : False}, keepRefToGeneratedCode=True) + template = template(searchList=[{}]) + + assert template, 'We should have a valid template object by now' + assert str(template), 'We weren\'t able to properly generate the result from the template' + + def test_ProperImportOfBadModule(self): + template = ''' + #from invalid import fail + + This should totally $fail + ''' + self.failUnlessRaises(ImportError, Cheetah.Template.Template.compile, template, compilerSettings={'useLegacyImportMode' : False}, keepRefToGeneratedCode=True) + + def test_AutoImporting(self): + template = ''' + #extends FakeyTemplate + + Boo! + ''' + self.failUnlessRaises(ImportError, Cheetah.Template.Template.compile, template) + + def test_StuffBeforeImport_Legacy(self): + template = ''' +### +### I like comments before import +### +#extends Foo +Bar +''' + self.failUnlessRaises(ImportError, Cheetah.Template.Template.compile, template, compilerSettings={'useLegacyImportMode' : True}, keepRefToGeneratedCode=True) + + if __name__ == '__main__': unittest.main() diff --git a/src/Tests/Template.py b/src/Tests/Template.py index c8b6ee8..bca1b30 100644 --- a/src/Tests/Template.py +++ b/src/Tests/Template.py @@ -319,7 +319,7 @@ class TryExceptImportTest(TemplateTest): #end def ''' # This should raise an IndentationError (if the bug exists) - klass = Template.compile(source=source) + klass = Template.compile(source=source, compilerSettings={'useLegacyImportMode' : False}) t = klass(namespaces={'foo' : 1234}) diff --git a/src/Tests/Test.py b/src/Tests/Test.py index 9e46a5d..e168fc9 100755 --- a/src/Tests/Test.py +++ b/src/Tests/Test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# $Id: Test.py,v 1.44 2006/01/15 20:45:10 tavis_rudd Exp $ -"""Core module of Cheetah's Unit-testing framework +''' +Core module of Cheetah's Unit-testing framework TODO ================================================================================ @@ -8,63 +8,40 @@ TODO # negative test cases for expected exceptions # black-box vs clear-box testing # do some tests that run the Template for long enough to check that the refresh code works - -Meta-Data -================================================================================ -Author: Tavis Rudd <tavis@damnsimple.com>, -License: This software is released for unlimited distribution under the - terms of the MIT license. See the LICENSE file. -Version: $Revision: 1.44 $ -Start Date: 2001/03/30 -Last Revision Date: $Date: 2006/01/15 20:45:10 $ -""" -__author__ = "Tavis Rudd <tavis@damnsimple.com>" -__revision__ = "$Revision: 1.44 $"[11:-2] - - -################################################## -## DEPENDENCIES ## +''' import sys import unittest_local_copy as unittest -################################################## -## CONSTANTS & GLOBALS -try: - True, False -except NameError: - True, False = (1==1),(1==0) - -################################################## -## TESTS import SyntaxAndOutput import NameMapper import Template -import FileRefresh import CheetahWrapper +import Regressions +import Unicode -SyntaxSuite = unittest.findTestCases(SyntaxAndOutput) -NameMapperSuite = unittest.findTestCases(NameMapper) -TemplateSuite = unittest.findTestCases(Template) -FileRefreshSuite = unittest.findTestCases(FileRefresh) -if not sys.platform.startswith('java'): - CheetahWrapperSuite = unittest.findTestCases(CheetahWrapper) - -from SyntaxAndOutput import * -from NameMapper import * -from Template import * -from FileRefresh import * +suites = [ + unittest.findTestCases(SyntaxAndOutput), + unittest.findTestCases(NameMapper), + unittest.findTestCases(Template), + unittest.findTestCases(Regressions), + unittest.findTestCases(Unicode), +] if not sys.platform.startswith('java'): - from CheetahWrapper import * - -################################################## -## if run from the command line + suites.append(unittest.findTestCases(CheetahWrapper)) if __name__ == '__main__': - unittest.main() + runner = unittest.TextTestRunner() + if 'xml' in sys.argv: + import xmlrunner + runner = xmlrunner.XMLTestRunner(filename='Cheetah-Tests.xml') + + results = runner.run(unittest.TestSuite(suites)) + + diff --git a/src/Tests/Unicode.py b/src/Tests/Unicode.py new file mode 100644 index 0000000..8b50980 --- /dev/null +++ b/src/Tests/Unicode.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +# -*- encoding: utf8 -*- + +from Cheetah.Template import Template +import traceback +import unittest_local_copy as unittest # This is stupid + +class JPQ_UTF8_Test1(unittest.TestCase): + def runTest(self): + t = Template.compile(source="""Main file with |$v| + + $other""") + + otherT = Template.compile(source="Other template with |$v|") + other = otherT() + t.other = other + + t.v = u'Unicode String' + t.other.v = u'Unicode String' + + assert t() + +class JPQ_UTF8_Test2(unittest.TestCase): + def runTest(self): + t = Template.compile(source="""Main file with |$v| + + $other""") + + otherT = Template.compile(source="Other template with |$v|") + other = otherT() + t.other = other + + t.v = u'Unicode String with eacute é' + t.other.v = u'Unicode String' + + assert unicode(t()) + assert t().__str__() + + +class JPQ_UTF8_Test3(unittest.TestCase): + def runTest(self): + t = Template.compile(source="""Main file with |$v| + + $other""") + + otherT = Template.compile(source="Other template with |$v|") + other = otherT() + t.other = other + + t.v = u'Unicode String with eacute é' + t.other.v = u'Unicode String and an eacute é' + + assert unicode(t()) + +class JPQ_UTF8_Test4(unittest.TestCase): + def runTest(self): + t = Template.compile(source="""Main file with |$v| and eacute in the template é""") + + t.v = 'Unicode String' + + assert t() + +class JPQ_UTF8_Test5(unittest.TestCase): + def runTest(self): + t = Template.compile(source="""#encoding utf-8 + Main file with |$v| and eacute in the template é""") + + t.v = u'Unicode String' + rc = t().__str__() + assert rc + +if __name__ == '__main__': + unittest.main() diff --git a/src/Tests/unittest_local_copy.py b/src/Tests/unittest_local_copy.py index 54061ae..a5f5499 100755 --- a/src/Tests/unittest_local_copy.py +++ b/src/Tests/unittest_local_copy.py @@ -834,8 +834,9 @@ class TestLoader: def getTestCaseNames(self, testCaseClass): """Return a sorted sequence of method names found within testCaseClass. """ - testFnNames = filter(lambda n,p=self.testMethodPrefix: n[:len(p)] == p, - dir(testCaseClass)) + testFnNames = [fn for fn in dir(testCaseClass) if fn.startswith(self.testMethodPrefix)] + if hasattr(testCaseClass, 'runTest'): + testFnNames.append('runTest') for baseclass in testCaseClass.__bases__: for testFnName in self.getTestCaseNames(baseclass): if testFnName not in testFnNames: # handle overridden methods diff --git a/src/Tests/xmlrunner.py b/src/Tests/xmlrunner.py index a4fed9f..dc49c56 100644 --- a/src/Tests/xmlrunner.py +++ b/src/Tests/xmlrunner.py @@ -19,6 +19,7 @@ from xml.sax.saxutils import escape from StringIO import StringIO + class _TestInfo(object): """Information about a particular test. @@ -28,29 +29,12 @@ class _TestInfo(object): """ def __init__(self, test, time): - (self._class, self._method) = test.id().rsplit(".", 1) + _pieces = test.id().split('.') + (self._class, self._method) = ('.'.join(_pieces[:-1]), _pieces[-1]) self._time = time self._error = None self._failure = None - @staticmethod - def create_success(test, time): - """Create a _TestInfo instance for a successful test.""" - return _TestInfo(test, time) - - @staticmethod - def create_failure(test, time, failure): - """Create a _TestInfo instance for a failed test.""" - info = _TestInfo(test, time) - info._failure = failure - return info - - @staticmethod - def create_error(test, time, error): - """Create a _TestInfo instance for an erroneous test.""" - info = _TestInfo(test, time) - info._error = error - return info def print_report(self, stream): """Print information about this test case in XML format to the @@ -74,13 +58,29 @@ class _TestInfo(object): text = escape(str(error[1])) stream.write('\n') stream.write(' <%s type="%s">%s\n' \ - % (tagname, str(error[0]), text)) + % (tagname, issubclass(error[0], Exception) and error[0].__name__ or str(error[0]), text)) tb_stream = StringIO() traceback.print_tb(error[2], None, tb_stream) stream.write(escape(tb_stream.getvalue())) stream.write(' </%s>\n' % tagname) stream.write(' ') +# Module level functions since Python 2.3 doesn't grok decorators +def create_success(test, time): + """Create a _TestInfo instance for a successful test.""" + return _TestInfo(test, time) + +def create_failure(test, time, failure): + """Create a _TestInfo instance for a failed test.""" + info = _TestInfo(test, time) + info._failure = failure + return info + +def create_error(test, time, error): + """Create a _TestInfo instance for an erroneous test.""" + info = _TestInfo(test, time) + info._error = error + return info class _XMLTestResult(unittest.TestResult): @@ -108,11 +108,11 @@ class _XMLTestResult(unittest.TestResult): time_taken = time.time() - self._start_time unittest.TestResult.stopTest(self, test) if self._error: - info = _TestInfo.create_error(test, time_taken, self._error) + info = create_error(test, time_taken, self._error) elif self._failure: - info = _TestInfo.create_failure(test, time_taken, self._failure) + info = create_failure(test, time_taken, self._failure) else: - info = _TestInfo.create_success(test, time_taken) + info = create_success(test, time_taken) self._tests.append(info) def addError(self, test, err): @@ -158,8 +158,9 @@ class XMLTestRunner(object): """ - def __init__(self, stream=None): - self._stream = stream + def __init__(self, *args, **kwargs): + self._stream = kwargs.get('stream') + self._filename = kwargs.get('filename') self._path = "." def run(self, test): @@ -168,6 +169,8 @@ class XMLTestRunner(object): classname = class_.__module__ + "." + class_.__name__ if self._stream == None: filename = "TEST-%s.xml" % classname + if self._filename: + filename = self._filename stream = file(os.path.join(self._path, filename), "w") stream.write('<?xml version="1.0" encoding="utf-8"?>\n') else: diff --git a/src/Tools/.cvsignore b/src/Tools/.cvsignore deleted file mode 100644 index d5bd613..0000000 --- a/src/Tools/.cvsignore +++ /dev/null @@ -1,2 +0,0 @@ -.cvsignore -*.pyc diff --git a/src/Tools/RecursiveNull.py b/src/Tools/RecursiveNull.py index 4897d80..d4a0558 100644 --- a/src/Tools/RecursiveNull.py +++ b/src/Tools/RecursiveNull.py @@ -1,23 +1,29 @@ #!/usr/bin/env python -"""Nothing, but in a friendly way. Good for filling in for objects you want to +""" +Nothing, but in a friendly way. Good for filling in for objects you want to hide. If $form.f1 is a RecursiveNull object, then $form.f1.anything["you"].might("use") will resolve to the empty string. This module was contributed by Ian Bicking. """ -class RecursiveNull: - __doc__ = __doc__ # Use the module's docstring for the class's docstring. - def __getattr__(self, attr): - return self - def __getitem__(self, item): - return self - def __call__(self, *vars, **kw): - return self - def __str__(self): - return '' - def __repr__(self): - return '' - def __nonzero__(self): - return 0 +class RecursiveNull(object): + def __getattr__(self, attr): + return self + def __getitem__(self, item): + return self + def __call__(self, *args, **kwargs): + return self + def __str__(self): + return '' + def __repr__(self): + return '' + def __nonzero__(self): + return 0 + def __eq__(self, x): + if x: + return False + return True + def __ne__(self, x): + return x and True or False diff --git a/src/Utils/.cvsignore b/src/Utils/.cvsignore deleted file mode 100644 index d5bd613..0000000 --- a/src/Utils/.cvsignore +++ /dev/null @@ -1,2 +0,0 @@ -.cvsignore -*.pyc diff --git a/src/Utils/optik/.cvsignore b/src/Utils/optik/.cvsignore deleted file mode 100644 index d5bd613..0000000 --- a/src/Utils/optik/.cvsignore +++ /dev/null @@ -1,2 +0,0 @@ -.cvsignore -*.pyc diff --git a/src/Utils/optik/__init__.py b/src/Utils/optik/__init__.py deleted file mode 100644 index 75a30ba..0000000 --- a/src/Utils/optik/__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -"""optik - -A powerful, extensible, and easy-to-use command-line parser for Python. - -By Greg Ward <gward@python.net> - -See http://optik.sourceforge.net/ - -Cheetah modifications: added "Cheetah.Utils.optik." prefix to - all intra-Optik imports. -""" - -# Copyright (c) 2001 Gregory P. Ward. All rights reserved. -# See the README.txt distributed with Optik for licensing terms. - -__revision__ = "$Id: __init__.py,v 1.2 2002/09/12 06:56:51 hierro Exp $" - -__version__ = "1.3" - - -# Re-import these for convenience -from Cheetah.Utils.optik.option import Option -from Cheetah.Utils.optik.option_parser import \ - OptionParser, SUPPRESS_HELP, SUPPRESS_USAGE, STD_HELP_OPTION -from Cheetah.Utils.optik.errors import OptionValueError - - -# Some day, there might be many Option classes. As of Optik 1.3, the -# preferred way to instantiate Options is indirectly, via make_option(), -# which will become a factory function when there are many Option -# classes. -make_option = Option diff --git a/src/Utils/optik/errors.py b/src/Utils/optik/errors.py deleted file mode 100644 index 2ed75e6..0000000 --- a/src/Utils/optik/errors.py +++ /dev/null @@ -1,52 +0,0 @@ -"""optik.errors - -Exception classes used by Optik. -""" - -__revision__ = "$Id: errors.py,v 1.1 2002/08/24 17:10:06 hierro Exp $" - -# Copyright (c) 2001 Gregory P. Ward. All rights reserved. -# See the README.txt distributed with Optik for licensing terms. - -# created 2001/10/17 GPW (from optik.py) - - -class OptikError (Exception): - def __init__ (self, msg): - self.msg = msg - - def __str__ (self): - return self.msg - - -class OptionError (OptikError): - """ - Raised if an Option instance is created with invalid or - inconsistent arguments. - """ - - def __init__ (self, msg, option): - self.msg = msg - self.option_id = str(option) - - def __str__ (self): - if self.option_id: - return "option %s: %s" % (self.option_id, self.msg) - else: - return self.msg - -class OptionConflictError (OptionError): - """ - Raised if conflicting options are added to an OptionParser. - """ - -class OptionValueError (OptikError): - """ - Raised if an invalid option value is encountered on the command - line. - """ - -class BadOptionError (OptikError): - """ - Raised if an invalid or ambiguous option is seen on the command-line. - """ diff --git a/src/Utils/optik/option.py b/src/Utils/optik/option.py deleted file mode 100644 index ac85c3d..0000000 --- a/src/Utils/optik/option.py +++ /dev/null @@ -1,354 +0,0 @@ -"""optik.option - -Defines the Option class and some standard value-checking functions. - -Cheetah modifications: added "Cheetah.Utils.optik." prefix to - all intra-Optik imports. -""" - -__revision__ = "$Id: option.py,v 1.2 2002/09/12 06:56:51 hierro Exp $" - -# Copyright (c) 2001 Gregory P. Ward. All rights reserved. -# See the README.txt distributed with Optik for licensing terms. - -# created 2001/10/17, GPW (from optik.py) - -import sys -from types import TupleType, DictType -from Cheetah.Utils.optik.errors import OptionError, OptionValueError - -_builtin_cvt = { "int" : (int, "integer"), - "long" : (long, "long integer"), - "float" : (float, "floating-point"), - "complex" : (complex, "complex") } - -def check_builtin (option, opt, value): - (cvt, what) = _builtin_cvt[option.type] - try: - return cvt(value) - except ValueError: - raise OptionValueError( - #"%s: invalid %s argument %r" % (opt, what, value)) - "option %s: invalid %s value: %r" % (opt, what, value)) - -# Not supplying a default is different from a default of None, -# so we need an explicit "not supplied" value. -NO_DEFAULT = "NO"+"DEFAULT" - - -class Option: - """ - Instance attributes: - _short_opts : [string] - _long_opts : [string] - - action : string - type : string - dest : string - default : any - nargs : int - const : any - callback : function - callback_args : (any*) - callback_kwargs : { string : any } - help : string - metavar : string - """ - - # The list of instance attributes that may be set through - # keyword args to the constructor. - ATTRS = ['action', - 'type', - 'dest', - 'default', - 'nargs', - 'const', - 'callback', - 'callback_args', - 'callback_kwargs', - 'help', - 'metavar'] - - # The set of actions allowed by option parsers. Explicitly listed - # here so the constructor can validate its arguments. - ACTIONS = ("store", - "store_const", - "store_true", - "store_false", - "append", - "count", - "callback", - "help", - "version") - - # The set of actions that involve storing a value somewhere; - # also listed just for constructor argument validation. (If - # the action is one of these, there must be a destination.) - STORE_ACTIONS = ("store", - "store_const", - "store_true", - "store_false", - "append", - "count") - - # The set of actions for which it makes sense to supply a value - # type, ie. where we expect an argument to this option. - TYPED_ACTIONS = ("store", - "append", - "callback") - - # The set of known types for option parsers. Again, listed here for - # constructor argument validation. - TYPES = ("string", "int", "long", "float", "complex") - - # Dictionary of argument checking functions, which convert and - # validate option arguments according to the option type. - # - # Signature of checking functions is: - # check(option : Option, opt : string, value : string) -> any - # where - # option is the Option instance calling the checker - # opt is the actual option seen on the command-line - # (eg. "-a", "--file") - # value is the option argument seen on the command-line - # - # The return value should be in the appropriate Python type - # for option.type -- eg. an integer if option.type == "int". - # - # If no checker is defined for a type, arguments will be - # unchecked and remain strings. - TYPE_CHECKER = { "int" : check_builtin, - "long" : check_builtin, - "float" : check_builtin, - "complex" : check_builtin, - } - - - # CHECK_METHODS is a list of unbound method objects; they are called - # by the constructor, in order, after all attributes are - # initialized. The list is created and filled in later, after all - # the methods are actually defined. (I just put it here because I - # like to define and document all class attributes in the same - # place.) Subclasses that add another _check_*() method should - # define their own CHECK_METHODS list that adds their check method - # to those from this class. - CHECK_METHODS = None - - - # -- Constructor/initialization methods ---------------------------- - - def __init__ (self, *opts, **attrs): - # Set _short_opts, _long_opts attrs from 'opts' tuple - opts = self._check_opt_strings(opts) - self._set_opt_strings(opts) - - # Set all other attrs (action, type, etc.) from 'attrs' dict - self._set_attrs(attrs) - - # Check all the attributes we just set. There are lots of - # complicated interdependencies, but luckily they can be farmed - # out to the _check_*() methods listed in CHECK_METHODS -- which - # could be handy for subclasses! The one thing these all share - # is that they raise OptionError if they discover a problem. - for checker in self.CHECK_METHODS: - checker(self) - - def _check_opt_strings (self, opts): - # Filter out None because early versions of Optik had exactly - # one short option and one long option, either of which - # could be None. - opts = filter(None, opts) - if not opts: - raise OptionError("at least one option string must be supplied", - self) - return opts - - def _set_opt_strings (self, opts): - self._short_opts = [] - self._long_opts = [] - for opt in opts: - if len(opt) < 2: - raise OptionError( - "invalid option string %r: " - "must be at least two characters long" % opt, self) - elif len(opt) == 2: - if not (opt[0] == "-" and opt[1] != "-"): - raise OptionError( - "invalid short option string %r: " - "must be of the form -x, (x any non-dash char)" % opt, - self) - self._short_opts.append(opt) - else: - if not (opt[0:2] == "--" and opt[2] != "-"): - raise OptionError( - "invalid long option string %r: " - "must start with --, followed by non-dash" % opt, - self) - self._long_opts.append(opt) - - def _set_attrs (self, attrs): - for attr in self.ATTRS: - if attrs.has_key(attr): - setattr(self, attr, attrs[attr]) - del attrs[attr] - else: - if attr == 'default': - setattr(self, attr, NO_DEFAULT) - else: - setattr(self, attr, None) - if attrs: - raise OptionError( - "invalid keyword arguments: %s" % ", ".join(attrs.keys()), - self) - - - # -- Constructor validation methods -------------------------------- - - def _check_action (self): - if self.action is None: - self.action = "store" - elif self.action not in self.ACTIONS: - raise OptionError("invalid action: %r" % self.action, self) - - def _check_type (self): - if self.type is None: - # XXX should factor out another class attr here: list of - # actions that *require* a type - if self.action in ("store", "append"): - # No type given? "string" is the most sensible default. - self.type = "string" - else: - if self.type not in self.TYPES: - raise OptionError("invalid option type: %r" % self.type, self) - if self.action not in self.TYPED_ACTIONS: - raise OptionError( - "must not supply a type for action %r" % self.action, self) - - def _check_dest (self): - if self.action in self.STORE_ACTIONS and self.dest is None: - # No destination given, and we need one for this action. - # Glean a destination from the first long option string, - # or from the first short option string if no long options. - if self._long_opts: - # eg. "--foo-bar" -> "foo_bar" - self.dest = self._long_opts[0][2:].replace('-', '_') - else: - self.dest = self._short_opts[0][1] - - def _check_const (self): - if self.action != "store_const" and self.const is not None: - raise OptionError( - "'const' must not be supplied for action %r" % self.action, - self) - - def _check_nargs (self): - if self.action in self.TYPED_ACTIONS: - if self.nargs is None: - self.nargs = 1 - elif self.nargs is not None: - raise OptionError( - "'nargs' must not be supplied for action %r" % self.action, - self) - - def _check_callback (self): - if self.action == "callback": - if not callable(self.callback): - raise OptionError( - "callback not callable: %r" % self.callback, self) - if (self.callback_args is not None and - type(self.callback_args) is not TupleType): - raise OptionError( - "callback_args, if supplied, must be a tuple: not %r" - % self.callback_args, self) - if (self.callback_kwargs is not None and - type(self.callback_kwargs) is not DictType): - raise OptionError( - "callback_kwargs, if supplied, must be a dict: not %r" - % self.callback_kwargs, self) - else: - if self.callback is not None: - raise OptionError( - "callback supplied (%r) for non-callback option" - % self.callback, self) - if self.callback_args is not None: - raise OptionError( - "callback_args supplied for non-callback option", self) - if self.callback_kwargs is not None: - raise OptionError( - "callback_kwargs supplied for non-callback option", self) - - - CHECK_METHODS = [_check_action, - _check_type, - _check_dest, - _check_const, - _check_nargs, - _check_callback] - - - # -- Miscellaneous methods ----------------------------------------- - - def __str__ (self): - if self._short_opts or self._long_opts: - return "/".join(self._short_opts + self._long_opts) - else: - raise RuntimeError, "short_opts and long_opts both empty!" - - def takes_value (self): - return self.type is not None - - - # -- Processing methods -------------------------------------------- - - def check_value (self, opt, value): - checker = self.TYPE_CHECKER.get(self.type) - if checker is None: - return value - else: - return checker(self, opt, value) - - def process (self, opt, value, values, parser): - - # First, convert the value(s) to the right type. Howl if any - # value(s) are bogus. - if value is not None: - if self.nargs == 1: - value = self.check_value(opt, value) - else: - value = tuple([self.check_value(opt, v) for v in value]) - - # And then take whatever action is expected of us. - # This is a separate method to make life easier for - # subclasses to add new actions. - return self.take_action( - self.action, self.dest, opt, value, values, parser) - - def take_action (self, action, dest, opt, value, values, parser): - if action == "store": - setattr(values, dest, value) - elif action == "store_const": - setattr(values, dest, self.const) - elif action == "store_true": - setattr(values, dest, 1) - elif action == "store_false": - setattr(values, dest, 0) - elif action == "append": - values.ensure_value(dest, []).append(value) - elif action == "count": - setattr(values, dest, values.ensure_value(dest, 0) + 1) - elif action == "callback": - args = self.callback_args or () - kwargs = self.callback_kwargs or {} - self.callback(self, opt, value, parser, *args, **kwargs) - elif action == "help": - parser.print_help() - sys.exit(0) - elif action == "version": - parser.print_version() - sys.exit(0) - else: - raise RuntimeError, "unknown action %r" % self.action - - return 1 - -# class Option diff --git a/src/Utils/optik/option_parser.py b/src/Utils/optik/option_parser.py deleted file mode 100644 index 1b4e632..0000000 --- a/src/Utils/optik/option_parser.py +++ /dev/null @@ -1,667 +0,0 @@ -"""optik.option_parser - -Provides the OptionParser and Values classes. - -Cheetah modifications: added "Cheetah.Utils.optik." prefix to - all intra-Optik imports. -""" - -__revision__ = "$Id: option_parser.py,v 1.2 2002/09/12 06:56:51 hierro Exp $" - -# Copyright (c) 2001 Gregory P. Ward. All rights reserved. -# See the README.txt distributed with Optik for licensing terms. - -# created 2001/10/17, GPW (from optik.py) - -import sys, os -import types -from Cheetah.Utils.optik.option import Option, NO_DEFAULT -from Cheetah.Utils.optik.errors import OptionConflictError, OptionValueError, BadOptionError - -def get_prog_name (): - return os.path.basename(sys.argv[0]) - - -SUPPRESS_HELP = "SUPPRESS"+"HELP" -SUPPRESS_USAGE = "SUPPRESS"+"USAGE" - -STD_HELP_OPTION = Option("-h", "--help", - action="help", - help="show this help message and exit") -STD_VERSION_OPTION = Option("--version", - action="version", - help="show program's version number and exit") - - -class Values: - - def __init__ (self, defaults=None): - if defaults: - for (attr, val) in defaults.items(): - setattr(self, attr, val) - - - def _update_careful (self, dict): - """ - Update the option values from an arbitrary dictionary, but only - use keys from dict that already have a corresponding attribute - in self. Any keys in dict without a corresponding attribute - are silently ignored. - """ - for attr in dir(self): - if dict.has_key(attr): - dval = dict[attr] - if dval is not None: - setattr(self, attr, dval) - - def _update_loose (self, dict): - """ - Update the option values from an arbitrary dictionary, - using all keys from the dictionary regardless of whether - they have a corresponding attribute in self or not. - """ - self.__dict__.update(dict) - - def _update (self, dict, mode): - if mode == "careful": - self._update_careful(dict) - elif mode == "loose": - self._update_loose(dict) - else: - raise ValueError, "invalid update mode: %r" % mode - - def read_module (self, modname, mode="careful"): - __import__(modname) - mod = sys.modules[modname] - self._update(vars(mod), mode) - - def read_file (self, filename, mode="careful"): - vars = {} - execfile(filename, vars) - self._update(vars, mode) - - def ensure_value (self, attr, value): - if not hasattr(self, attr) or getattr(self, attr) is None: - setattr(self, attr, value) - return getattr(self, attr) - - -class OptionParser: - """ - Class attributes: - standard_option_list : [Option] - list of standard options that will be accepted by all instances - of this parser class (intended to be overridden by subclasses). - - Instance attributes: - usage : string - a usage string for your program. Before it is displayed - to the user, "%prog" will be expanded to the name of - your program (os.path.basename(sys.argv[0])). - option_list : [Option] - the list of all options accepted on the command-line of - this program - _short_opt : { string : Option } - dictionary mapping short option strings, eg. "-f" or "-X", - to the Option instances that implement them. If an Option - has multiple short option strings, it will appears in this - dictionary multiple times. - _long_opt : { string : Option } - dictionary mapping long option strings, eg. "--file" or - "--exclude", to the Option instances that implement them. - Again, a given Option can occur multiple times in this - dictionary. - _long_opts : [string] - list of long option strings recognized by this option - parser. Should be equal to _long_opt.values(). - defaults : { string : any } - dictionary mapping option destination names to default - values for each destination. - - allow_interspersed_args : boolean = true - if true, positional arguments may be interspersed with options. - Assuming -a and -b each take a single argument, the command-line - -ablah foo bar -bboo baz - will be interpreted the same as - -ablah -bboo -- foo bar baz - If this flag were false, that command line would be interpreted as - -ablah -- foo bar -bboo baz - -- ie. we stop processing options as soon as we see the first - non-option argument. (This is the tradition followed by - Python's getopt module, Perl's Getopt::Std, and other argument- - parsing libraries, but it is generally annoying to users.) - - rargs : [string] - the argument list currently being parsed. Only set when - parse_args() is active, and continually trimmed down as - we consume arguments. Mainly there for the benefit of - callback options. - largs : [string] - the list of leftover arguments that we have skipped while - parsing options. If allow_interspersed_args is false, this - list is always empty. - values : Values - the set of option values currently being accumulated. Only - set when parse_args() is active. Also mainly for callbacks. - - Because of the 'rargs', 'largs', and 'values' attributes, - OptionParser is not thread-safe. If, for some perverse reason, you - need to parse command-line arguments simultaneously in different - threads, use different OptionParser instances. - - """ - - standard_option_list = [STD_HELP_OPTION] - - - def __init__ (self, - usage=None, - option_list=None, - option_class=Option, - version=None, - conflict_handler="error"): - self.set_usage(usage) - self.option_class = option_class - self.version = version - self.set_conflict_handler(conflict_handler) - self.allow_interspersed_args = 1 - - # Create the various lists and dicts that constitute the - # "option list". See class docstring for details about - # each attribute. - self._create_option_list() - - # Populate the option list; initial sources are the - # standard_option_list class attribute, the 'option_list' - # argument, and the STD_VERSION_OPTION global (if 'version' - # supplied). - self._populate_option_list(option_list) - - self._init_parsing_state() - - # -- Private methods ----------------------------------------------- - # (used by the constructor) - - def _create_option_list (self): - self.option_list = [] - self._short_opt = {} # single letter -> Option instance - self._long_opt = {} # long option -> Option instance - self._long_opts = [] # list of long options - self.defaults = {} # maps option dest -> default value - - def _populate_option_list (self, option_list): - if self.standard_option_list: - self.add_options(self.standard_option_list) - if self.version: - self.add_option(STD_VERSION_OPTION) - if option_list: - self.add_options(option_list) - - def _init_parsing_state (self): - # These are set in parse_args() for the convenience of callbacks. - self.rargs = None - self.largs = None - self.values = None - - - # -- Simple modifier methods --------------------------------------- - - def set_usage (self, usage): - if usage is None: - self.usage = "usage: %prog [options]" - elif usage is SUPPRESS_USAGE: - self.usage = None - else: - self.usage = usage - - def enable_interspersed_args (self): - self.allow_interspersed_args = 1 - - def disable_interspersed_args (self): - self.allow_interspersed_args = 0 - - def set_conflict_handler (self, handler): - if handler not in ("ignore", "error", "resolve"): - raise ValueError, "invalid conflict_resolution value %r" % handler - self.conflict_handler = handler - - def set_default (self, dest, value): - self.defaults[dest] = value - - def set_defaults (self, **kwargs): - self.defaults.update(kwargs) - - - # -- Option-adding methods ----------------------------------------- - - def _check_conflict (self, option): - conflict_opts = [] - for opt in option._short_opts: - if self._short_opt.has_key(opt): - conflict_opts.append((opt, self._short_opt[opt])) - for opt in option._long_opts: - if self._long_opt.has_key(opt): - conflict_opts.append((opt, self._long_opt[opt])) - - if conflict_opts: - handler = self.conflict_handler - if handler == "ignore": # behaviour for Optik 1.0, 1.1 - pass - elif handler == "error": # new in 1.2 - raise OptionConflictError( - "conflicting option string(s): %s" - % ", ".join([co[0] for co in conflict_opts]), - option) - elif handler == "resolve": # new in 1.2 - for (opt, c_option) in conflict_opts: - if opt.startswith("--"): - c_option._long_opts.remove(opt) - del self._long_opt[opt] - else: - c_option._short_opts.remove(opt) - del self._short_opt[opt] - if not (c_option._short_opts or c_option._long_opts): - self.option_list.remove(c_option) - - - def add_option (self, *args, **kwargs): - """add_option(Option) - add_option(opt_str, ..., kwarg=val, ...) - """ - if type(args[0]) is types.StringType: - option = self.option_class(*args, **kwargs) - elif len(args) == 1 and not kwargs: - option = args[0] - if not isinstance(option, Option): - raise TypeError, "not an Option instance: %r" % option - else: - raise TypeError, "invalid arguments" - - self._check_conflict(option) - - self.option_list.append(option) - for opt in option._short_opts: - self._short_opt[opt] = option - for opt in option._long_opts: - self._long_opt[opt] = option - self._long_opts.append(opt) - - if option.dest is not None: # option has a dest, we need a default - if option.default is not NO_DEFAULT: - self.defaults[option.dest] = option.default - elif not self.defaults.has_key(option.dest): - self.defaults[option.dest] = None - - def add_options (self, option_list): - for option in option_list: - self.add_option(option) - - - # -- Option query/removal methods ---------------------------------- - - def get_option (self, opt_str): - return (self._short_opt.get(opt_str) or - self._long_opt.get(opt_str)) - - def has_option (self, opt_str): - return (self._short_opt.has_key(opt_str) or - self._long_opt.has_key(opt_str)) - - - def remove_option (self, opt_str): - option = self._short_opt.get(opt_str) - if option is None: - option = self._long_opt.get(opt_str) - if option is None: - raise ValueError("no such option %r" % opt_str) - - for opt in option._short_opts: - del self._short_opt[opt] - for opt in option._long_opts: - del self._long_opt[opt] - self._long_opts.remove(opt) - self.option_list.remove(option) - - - # -- Option-parsing methods ---------------------------------------- - - def _get_args (self, args): - if args is None: - return sys.argv[1:] - else: - return args[:] # don't modify caller's list - - def parse_args (self, args=None, values=None): - """ - parse_args(args : [string] = sys.argv[1:], - values : Values = None) - -> (values : Values, args : [string]) - - Parse the command-line options found in 'args' (default: - sys.argv[1:]). Any errors result in a call to 'error()', which - by default prints the usage message to stderr and calls - sys.exit() with an error message. On success returns a pair - (values, args) where 'values' is an Values instance (with all - your option values) and 'args' is the list of arguments left - over after parsing options. - """ - rargs = self._get_args(args) - if values is None: - values = Values(self.defaults) - - # Store the halves of the argument list as attributes for the - # convenience of callbacks: - # rargs - # the rest of the command-line (the "r" stands for - # "remaining" or "right-hand") - # largs - # the leftover arguments -- ie. what's left after removing - # options and their arguments (the "l" stands for "leftover" - # or "left-hand") - - # Say this is the original argument list: - # [arg0, arg1, ..., arg(i-1), arg(i), arg(i+1), ..., arg(N-1)] - # ^ - # (we are about to process arg(i)). - # - # Then rargs is [arg(i), ..., arg(N-1)] - # and largs is a *subset* of [arg0, ..., arg(i-1)] - # (any options and their arguments will have been removed - # from largs). - # - # _process_arg() will always consume 1 or more arguments. - # If it consumes 1 (eg. arg is an option that takes no arguments), - # then after _process_arg() is done the situation is: - # largs = subset of [arg0, ..., arg(i)] - # rargs = [arg(i+1), ..., arg(N-1)] - # - # If allow_interspersed_args is false, largs will always be - # *empty* -- still a subset of [arg0, ..., arg(i-1)], but - # not a very interesting subset! - - self.rargs = rargs - self.largs = largs = [] - self.values = values - - stop = 0 - while rargs and not stop: - try: - stop = self._process_arg(largs, rargs, values) - except (BadOptionError, OptionValueError), err: - self.error(err.msg) - - args = largs + rargs - return self.check_values(values, args) - - def check_values (self, values, args): - """ - check_values(values : Values, args : [string]) - -> (values : Values, args : [string]) - - Check that the supplied option values and leftover arguments are - valid. Returns the option values and leftover arguments - (possibly adjusted, possibly completely new -- whatever you - like). Default implementation just returns the passed-in - values; subclasses may override as desired. - """ - return (values, args) - - def _process_arg (self, largs, rargs, values): - """_process_args(largs : [string], - rargs : [string], - values : Values) - -> stop : boolean - - Process a single command-line argument, consuming zero or more - arguments. The next argument to process is rargs[0], which will - almost certainly be consumed from rargs. (It might wind up in - largs, or it might affect a value in values, or -- if a callback - is involved -- almost anything might happen. It will not be - consumed if it is a non-option argument and - allow_interspersed_args is false.) More arguments from rargs - may also be consumed, depending on circumstances. - - Returns true if option processing should stop after this - argument is processed. - """ - - # We handle bare "--" explicitly, and bare "-" is handled by the - # standard arg handler since the short arg case ensures that the len - # of the opt string is greater than 1. - - arg = rargs[0] - if arg == "--": - del rargs[0] - return 1 - elif arg[0:2] == "--": - # process a single long option (possibly with value(s)) - self._process_long_opt(rargs, values) - elif arg[:1] == "-" and len(arg) > 1: - # process a cluster of short options (possibly with - # value(s) for the last one only) - self._process_short_opts(rargs, values) - else: - if self.allow_interspersed_args: - largs.append(arg) - del rargs[0] - else: - return 1 # stop now, leave this arg in rargs - - return 0 # keep processing args - - def _match_long_opt (self, opt): - """_match_long_opt(opt : string) -> string - - Determine which long option string 'opt' matches, ie. which one - it is an unambiguous abbrevation for. Raises BadOptionError if - 'opt' doesn't unambiguously match any long option string. - """ - return _match_abbrev(opt, self._long_opts) - - def _process_long_opt (self, rargs, values): - arg = rargs.pop(0) - - # Value explicitly attached to arg? Pretend it's the next - # argument. - if "=" in arg: - (opt, next_arg) = arg.split("=", 1) - rargs.insert(0, next_arg) - had_explicit_value = 1 - else: - opt = arg - had_explicit_value = 0 - - opt = self._match_long_opt(opt) - option = self._long_opt[opt] - if option.takes_value(): - nargs = option.nargs - if len(rargs) < nargs: - if nargs == 1: - self.error("%s option requires a value" % opt) - else: - self.error("%s option requires %d values" - % (opt, nargs)) - elif nargs == 1: - value = rargs.pop(0) - else: - value = tuple(rargs[0:nargs]) - del rargs[0:nargs] - - elif had_explicit_value: - self.error("%s option does not take a value" % opt) - - else: - value = None - - option.process(opt, value, values, self) - - def _process_short_opts (self, rargs, values): - arg = rargs.pop(0) - stop = 0 - i = 1 - for ch in arg[1:]: - opt = "-" + ch - option = self._short_opt.get(opt) - i += 1 # we have consumed a character - - if not option: - self.error("no such option: %s" % opt) - if option.takes_value(): - # Any characters left in arg? Pretend they're the - # next arg, and stop consuming characters of arg. - if i < len(arg): - rargs.insert(0, arg[i:]) - stop = 1 - - nargs = option.nargs - if len(rargs) < nargs: - if nargs == 1: - self.error("%s option requires a value" % opt) - else: - self.error("%s option requires %s values" - % (opt, nargs)) - elif nargs == 1: - value = rargs.pop(0) - else: - value = tuple(rargs[0:nargs]) - del rargs[0:nargs] - - else: # option doesn't take a value - value = None - - option.process(opt, value, values, self) - - if stop: - break - - - # -- Output/error methods ------------------------------------------ - - def error (self, msg): - self.print_usage(sys.stderr) - sys.exit("%s: error: %s" % (get_prog_name(), msg)) - - def print_usage (self, file=None): - if self.usage: - usage = self.usage.replace("%prog", get_prog_name()) - print >>file, usage - print >>file - - def print_version (self, file=None): - if self.version: - version = self.version.replace("%prog", get_prog_name()) - print >>file, version - - def print_help (self, file=None): - from distutils.fancy_getopt import wrap_text - - if file is None: - file = sys.stdout - - self.print_usage(file) - - # The help for each option consists of two parts: - # * the opt strings and metavars - # eg. ("-x", or "-fFILENAME, --file=FILENAME") - # * the user-supplied help string - # eg. ("turn on expert mode", "read data from FILENAME") - # - # If possible, we write both of these on the same line: - # -x turn on expert mode - # - # But if the opt string list is too long, we put the help - # string on a second line, indented to the same column it would - # start in if it fit on the first line. - # -fFILENAME, --file=FILENAME - # read data from FILENAME - - print >>file, "options:" - width = 78 # assume 80 cols for now - - option_help = [] # list of (string, string) tuples - lengths = [] - - for option in self.option_list: - takes_value = option.takes_value() - if takes_value: - metavar = option.metavar or option.dest.upper() - - opts = [] # list of "-a" or "--foo=FILE" strings - if option.help is SUPPRESS_HELP: - continue - - if takes_value: - for sopt in option._short_opts: - opts.append(sopt + metavar) - for lopt in option._long_opts: - opts.append(lopt + "=" + metavar) - else: - for opt in option._short_opts + option._long_opts: - opts.append(opt) - - opts = ", ".join(opts) - option_help.append((opts, option.help)) - lengths.append(len(opts)) - - max_opts = min(max(lengths), 20) - - for (opts, help) in option_help: - # how much to indent lines 2 .. N of help text - indent_rest = 2 + max_opts + 2 - help_width = width - indent_rest - - if len(opts) > max_opts: - opts = " " + opts + "\n" - indent_first = indent_rest - - else: # start help on same line as opts - opts = " %-*s " % (max_opts, opts) - indent_first = 0 - - file.write(opts) - - if help: - help_lines = wrap_text(help, help_width) - print >>file, "%*s%s" % (indent_first, "", help_lines[0]) - for line in help_lines[1:]: - print >>file, "%*s%s" % (indent_rest, "", line) - elif opts[-1] != "\n": - file.write("\n") - -# class OptionParser - - -def _match_abbrev (s, words): - """_match_abbrev(s : string, words : [string]) -> string - - Returns the string in 'words' for which 's' is an unambiguous - abbreviation. If 's' is found to be ambiguous or doesn't match any - of 'words', raises BadOptionError. - """ - match = None - for word in words: - # If s isn't even a prefix for this word, don't waste any - # more time on it: skip to the next word and try again. - if not word.startswith(s): - continue - - # Exact match? Great, return now. - if s == word: - return word - - # Now comes the tricky business of disambiguation. At this - # point, we know s is a proper prefix of word, eg. s='--foo' and - # word=='--foobar'. If we have already seen another word where - # this was the case, eg. '--foobaz', fail: s is ambiguous. - # Otherwise record this match and keep looping; we will return - # if we see an exact match, or when we fall out of the loop and - # it turns out that the current word is the match. - if match: - raise BadOptionError("ambiguous option: %s (%s, %s, ...?)" - % (s, match, word)) - match = word - - if match: - return match - else: - raise BadOptionError("no such option: %s" % s) diff --git a/src/Utils/statprof.py b/src/Utils/statprof.py new file mode 100644 index 0000000..55638eb --- /dev/null +++ b/src/Utils/statprof.py @@ -0,0 +1,304 @@ +## statprof.py +## Copyright (C) 2004,2005 Andy Wingo <wingo at pobox dot com> +## Copyright (C) 2001 Rob Browning <rlb at defaultvalue dot org> + +## This library is free software; you can redistribute it and/or +## modify it under the terms of the GNU Lesser General Public +## License as published by the Free Software Foundation; either +## version 2.1 of the License, or (at your option) any later version. +## +## This library is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +## Lesser General Public License for more details. +## +## You should have received a copy of the GNU Lesser General Public +## License along with this program; if not, contact: +## +## Free Software Foundation Voice: +1-617-542-5942 +## 59 Temple Place - Suite 330 Fax: +1-617-542-2652 +## Boston, MA 02111-1307, USA gnu@gnu.org + +""" +statprof is intended to be a fairly simple statistical profiler for +python. It was ported directly from a statistical profiler for guile, +also named statprof, available from guile-lib [0]. + +[0] http://wingolog.org/software/guile-lib/statprof/ + +To start profiling, call statprof.start(): +>>> start() + +Then run whatever it is that you want to profile, for example: +>>> import test.pystone; test.pystone.pystones() + +Then stop the profiling and print out the results: +>>> stop() +>>> display() + % cumulative self + time seconds seconds name + 26.72 1.40 0.37 pystone.py:79:Proc0 + 13.79 0.56 0.19 pystone.py:133:Proc1 + 13.79 0.19 0.19 pystone.py:208:Proc8 + 10.34 0.16 0.14 pystone.py:229:Func2 + 6.90 0.10 0.10 pystone.py:45:__init__ + 4.31 0.16 0.06 pystone.py:53:copy + ... + +All of the numerical data with the exception of the calls column is +statistically approximate. In the following column descriptions, and +in all of statprof, "time" refers to execution time (both user and +system), not wall clock time. + +% time + The percent of the time spent inside the procedure itself (not + counting children). + +cumulative seconds + The total number of seconds spent in the procedure, including + children. + +self seconds + The total number of seconds spent in the procedure itself (not + counting children). + +name + The name of the procedure. + +By default statprof keeps the data collected from previous runs. If you +want to clear the collected data, call reset(): +>>> reset() + +reset() can also be used to change the sampling frequency. For example, +to tell statprof to sample 50 times a second: +>>> reset(50) + +This means that statprof will sample the call stack after every 1/50 of +a second of user + system time spent running on behalf of the python +process. When your process is idle (for example, blocking in a read(), +as is the case at the listener), the clock does not advance. For this +reason statprof is not currently not suitable for profiling io-bound +operations. + +The profiler uses the hash of the code object itself to identify the +procedures, so it won't confuse different procedures with the same name. +They will show up as two different rows in the output. + +Right now the profiler is quite simplistic. I cannot provide +call-graphs or other higher level information. What you see in the +table is pretty much all there is. Patches are welcome :-) + + +Threading +--------- + +Because signals only get delivered to the main thread in Python, +statprof only profiles the main thread. However because the time +reporting function uses per-process timers, the results can be +significantly off if other threads' work patterns are not similar to the +main thread's work patterns. + + +Implementation notes +-------------------- + +The profiler works by setting the unix profiling signal ITIMER_PROF to +go off after the interval you define in the call to reset(). When the +signal fires, a sampling routine is run which looks at the current +procedure that's executing, and then crawls up the stack, and for each +frame encountered, increments that frame's code object's sample count. +Note that if a procedure is encountered multiple times on a given stack, +it is only counted once. After the sampling is complete, the profiler +resets profiling timer to fire again after the appropriate interval. + +Meanwhile, the profiler keeps track, via os.times(), how much CPU time +(system and user -- which is also what ITIMER_PROF tracks), has elapsed +while code has been executing within a start()/stop() block. + +The profiler also tries to avoid counting or timing its own code as +much as possible. +""" + + +from __future__ import division + +try: + import itimer +except ImportError: + raise ImportError('''statprof requires the itimer python extension. +To install it, enter the following commands from a terminal: + +wget http://www.cute.fi/~torppa/py-itimer/py-itimer.tar.gz +tar zxvf py-itimer.tar.gz +cd py-itimer +sudo python setup.py install +''') + +import signal +import os + + +__all__ = ['start', 'stop', 'reset', 'display'] + + +########################################################################### +## Utils + +def clock(): + times = os.times() + return times[0] + times[1] + + +########################################################################### +## Collection data structures + +class ProfileState(object): + def __init__(self, frequency=None): + self.reset(frequency) + + def reset(self, frequency=None): + # total so far + self.accumulated_time = 0.0 + # start_time when timer is active + self.last_start_time = None + # total count of sampler calls + self.sample_count = 0 + # a float + if frequency: + self.sample_interval = 1.0/frequency + elif not hasattr(self, 'sample_interval'): + # default to 100 Hz + self.sample_interval = 1.0/100.0 + else: + # leave the frequency as it was + pass + self.remaining_prof_time = None + # for user start/stop nesting + self.profile_level = 0 + # whether to catch apply-frame + self.count_calls = False + # gc time between start() and stop() + self.gc_time_taken = 0 + + def accumulate_time(self, stop_time): + self.accumulated_time += stop_time - self.last_start_time + +state = ProfileState() + +## call_data := { code object: CallData } +call_data = {} +class CallData(object): + def __init__(self, code): + self.name = code.co_name + self.filename = code.co_filename + self.lineno = code.co_firstlineno + self.call_count = 0 + self.cum_sample_count = 0 + self.self_sample_count = 0 + call_data[code] = self + +def get_call_data(code): + return call_data.get(code, None) or CallData(code) + + +########################################################################### +## SIGPROF handler + +def sample_stack_procs(frame): + state.sample_count += 1 + get_call_data(frame.f_code).self_sample_count += 1 + + code_seen = {} + while frame: + code_seen[frame.f_code] = True + frame = frame.f_back + for code in code_seen.iterkeys(): + get_call_data(code).cum_sample_count += 1 + +def profile_signal_handler(signum, frame): + if state.profile_level > 0: + state.accumulate_time(clock()) + sample_stack_procs(frame) + itimer.setitimer(itimer.ITIMER_PROF, + state.sample_interval, 0.0) + state.last_start_time = clock() + + +########################################################################### +## Profiling API + +def is_active(): + return state.profile_level > 0 + +def start(): + state.profile_level += 1 + if state.profile_level == 1: + state.last_start_time = clock() + rpt = state.remaining_prof_time + state.remaining_prof_time = None + signal.signal(signal.SIGPROF, profile_signal_handler) + itimer.setitimer(itimer.ITIMER_PROF, + rpt or state.sample_interval, 0.0) + state.gc_time_taken = 0 # dunno + +def stop(): + state.profile_level -= 1 + if state.profile_level == 0: + state.accumulate_time(clock()) + state.last_start_time = None + rpt = itimer.setitimer(itimer.ITIMER_PROF, 0.0, 0.0) + signal.signal(signal.SIGPROF, signal.SIG_IGN) + state.remaining_prof_time = rpt[0] + state.gc_time_taken = 0 # dunno + +def reset(frequency=None): + assert state.profile_level == 0, "Can't reset() while statprof is running" + call_data.clear() + state.reset(frequency) + + +########################################################################### +## Reporting API + +class CallStats(object): + def __init__(self, call_data): + self_samples = call_data.self_sample_count + cum_samples = call_data.cum_sample_count + nsamples = state.sample_count + secs_per_sample = state.accumulated_time / nsamples + basename = os.path.basename(call_data.filename) + + self.name = '%s:%d:%s' % (basename, call_data.lineno, call_data.name) + self.pcnt_time_in_proc = self_samples / nsamples * 100 + self.cum_secs_in_proc = cum_samples * secs_per_sample + self.self_secs_in_proc = self_samples * secs_per_sample + self.num_calls = None + self.self_secs_per_call = None + self.cum_secs_per_call = None + + def display(self): + print '%6.2f %9.2f %9.2f %s' % (self.pcnt_time_in_proc, + self.cum_secs_in_proc, + self.self_secs_in_proc, + self.name) + + +def display(): + if state.sample_count == 0: + print 'No samples recorded.' + return + + l = [CallStats(x) for x in call_data.itervalues()] + l = [(x.self_secs_in_proc, x.cum_secs_in_proc, x) for x in l] + l.sort(reverse=True) + l = [x[2] for x in l] + + print '%5.5s %10.10s %7.7s %-8.8s' % ('% ', 'cumulative', 'self', '') + print '%5.5s %9.9s %8.8s %-8.8s' % ("time", "seconds", "seconds", "name") + + for x in l: + x.display() + + print '---' + print 'Sample count: %d' % state.sample_count + print 'Total time: %f seconds' % state.accumulated_time diff --git a/src/Version.py b/src/Version.py index 27f0725..30bf439 100644 --- a/src/Version.py +++ b/src/Version.py @@ -1,5 +1,5 @@ -Version = '2.1.0.1' -VersionTuple = (2,1,0,1,'final',0) +Version = '2.1.1' +VersionTuple = (2,1,1,'final',0) MinCompatibleVersion = '2.0rc6' MinCompatibleVersionTuple = (2,0,0,'candidate',6) diff --git a/src/_namemapper.c b/src/_namemapper.c index 0bd947e..5767e16 100644 --- a/src/_namemapper.c +++ b/src/_namemapper.c @@ -212,56 +212,57 @@ PyNamemapper_valueForName(PyObject *obj, char *nameChunks[], int numChunks, int executeCallables) { - int i; - char *currentKey; - PyObject *currentVal = NULL; - PyObject *nextVal = NULL; - - currentVal = obj; - for (i=0; i < numChunks;i++) { - currentKey = nameChunks[i]; - if (PyErr_CheckSignals()) { /* not sure if I really need to do this here, but what the hell */ - if (i>0) { - Py_DECREF(currentVal); - } - return NULL; - } - - if (PyMapping_Check(currentVal) && PyMapping_HasKeyString(currentVal, currentKey)) { - nextVal = PyMapping_GetItemString(currentVal, currentKey); - } else { - nextVal = PyObject_GetAttrString(currentVal, currentKey); - PyObject *exc = PyErr_Occurred(); - if (exc != NULL) { - // if exception == AttributeError - if (PyErr_ExceptionMatches(PyExc_AttributeError)) { - setNotFoundException(currentKey, currentVal); - if (i > 0) + int i; + char *currentKey; + PyObject *currentVal = NULL; + PyObject *nextVal = NULL; + + currentVal = obj; + for (i=0; i < numChunks;i++) { + currentKey = nameChunks[i]; + if (PyErr_CheckSignals()) { /* not sure if I really need to do this here, but what the hell */ + if (i>0) { Py_DECREF(currentVal); + } return NULL; } - } - } - if (i > 0) - Py_DECREF(currentVal); - - if (executeCallables && PyCallable_Check(nextVal) && (!isInstanceOrClass(nextVal)) ) { - - //if (executeCallables && PyCallable_Check(nextVal) && (!PyInstance_Check(nextVal)) - //&& (!PyClass_Check(nextVal)) && (!PyType_Check(nextVal)) ) { - + + if (PyMapping_Check(currentVal) && PyMapping_HasKeyString(currentVal, currentKey)) { + nextVal = PyMapping_GetItemString(currentVal, currentKey); + } + else { + nextVal = PyObject_GetAttrString(currentVal, currentKey); + PyObject *exc = PyErr_Occurred(); + if (exc != NULL) { + // if exception == AttributeError + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + setNotFoundException(currentKey, currentVal); + if (i > 0) { + Py_DECREF(currentVal); + } + return NULL; + } + } + } + if (i > 0) { + Py_DECREF(currentVal); + } - if (!(currentVal = PyObject_CallObject(nextVal, NULL))){ - Py_DECREF(nextVal); - return NULL; - }; - Py_DECREF(nextVal); - } else { - currentVal = nextVal; + if (executeCallables && PyCallable_Check(nextVal) && (!isInstanceOrClass(nextVal)) ) { + //if (executeCallables && PyCallable_Check(nextVal) && (!PyInstance_Check(nextVal)) + //&& (!PyClass_Check(nextVal)) && (!PyType_Check(nextVal)) ) { + if (!(currentVal = PyObject_CallObject(nextVal, NULL))) { + Py_DECREF(nextVal); + return NULL; + } + + Py_DECREF(nextVal); + } else { + currentVal = nextVal; + } } - } - return currentVal; + return currentVal; } diff --git a/src/contrib/__init__.py b/src/contrib/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/src/contrib/__init__.py diff --git a/src/contrib/markdown/LICENSE b/src/contrib/markdown/LICENSE new file mode 100644 index 0000000..4cd8b14 --- /dev/null +++ b/src/contrib/markdown/LICENSE @@ -0,0 +1,30 @@ +Copyright 2007, 2008 The Python Markdown Project (v. 1.7 and later) +Copyright 2004, 2005, 2006 Yuri Takhteyev (v. 0.2-1.6b) +Copyright 2004 Manfred Stienstra (the original version) + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +* Neither the name of the <organization> nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE PYTHON MARKDOWN PROJECT ''AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL ANY CONTRIBUTORS TO THE PYTHON MARKDOWN PROJECT +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + diff --git a/src/contrib/markdown/__init__.py b/src/contrib/markdown/__init__.py new file mode 100644 index 0000000..af5a2c1 --- /dev/null +++ b/src/contrib/markdown/__init__.py @@ -0,0 +1,603 @@ +""" +Python Markdown +=============== + +Python Markdown converts Markdown to HTML and can be used as a library or +called from the command line. + +## Basic usage as a module: + + import markdown + md = Markdown() + html = md.convert(your_text_string) + +## Basic use from the command line: + + python markdown.py source.txt > destination.html + +Run "python markdown.py --help" to see more options. + +## Extensions + +See <http://www.freewisdom.org/projects/python-markdown/> for more +information and instructions on how to extend the functionality of +Python Markdown. Read that before you try modifying this file. + +## Authors and License + +Started by [Manfred Stienstra](http://www.dwerg.net/). Continued and +maintained by [Yuri Takhteyev](http://www.freewisdom.org), [Waylan +Limberg](http://achinghead.com/) and [Artem Yunusov](http://blog.splyer.com). + +Contact: markdown@freewisdom.org + +Copyright 2007, 2008 The Python Markdown Project (v. 1.7 and later) +Copyright 200? Django Software Foundation (OrderedDict implementation) +Copyright 2004, 2005, 2006 Yuri Takhteyev (v. 0.2-1.6b) +Copyright 2004 Manfred Stienstra (the original version) + +License: BSD (see docs/LICENSE for details). +""" + +version = "2.0-rc2" +version_info = (2,0,0, "rc2") + +import re +import codecs +import sys +import warnings +import logging +from logging import DEBUG, INFO, WARN, ERROR, CRITICAL + + +""" +CONSTANTS +============================================================================= +""" + +""" +Constants you might want to modify +----------------------------------------------------------------------------- +""" + +# default logging level for command-line use +COMMAND_LINE_LOGGING_LEVEL = CRITICAL +TAB_LENGTH = 4 # expand tabs to this many spaces +ENABLE_ATTRIBUTES = True # @id = xyz -> <... id="xyz"> +SMART_EMPHASIS = True # this_or_that does not become this<i>or</i>that +DEFAULT_OUTPUT_FORMAT = 'xhtml1' # xhtml or html4 output +HTML_REMOVED_TEXT = "[HTML_REMOVED]" # text used instead of HTML in safe mode +BLOCK_LEVEL_ELEMENTS = re.compile("p|div|h[1-6]|blockquote|pre|table|dl|ol|ul" + "|script|noscript|form|fieldset|iframe|math" + "|ins|del|hr|hr/|style|li|dt|dd|thead|tbody" + "|tr|th|td") +DOC_TAG = "div" # Element used to wrap document - later removed + +# Placeholders +STX = u'\u0002' # Use STX ("Start of text") for start-of-placeholder +ETX = u'\u0003' # Use ETX ("End of text") for end-of-placeholder +INLINE_PLACEHOLDER_PREFIX = STX+"klzzwxh:" +INLINE_PLACEHOLDER = INLINE_PLACEHOLDER_PREFIX + "%s" + ETX +AMP_SUBSTITUTE = STX+"amp"+ETX + + +""" +Constants you probably do not need to change +----------------------------------------------------------------------------- +""" + +RTL_BIDI_RANGES = ( (u'\u0590', u'\u07FF'), + # Hebrew (0590-05FF), Arabic (0600-06FF), + # Syriac (0700-074F), Arabic supplement (0750-077F), + # Thaana (0780-07BF), Nko (07C0-07FF). + (u'\u2D30', u'\u2D7F'), # Tifinagh + ) + + +""" +AUXILIARY GLOBAL FUNCTIONS +============================================================================= +""" + + +def message(level, text): + """ A wrapper method for logging debug messages. """ + logger = logging.getLogger('MARKDOWN') + if logger.handlers: + # The logger is configured + logger.log(level, text) + if level > WARN: + sys.exit(0) + elif level > WARN: + raise MarkdownException, text + else: + warnings.warn(text, MarkdownWarning) + + +def isBlockLevel(tag): + """Check if the tag is a block level HTML tag.""" + return BLOCK_LEVEL_ELEMENTS.match(tag) + +""" +MISC AUXILIARY CLASSES +============================================================================= +""" + +class AtomicString(unicode): + """A string which should not be further processed.""" + pass + + +class MarkdownException(Exception): + """ A Markdown Exception. """ + pass + + +class MarkdownWarning(Warning): + """ A Markdown Warning. """ + pass + + +""" +OVERALL DESIGN +============================================================================= + +Markdown processing takes place in four steps: + +1. A bunch of "preprocessors" munge the input text. +2. BlockParser() parses the high-level structural elements of the + pre-processed text into an ElementTree. +3. A bunch of "treeprocessors" are run against the ElementTree. One such + treeprocessor runs InlinePatterns against the ElementTree, detecting inline + markup. +4. Some post-processors are run against the text after the ElementTree has + been serialized into text. +5. The output is written to a string. + +Those steps are put together by the Markdown() class. + +""" + +import preprocessors +import blockprocessors +import treeprocessors +import inlinepatterns +import postprocessors +import blockparser +import etree_loader +import odict + +# Extensions should use "markdown.etree" instead of "etree" (or do `from +# markdown import etree`). Do not import it by yourself. + +etree = etree_loader.importETree() + +# Adds the ability to output html4 +import html4 + + +class Markdown: + """Convert Markdown to HTML.""" + + def __init__(self, + extensions=[], + extension_configs={}, + safe_mode = False, + output_format=DEFAULT_OUTPUT_FORMAT): + """ + Creates a new Markdown instance. + + Keyword arguments: + + * extensions: A list of extensions. + If they are of type string, the module mdx_name.py will be loaded. + If they are a subclass of markdown.Extension, they will be used + as-is. + * extension-configs: Configuration setting for extensions. + * safe_mode: Disallow raw html. One of "remove", "replace" or "escape". + * output_format: Format of output. Supported formats are: + * "xhtml1": Outputs XHTML 1.x. Default. + * "xhtml": Outputs latest supported version of XHTML (currently XHTML 1.1). + * "html4": Outputs HTML 4 + * "html": Outputs latest supported version of HTML (currently HTML 4). + Note that it is suggested that the more specific formats ("xhtml1" + and "html4") be used as "xhtml" or "html" may change in the future + if it makes sense at that time. + + """ + + self.safeMode = safe_mode + self.registeredExtensions = [] + self.docType = "" + self.stripTopLevelTags = True + + # Preprocessors + self.preprocessors = odict.OrderedDict() + self.preprocessors["html_block"] = \ + preprocessors.HtmlBlockPreprocessor(self) + self.preprocessors["reference"] = \ + preprocessors.ReferencePreprocessor(self) + # footnote preprocessor will be inserted with "<reference" + + # Block processors - ran by the parser + self.parser = blockparser.BlockParser() + self.parser.blockprocessors['empty'] = \ + blockprocessors.EmptyBlockProcessor(self.parser) + self.parser.blockprocessors['indent'] = \ + blockprocessors.ListIndentProcessor(self.parser) + self.parser.blockprocessors['code'] = \ + blockprocessors.CodeBlockProcessor(self.parser) + self.parser.blockprocessors['hashheader'] = \ + blockprocessors.HashHeaderProcessor(self.parser) + self.parser.blockprocessors['setextheader'] = \ + blockprocessors.SetextHeaderProcessor(self.parser) + self.parser.blockprocessors['hr'] = \ + blockprocessors.HRProcessor(self.parser) + self.parser.blockprocessors['olist'] = \ + blockprocessors.OListProcessor(self.parser) + self.parser.blockprocessors['ulist'] = \ + blockprocessors.UListProcessor(self.parser) + self.parser.blockprocessors['quote'] = \ + blockprocessors.BlockQuoteProcessor(self.parser) + self.parser.blockprocessors['paragraph'] = \ + blockprocessors.ParagraphProcessor(self.parser) + + + #self.prePatterns = [] + + # Inline patterns - Run on the tree + self.inlinePatterns = odict.OrderedDict() + self.inlinePatterns["backtick"] = \ + inlinepatterns.BacktickPattern(inlinepatterns.BACKTICK_RE) + self.inlinePatterns["escape"] = \ + inlinepatterns.SimpleTextPattern(inlinepatterns.ESCAPE_RE) + self.inlinePatterns["reference"] = \ + inlinepatterns.ReferencePattern(inlinepatterns.REFERENCE_RE, self) + self.inlinePatterns["link"] = \ + inlinepatterns.LinkPattern(inlinepatterns.LINK_RE, self) + self.inlinePatterns["image_link"] = \ + inlinepatterns.ImagePattern(inlinepatterns.IMAGE_LINK_RE, self) + self.inlinePatterns["image_reference"] = \ + inlinepatterns.ImageReferencePattern(inlinepatterns.IMAGE_REFERENCE_RE, self) + self.inlinePatterns["autolink"] = \ + inlinepatterns.AutolinkPattern(inlinepatterns.AUTOLINK_RE, self) + self.inlinePatterns["automail"] = \ + inlinepatterns.AutomailPattern(inlinepatterns.AUTOMAIL_RE, self) + self.inlinePatterns["linebreak2"] = \ + inlinepatterns.SubstituteTagPattern(inlinepatterns.LINE_BREAK_2_RE, 'br') + self.inlinePatterns["linebreak"] = \ + inlinepatterns.SubstituteTagPattern(inlinepatterns.LINE_BREAK_RE, 'br') + self.inlinePatterns["html"] = \ + inlinepatterns.HtmlPattern(inlinepatterns.HTML_RE, self) + self.inlinePatterns["entity"] = \ + inlinepatterns.HtmlPattern(inlinepatterns.ENTITY_RE, self) + self.inlinePatterns["not_strong"] = \ + inlinepatterns.SimpleTextPattern(inlinepatterns.NOT_STRONG_RE) + self.inlinePatterns["strong_em"] = \ + inlinepatterns.DoubleTagPattern(inlinepatterns.STRONG_EM_RE, 'strong,em') + self.inlinePatterns["strong"] = \ + inlinepatterns.SimpleTagPattern(inlinepatterns.STRONG_RE, 'strong') + self.inlinePatterns["emphasis"] = \ + inlinepatterns.SimpleTagPattern(inlinepatterns.EMPHASIS_RE, 'em') + self.inlinePatterns["emphasis2"] = \ + inlinepatterns.SimpleTagPattern(inlinepatterns.EMPHASIS_2_RE, 'em') + # The order of the handlers matters!!! + + + # Tree processors - run once we have a basic parse. + self.treeprocessors = odict.OrderedDict() + self.treeprocessors["inline"] = treeprocessors.InlineProcessor(self) + self.treeprocessors["prettify"] = \ + treeprocessors.PrettifyTreeprocessor(self) + + # Postprocessors - finishing touches. + self.postprocessors = odict.OrderedDict() + self.postprocessors["raw_html"] = \ + postprocessors.RawHtmlPostprocessor(self) + self.postprocessors["amp_substitute"] = \ + postprocessors.AndSubstitutePostprocessor() + # footnote postprocessor will be inserted with ">amp_substitute" + + # Map format keys to serializers + self.output_formats = { + 'html' : html4.to_html_string, + 'html4' : html4.to_html_string, + 'xhtml' : etree.tostring, + 'xhtml1': etree.tostring, + } + + self.references = {} + self.htmlStash = preprocessors.HtmlStash() + self.registerExtensions(extensions = extensions, + configs = extension_configs) + self.set_output_format(output_format) + self.reset() + + def registerExtensions(self, extensions, configs): + """ + Register extensions with this instance of Markdown. + + Keyword aurguments: + + * extensions: A list of extensions, which can either + be strings or objects. See the docstring on Markdown. + * configs: A dictionary mapping module names to config options. + + """ + for ext in extensions: + if isinstance(ext, basestring): + ext = load_extension(ext, configs.get(ext, [])) + try: + ext.extendMarkdown(self, globals()) + except AttributeError: + message(ERROR, "Incorrect type! Extension '%s' is " + "neither a string or an Extension." %(repr(ext))) + + + def registerExtension(self, extension): + """ This gets called by the extension """ + self.registeredExtensions.append(extension) + + def reset(self): + """ + Resets all state variables so that we can start with a new text. + """ + self.htmlStash.reset() + self.references.clear() + + for extension in self.registeredExtensions: + extension.reset() + + def set_output_format(self, format): + """ Set the output format for the class instance. """ + try: + self.serializer = self.output_formats[format.lower()] + except KeyError: + message(CRITICAL, 'Invalid Output Format: "%s". Use one of %s.' \ + % (format, self.output_formats.keys())) + + def convert(self, source): + """ + Convert markdown to serialized XHTML or HTML. + + Keyword arguments: + + * source: Source text as a Unicode string. + + """ + + # Fixup the source text + if not source.strip(): + return u"" # a blank unicode string + try: + source = unicode(source) + except UnicodeDecodeError: + message(CRITICAL, 'UnicodeDecodeError: Markdown only accepts unicode or ascii input.') + return u"" + + source = source.replace(STX, "").replace(ETX, "") + source = source.replace("\r\n", "\n").replace("\r", "\n") + "\n\n" + source = re.sub(r'\n\s+\n', '\n\n', source) + source = source.expandtabs(TAB_LENGTH) + + # Split into lines and run the line preprocessors. + self.lines = source.split("\n") + for prep in self.preprocessors.values(): + self.lines = prep.run(self.lines) + + # Parse the high-level elements. + root = self.parser.parseDocument(self.lines).getroot() + + # Run the tree-processors + for treeprocessor in self.treeprocessors.values(): + newRoot = treeprocessor.run(root) + if newRoot: + root = newRoot + + # Serialize _properly_. Strip top-level tags. + output, length = codecs.utf_8_decode(self.serializer(root, encoding="utf8")) + if self.stripTopLevelTags: + start = output.index('<%s>'%DOC_TAG)+len(DOC_TAG)+2 + end = output.rindex('</%s>'%DOC_TAG) + output = output[start:end].strip() + + # Run the text post-processors + for pp in self.postprocessors.values(): + output = pp.run(output) + + return output.strip() + + def convertFile(self, input=None, output=None, encoding=None): + """Converts a markdown file and returns the HTML as a unicode string. + + Decodes the file using the provided encoding (defaults to utf-8), + passes the file content to markdown, and outputs the html to either + the provided stream or the file with provided name, using the same + encoding as the source file. + + **Note:** This is the only place that decoding and encoding of unicode + takes place in Python-Markdown. (All other code is unicode-in / + unicode-out.) + + Keyword arguments: + + * input: Name of source text file. + * output: Name of output file. Writes to stdout if `None`. + * encoding: Encoding of input and output files. Defaults to utf-8. + + """ + + encoding = encoding or "utf-8" + + # Read the source + input_file = codecs.open(input, mode="r", encoding=encoding) + text = input_file.read() + input_file.close() + text = text.lstrip(u'\ufeff') # remove the byte-order mark + + # Convert + html = self.convert(text) + + # Write to file or stdout + if isinstance(output, (str, unicode)): + output_file = codecs.open(output, "w", encoding=encoding) + output_file.write(html) + output_file.close() + else: + output.write(html.encode(encoding)) + + +""" +Extensions +----------------------------------------------------------------------------- +""" + +class Extension: + """ Base class for extensions to subclass. """ + def __init__(self, configs = {}): + """Create an instance of an Extention. + + Keyword arguments: + + * configs: A dict of configuration setting used by an Extension. + """ + self.config = configs + + def getConfig(self, key): + """ Return a setting for the given key or an empty string. """ + if key in self.config: + return self.config[key][0] + else: + return "" + + def getConfigInfo(self): + """ Return all config settings as a list of tuples. """ + return [(key, self.config[key][1]) for key in self.config.keys()] + + def setConfig(self, key, value): + """ Set a config setting for `key` with the given `value`. """ + self.config[key][0] = value + + def extendMarkdown(self, md, md_globals): + """ + Add the various proccesors and patterns to the Markdown Instance. + + This method must be overriden by every extension. + + Keyword arguments: + + * md: The Markdown instance. + + * md_globals: Global variables in the markdown module namespace. + + """ + pass + + +def load_extension(ext_name, configs = []): + """Load extension by name, then return the module. + + The extension name may contain arguments as part of the string in the + following format: "extname(key1=value1,key2=value2)" + + """ + + # Parse extensions config params (ignore the order) + configs = dict(configs) + pos = ext_name.find("(") # find the first "(" + if pos > 0: + ext_args = ext_name[pos+1:-1] + ext_name = ext_name[:pos] + pairs = [x.split("=") for x in ext_args.split(",")] + configs.update([(x.strip(), y.strip()) for (x, y) in pairs]) + + # Setup the module names + ext_module = 'markdown.extensions' + module_name_new_style = '.'.join([ext_module, ext_name]) + module_name_old_style = '_'.join(['mdx', ext_name]) + + # Try loading the extention first from one place, then another + try: # New style (markdown.extensons.<extension>) + module = __import__(module_name_new_style, {}, {}, [ext_module]) + except ImportError: + try: # Old style (mdx.<extension>) + module = __import__(module_name_old_style) + except ImportError: + message(WARN, "Failed loading extension '%s' from '%s' or '%s'" + % (ext_name, module_name_new_style, module_name_old_style)) + # Return None so we don't try to initiate none-existant extension + return None + + # If the module is loaded successfully, we expect it to define a + # function called makeExtension() + try: + return module.makeExtension(configs.items()) + except AttributeError: + message(CRITICAL, "Failed to initiate extension '%s'" % ext_name) + + +def load_extensions(ext_names): + """Loads multiple extensions""" + extensions = [] + for ext_name in ext_names: + extension = load_extension(ext_name) + if extension: + extensions.append(extension) + return extensions + + +""" +EXPORTED FUNCTIONS +============================================================================= + +Those are the two functions we really mean to export: markdown() and +markdownFromFile(). +""" + +def markdown(text, + extensions = [], + safe_mode = False, + output_format = DEFAULT_OUTPUT_FORMAT): + """Convert a markdown string to HTML and return HTML as a unicode string. + + This is a shortcut function for `Markdown` class to cover the most + basic use case. It initializes an instance of Markdown, loads the + necessary extensions and runs the parser on the given text. + + Keyword arguments: + + * text: Markdown formatted text as Unicode or ASCII string. + * extensions: A list of extensions or extension names (may contain config args). + * safe_mode: Disallow raw html. One of "remove", "replace" or "escape". + * output_format: Format of output. Supported formats are: + * "xhtml1": Outputs XHTML 1.x. Default. + * "xhtml": Outputs latest supported version of XHTML (currently XHTML 1.1). + * "html4": Outputs HTML 4 + * "html": Outputs latest supported version of HTML (currently HTML 4). + Note that it is suggested that the more specific formats ("xhtml1" + and "html4") be used as "xhtml" or "html" may change in the future + if it makes sense at that time. + + Returns: An HTML document as a string. + + """ + md = Markdown(extensions=load_extensions(extensions), + safe_mode=safe_mode, + output_format=output_format) + return md.convert(text) + + +def markdownFromFile(input = None, + output = None, + extensions = [], + encoding = None, + safe_mode = False, + output_format = DEFAULT_OUTPUT_FORMAT): + """Read markdown code from a file and write it to a file or a stream.""" + md = Markdown(extensions=load_extensions(extensions), + safe_mode=safe_mode, + output_format=output_format) + md.convertFile(input, output, encoding) + + + diff --git a/src/contrib/markdown/blockparser.py b/src/contrib/markdown/blockparser.py new file mode 100644 index 0000000..e18b338 --- /dev/null +++ b/src/contrib/markdown/blockparser.py @@ -0,0 +1,95 @@ + +import markdown + +class State(list): + """ Track the current and nested state of the parser. + + This utility class is used to track the state of the BlockParser and + support multiple levels if nesting. It's just a simple API wrapped around + a list. Each time a state is set, that state is appended to the end of the + list. Each time a state is reset, that state is removed from the end of + the list. + + Therefore, each time a state is set for a nested block, that state must be + reset when we back out of that level of nesting or the state could be + corrupted. + + While all the methods of a list object are available, only the three + defined below need be used. + + """ + + def set(self, state): + """ Set a new state. """ + self.append(state) + + def reset(self): + """ Step back one step in nested state. """ + self.pop() + + def isstate(self, state): + """ Test that top (current) level is of given state. """ + if len(self): + return self[-1] == state + else: + return False + +class BlockParser: + """ Parse Markdown blocks into an ElementTree object. + + A wrapper class that stitches the various BlockProcessors together, + looping through them and creating an ElementTree object. + """ + + def __init__(self): + self.blockprocessors = markdown.odict.OrderedDict() + self.state = State() + + def parseDocument(self, lines): + """ Parse a markdown document into an ElementTree. + + Given a list of lines, an ElementTree object (not just a parent Element) + is created and the root element is passed to the parser as the parent. + The ElementTree object is returned. + + This should only be called on an entire document, not pieces. + + """ + # Create a ElementTree from the lines + self.root = markdown.etree.Element(markdown.DOC_TAG) + self.parseChunk(self.root, '\n'.join(lines)) + return markdown.etree.ElementTree(self.root) + + def parseChunk(self, parent, text): + """ Parse a chunk of markdown text and attach to given etree node. + + While the ``text`` argument is generally assumed to contain multiple + blocks which will be split on blank lines, it could contain only one + block. Generally, this method would be called by extensions when + block parsing is required. + + The ``parent`` etree Element passed in is altered in place. + Nothing is returned. + + """ + self.parseBlocks(parent, text.split('\n\n')) + + def parseBlocks(self, parent, blocks): + """ Process blocks of markdown text and attach to given etree node. + + Given a list of ``blocks``, each blockprocessor is stepped through + until there are no blocks left. While an extension could potentially + call this method directly, it's generally expected to be used internally. + + This is a public method as an extension may need to add/alter additional + BlockProcessors which call this method to recursively parse a nested + block. + + """ + while blocks: + for processor in self.blockprocessors.values(): + if processor.test(parent, blocks[0]): + processor.run(parent, blocks) + break + + diff --git a/src/contrib/markdown/blockprocessors.py b/src/contrib/markdown/blockprocessors.py new file mode 100644 index 0000000..79f4db9 --- /dev/null +++ b/src/contrib/markdown/blockprocessors.py @@ -0,0 +1,460 @@ +""" +CORE MARKDOWN BLOCKPARSER +============================================================================= + +This parser handles basic parsing of Markdown blocks. It doesn't concern itself +with inline elements such as **bold** or *italics*, but rather just catches +blocks, lists, quotes, etc. + +The BlockParser is made up of a bunch of BlockProssors, each handling a +different type of block. Extensions may add/replace/remove BlockProcessors +as they need to alter how markdown blocks are parsed. + +""" + +import re +import markdown + +class BlockProcessor: + """ Base class for block processors. + + Each subclass will provide the methods below to work with the source and + tree. Each processor will need to define it's own ``test`` and ``run`` + methods. The ``test`` method should return True or False, to indicate + whether the current block should be processed by this processor. If the + test passes, the parser will call the processors ``run`` method. + + """ + + def __init__(self, parser=None): + self.parser = parser + + def lastChild(self, parent): + """ Return the last child of an etree element. """ + if len(parent): + return parent[-1] + else: + return None + + def detab(self, text): + """ Remove a tab from the front of each line of the given text. """ + newtext = [] + lines = text.split('\n') + for line in lines: + if line.startswith(' '*markdown.TAB_LENGTH): + newtext.append(line[markdown.TAB_LENGTH:]) + elif not line.strip(): + newtext.append('') + else: + break + return '\n'.join(newtext), '\n'.join(lines[len(newtext):]) + + def looseDetab(self, text, level=1): + """ Remove a tab from front of lines but allowing dedented lines. """ + lines = text.split('\n') + for i in range(len(lines)): + if lines[i].startswith(' '*markdown.TAB_LENGTH*level): + lines[i] = lines[i][markdown.TAB_LENGTH*level:] + return '\n'.join(lines) + + def test(self, parent, block): + """ Test for block type. Must be overridden by subclasses. + + As the parser loops through processors, it will call the ``test`` method + on each to determine if the given block of text is of that type. This + method must return a boolean ``True`` or ``False``. The actual method of + testing is left to the needs of that particular block type. It could + be as simple as ``block.startswith(some_string)`` or a complex regular + expression. As the block type may be different depending on the parent + of the block (i.e. inside a list), the parent etree element is also + provided and may be used as part of the test. + + Keywords: + + * ``parent``: A etree element which will be the parent of the block. + * ``block``: A block of text from the source which has been split at + blank lines. + """ + pass + + def run(self, parent, blocks): + """ Run processor. Must be overridden by subclasses. + + When the parser determines the appropriate type of a block, the parser + will call the corresponding processor's ``run`` method. This method + should parse the individual lines of the block and append them to + the etree. + + Note that both the ``parent`` and ``etree`` keywords are pointers + to instances of the objects which should be edited in place. Each + processor must make changes to the existing objects as there is no + mechanism to return new/different objects to replace them. + + This means that this method should be adding SubElements or adding text + to the parent, and should remove (``pop``) or add (``insert``) items to + the list of blocks. + + Keywords: + + * ``parent``: A etree element which is the parent of the current block. + * ``blocks``: A list of all remaining blocks of the document. + """ + pass + + +class ListIndentProcessor(BlockProcessor): + """ Process children of list items. + + Example: + * a list item + process this part + + or this part + + """ + + INDENT_RE = re.compile(r'^(([ ]{%s})+)'% markdown.TAB_LENGTH) + ITEM_TYPES = ['li'] + LIST_TYPES = ['ul', 'ol'] + + def test(self, parent, block): + return block.startswith(' '*markdown.TAB_LENGTH) and \ + not self.parser.state.isstate('detabbed') and \ + (parent.tag in self.ITEM_TYPES or \ + (len(parent) and parent[-1] and \ + (parent[-1].tag in self.LIST_TYPES) + ) + ) + + def run(self, parent, blocks): + block = blocks.pop(0) + level, sibling = self.get_level(parent, block) + block = self.looseDetab(block, level) + + self.parser.state.set('detabbed') + if parent.tag in self.ITEM_TYPES: + # The parent is already a li. Just parse the child block. + self.parser.parseBlocks(parent, [block]) + elif sibling.tag in self.ITEM_TYPES: + # The sibling is a li. Use it as parent. + self.parser.parseBlocks(sibling, [block]) + elif len(sibling) and sibling[-1].tag in self.ITEM_TYPES: + # The parent is a list (``ol`` or ``ul``) which has children. + # Assume the last child li is the parent of this block. + if sibling[-1].text: + # If the parent li has text, that text needs to be moved to a p + block = '%s\n\n%s' % (sibling[-1].text, block) + sibling[-1].text = '' + self.parser.parseChunk(sibling[-1], block) + else: + self.create_item(sibling, block) + self.parser.state.reset() + + def create_item(self, parent, block): + """ Create a new li and parse the block with it as the parent. """ + li = markdown.etree.SubElement(parent, 'li') + self.parser.parseBlocks(li, [block]) + + def get_level(self, parent, block): + """ Get level of indent based on list level. """ + # Get indent level + m = self.INDENT_RE.match(block) + if m: + indent_level = len(m.group(1))/markdown.TAB_LENGTH + else: + indent_level = 0 + if self.parser.state.isstate('list'): + # We're in a tightlist - so we already are at correct parent. + level = 1 + else: + # We're in a looselist - so we need to find parent. + level = 0 + # Step through children of tree to find matching indent level. + while indent_level > level: + child = self.lastChild(parent) + if child and (child.tag in self.LIST_TYPES or child.tag in self.ITEM_TYPES): + if child.tag in self.LIST_TYPES: + level += 1 + parent = child + else: + # No more child levels. If we're short of indent_level, + # we have a code block. So we stop here. + break + return level, parent + + +class CodeBlockProcessor(BlockProcessor): + """ Process code blocks. """ + + def test(self, parent, block): + return block.startswith(' '*markdown.TAB_LENGTH) + + def run(self, parent, blocks): + sibling = self.lastChild(parent) + block = blocks.pop(0) + theRest = '' + if sibling and sibling.tag == "pre" and len(sibling) \ + and sibling[0].tag == "code": + # The previous block was a code block. As blank lines do not start + # new code blocks, append this block to the previous, adding back + # linebreaks removed from the split into a list. + code = sibling[0] + block, theRest = self.detab(block) + code.text = markdown.AtomicString('%s\n%s\n' % (code.text, block.rstrip())) + else: + # This is a new codeblock. Create the elements and insert text. + pre = markdown.etree.SubElement(parent, 'pre') + code = markdown.etree.SubElement(pre, 'code') + block, theRest = self.detab(block) + code.text = markdown.AtomicString('%s\n' % block.rstrip()) + if theRest: + # This block contained unindented line(s) after the first indented + # line. Insert these lines as the first block of the master blocks + # list for future processing. + blocks.insert(0, theRest) + + +class BlockQuoteProcessor(BlockProcessor): + + RE = re.compile(r'(^|\n)[ ]{0,3}>[ ]?(.*)') + + def test(self, parent, block): + return bool(self.RE.search(block)) + + def run(self, parent, blocks): + block = blocks.pop(0) + m = self.RE.search(block) + if m: + before = block[:m.start()] # Lines before blockquote + # Pass lines before blockquote in recursively for parsing forst. + self.parser.parseBlocks(parent, [before]) + # Remove ``> `` from begining of each line. + block = '\n'.join([self.clean(line) for line in + block[m.start():].split('\n')]) + sibling = self.lastChild(parent) + if sibling and sibling.tag == "blockquote": + # Previous block was a blockquote so set that as this blocks parent + quote = sibling + else: + # This is a new blockquote. Create a new parent element. + quote = markdown.etree.SubElement(parent, 'blockquote') + # Recursively parse block with blockquote as parent. + self.parser.parseChunk(quote, block) + + def clean(self, line): + """ Remove ``>`` from beginning of a line. """ + m = self.RE.match(line) + if line.strip() == ">": + return "" + elif m: + return m.group(2) + else: + return line + +class OListProcessor(BlockProcessor): + """ Process ordered list blocks. """ + + TAG = 'ol' + # Detect an item (``1. item``). ``group(1)`` contains contents of item. + RE = re.compile(r'^[ ]{0,3}\d+\.[ ](.*)') + # Detect items on secondary lines. they can be of either list type. + CHILD_RE = re.compile(r'^[ ]{0,3}((\d+\.)|[*+-])[ ](.*)') + # Detect indented (nested) items of either type + INDENT_RE = re.compile(r'^[ ]{4,7}((\d+\.)|[*+-])[ ].*') + + def test(self, parent, block): + return bool(self.RE.match(block)) + + def run(self, parent, blocks): + # Check fr multiple items in one block. + items = self.get_items(blocks.pop(0)) + sibling = self.lastChild(parent) + if sibling and sibling.tag in ['ol', 'ul']: + # Previous block was a list item, so set that as parent + lst = sibling + # make sure previous item is in a p. + if len(lst) and lst[-1].text and not len(lst[-1]): + p = markdown.etree.SubElement(lst[-1], 'p') + p.text = lst[-1].text + lst[-1].text = '' + # parse first block differently as it gets wrapped in a p. + li = markdown.etree.SubElement(lst, 'li') + self.parser.state.set('looselist') + firstitem = items.pop(0) + self.parser.parseBlocks(li, [firstitem]) + self.parser.state.reset() + else: + # This is a new list so create parent with appropriate tag. + lst = markdown.etree.SubElement(parent, self.TAG) + self.parser.state.set('list') + # Loop through items in block, recursively parsing each with the + # appropriate parent. + for item in items: + if item.startswith(' '*markdown.TAB_LENGTH): + # Item is indented. Parse with last item as parent + self.parser.parseBlocks(lst[-1], [item]) + else: + # New item. Create li and parse with it as parent + li = markdown.etree.SubElement(lst, 'li') + self.parser.parseBlocks(li, [item]) + self.parser.state.reset() + + def get_items(self, block): + """ Break a block into list items. """ + items = [] + for line in block.split('\n'): + m = self.CHILD_RE.match(line) + if m: + # This is a new item. Append + items.append(m.group(3)) + elif self.INDENT_RE.match(line): + # This is an indented (possibly nested) item. + if items[-1].startswith(' '*markdown.TAB_LENGTH): + # Previous item was indented. Append to that item. + items[-1] = '%s\n%s' % (items[-1], line) + else: + items.append(line) + else: + # This is another line of previous item. Append to that item. + items[-1] = '%s\n%s' % (items[-1], line) + return items + + +class UListProcessor(OListProcessor): + """ Process unordered list blocks. """ + + TAG = 'ul' + RE = re.compile(r'^[ ]{0,3}[*+-][ ](.*)') + + +class HashHeaderProcessor(BlockProcessor): + """ Process Hash Headers. """ + + # Detect a header at start of any line in block + RE = re.compile(r'(^|\n)(?P<level>#{1,6})(?P<header>.*?)#*(\n|$)') + + def test(self, parent, block): + return bool(self.RE.search(block)) + + def run(self, parent, blocks): + block = blocks.pop(0) + m = self.RE.search(block) + if m: + before = block[:m.start()] # All lines before header + after = block[m.end():] # All lines after header + if before: + # As the header was not the first line of the block and the + # lines before the header must be parsed first, + # recursively parse this lines as a block. + self.parser.parseBlocks(parent, [before]) + # Create header using named groups from RE + h = markdown.etree.SubElement(parent, 'h%d' % len(m.group('level'))) + h.text = m.group('header').strip() + if after: + # Insert remaining lines as first block for future parsing. + blocks.insert(0, after) + else: + # This should never happen, but just in case... + message(CRITICAL, "We've got a problem header!") + + +class SetextHeaderProcessor(BlockProcessor): + """ Process Setext-style Headers. """ + + # Detect Setext-style header. Must be first 2 lines of block. + RE = re.compile(r'^.*?\n[=-]{3,}', re.MULTILINE) + + def test(self, parent, block): + return bool(self.RE.match(block)) + + def run(self, parent, blocks): + lines = blocks.pop(0).split('\n') + # Determine level. ``=`` is 1 and ``-`` is 2. + if lines[1].startswith('='): + level = 1 + else: + level = 2 + h = markdown.etree.SubElement(parent, 'h%d' % level) + h.text = lines[0].strip() + if len(lines) > 2: + # Block contains additional lines. Add to master blocks for later. + blocks.insert(0, '\n'.join(lines[2:])) + + +class HRProcessor(BlockProcessor): + """ Process Horizontal Rules. """ + + RE = r'[ ]{0,3}(?P<ch>[*_-])[ ]?((?P=ch)[ ]?){2,}[ ]*' + # Detect hr on any line of a block. + SEARCH_RE = re.compile(r'(^|\n)%s(\n|$)' % RE) + # Match a hr on a single line of text. + MATCH_RE = re.compile(r'^%s$' % RE) + + def test(self, parent, block): + return bool(self.SEARCH_RE.search(block)) + + def run(self, parent, blocks): + lines = blocks.pop(0).split('\n') + prelines = [] + # Check for lines in block before hr. + for line in lines: + m = self.MATCH_RE.match(line) + if m: + break + else: + prelines.append(line) + if len(prelines): + # Recursively parse lines before hr so they get parsed first. + self.parser.parseBlocks(parent, ['\n'.join(prelines)]) + # create hr + hr = markdown.etree.SubElement(parent, 'hr') + # check for lines in block after hr. + lines = lines[len(prelines)+1:] + if len(lines): + # Add lines after hr to master blocks for later parsing. + blocks.insert(0, '\n'.join(lines)) + + +class EmptyBlockProcessor(BlockProcessor): + """ Process blocks and start with an empty line. """ + + # Detect a block that only contains whitespace + # or only whitespace on the first line. + RE = re.compile(r'^\s*\n') + + def test(self, parent, block): + return bool(self.RE.match(block)) + + def run(self, parent, blocks): + block = blocks.pop(0) + m = self.RE.match(block) + if m: + # Add remaining line to master blocks for later. + blocks.insert(0, block[m.end():]) + sibling = self.lastChild(parent) + if sibling and sibling.tag == 'pre' and sibling[0] and \ + sibling[0].tag == 'code': + # Last block is a codeblock. Append to preserve whitespace. + sibling[0].text = markdown.AtomicString('%s/n/n/n' % sibling[0].text ) + + +class ParagraphProcessor(BlockProcessor): + """ Process Paragraph blocks. """ + + def test(self, parent, block): + return True + + def run(self, parent, blocks): + block = blocks.pop(0) + if block.strip(): + # Not a blank block. Add to parent, otherwise throw it away. + if self.parser.state.isstate('list'): + # The parent is a tight-list. Append to parent.text + if parent.text: + parent.text = '%s\n%s' % (parent.text, block) + else: + parent.text = block.lstrip() + else: + # Create a regular paragraph + p = markdown.etree.SubElement(parent, 'p') + p.text = block.lstrip() diff --git a/src/contrib/markdown/commandline.py b/src/contrib/markdown/commandline.py new file mode 100644 index 0000000..1eedc6d --- /dev/null +++ b/src/contrib/markdown/commandline.py @@ -0,0 +1,96 @@ +""" +COMMAND-LINE SPECIFIC STUFF +============================================================================= + +The rest of the code is specifically for handling the case where Python +Markdown is called from the command line. +""" + +import markdown +import sys +import logging +from logging import DEBUG, INFO, WARN, ERROR, CRITICAL + +EXECUTABLE_NAME_FOR_USAGE = "python markdown.py" +""" The name used in the usage statement displayed for python versions < 2.3. +(With python 2.3 and higher the usage statement is generated by optparse +and uses the actual name of the executable called.) """ + +OPTPARSE_WARNING = """ +Python 2.3 or higher required for advanced command line options. +For lower versions of Python use: + + %s INPUT_FILE > OUTPUT_FILE + +""" % EXECUTABLE_NAME_FOR_USAGE + +def parse_options(): + """ + Define and parse `optparse` options for command-line usage. + """ + + try: + optparse = __import__("optparse") + except: + if len(sys.argv) == 2: + return {'input': sys.argv[1], + 'output': None, + 'safe': False, + 'extensions': [], + 'encoding': None }, CRITICAL + else: + print OPTPARSE_WARNING + return None, None + + parser = optparse.OptionParser(usage="%prog INPUTFILE [options]") + parser.add_option("-f", "--file", dest="filename", default=sys.stdout, + help="write output to OUTPUT_FILE", + metavar="OUTPUT_FILE") + parser.add_option("-e", "--encoding", dest="encoding", + help="encoding for input and output files",) + parser.add_option("-q", "--quiet", default = CRITICAL, + action="store_const", const=CRITICAL+10, dest="verbose", + help="suppress all messages") + parser.add_option("-v", "--verbose", + action="store_const", const=INFO, dest="verbose", + help="print info messages") + parser.add_option("-s", "--safe", dest="safe", default=False, + metavar="SAFE_MODE", + help="safe mode ('replace', 'remove' or 'escape' user's HTML tag)") + parser.add_option("-o", "--output_format", dest="output_format", + default='xhtml1', metavar="OUTPUT_FORMAT", + help="Format of output. One of 'xhtml1' (default) or 'html4'.") + parser.add_option("--noisy", + action="store_const", const=DEBUG, dest="verbose", + help="print debug messages") + parser.add_option("-x", "--extension", action="append", dest="extensions", + help = "load extension EXTENSION", metavar="EXTENSION") + + (options, args) = parser.parse_args() + + if not len(args) == 1: + parser.print_help() + return None, None + else: + input_file = args[0] + + if not options.extensions: + options.extensions = [] + + return {'input': input_file, + 'output': options.filename, + 'safe_mode': options.safe, + 'extensions': options.extensions, + 'encoding': options.encoding, + 'output_format': options.output_format}, options.verbose + +def run(): + """Run Markdown from the command line.""" + + # Parse options and adjust logging level if necessary + options, logging_level = parse_options() + if not options: sys.exit(0) + if logging_level: logging.getLogger('MARKDOWN').setLevel(logging_level) + + # Run + markdown.markdownFromFile(**options) diff --git a/src/contrib/markdown/etree_loader.py b/src/contrib/markdown/etree_loader.py new file mode 100644 index 0000000..e2599b2 --- /dev/null +++ b/src/contrib/markdown/etree_loader.py @@ -0,0 +1,33 @@ + +from markdown import message, CRITICAL +import sys + +## Import +def importETree(): + """Import the best implementation of ElementTree, return a module object.""" + etree_in_c = None + try: # Is it Python 2.5+ with C implemenation of ElementTree installed? + import xml.etree.cElementTree as etree_in_c + except ImportError: + try: # Is it Python 2.5+ with Python implementation of ElementTree? + import xml.etree.ElementTree as etree + except ImportError: + try: # An earlier version of Python with cElementTree installed? + import cElementTree as etree_in_c + except ImportError: + try: # An earlier version of Python with Python ElementTree? + import elementtree.ElementTree as etree + except ImportError: + message(CRITICAL, "Failed to import ElementTree") + sys.exit(1) + if etree_in_c and etree_in_c.VERSION < "1.0": + message(CRITICAL, "For cElementTree version 1.0 or higher is required.") + sys.exit(1) + elif etree_in_c : + return etree_in_c + elif etree.VERSION < "1.1": + message(CRITICAL, "For ElementTree version 1.1 or higher is required") + sys.exit(1) + else : + return etree + diff --git a/src/contrib/markdown/extensions/__init__.py b/src/contrib/markdown/extensions/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/src/contrib/markdown/extensions/__init__.py diff --git a/src/contrib/markdown/extensions/abbr.py b/src/contrib/markdown/extensions/abbr.py new file mode 100644 index 0000000..783220e --- /dev/null +++ b/src/contrib/markdown/extensions/abbr.py @@ -0,0 +1,95 @@ +''' +Abbreviation Extension for Python-Markdown +========================================== + +This extension adds abbreviation handling to Python-Markdown. + +Simple Usage: + + >>> import markdown + >>> text = """ + ... Some text with an ABBR and a REF. Ignore REFERENCE and ref. + ... + ... *[ABBR]: Abbreviation + ... *[REF]: Abbreviation Reference + ... """ + >>> markdown.markdown(text, ['abbr']) + u'<p>Some text with an <abbr title="Abbreviation">ABBR</abbr> and a <abbr title="Abbreviation Reference">REF</abbr>. Ignore REFERENCE and ref.</p>' + +Copyright 2007-2008 +* [Waylan Limberg](http://achinghead.com/) +* [Seemant Kulleen](http://www.kulleen.org/) + + +''' + +import markdown, re +from markdown import etree + +# Global Vars +ABBR_REF_RE = re.compile(r'[*]\[(?P<abbr>[^\]]*)\][ ]?:\s*(?P<title>.*)') + +class AbbrExtension(markdown.Extension): + """ Abbreviation Extension for Python-Markdown. """ + + def extendMarkdown(self, md, md_globals): + """ Insert AbbrPreprocessor before ReferencePreprocessor. """ + md.preprocessors.add('abbr', AbbrPreprocessor(md), '<reference') + + +class AbbrPreprocessor(markdown.preprocessors.Preprocessor): + """ Abbreviation Preprocessor - parse text for abbr references. """ + + def run(self, lines): + ''' + Find and remove all Abbreviation references from the text. + Each reference is set as a new AbbrPattern in the markdown instance. + + ''' + new_text = [] + for line in lines: + m = ABBR_REF_RE.match(line) + if m: + abbr = m.group('abbr').strip() + title = m.group('title').strip() + self.markdown.inlinePatterns['abbr-%s'%abbr] = \ + AbbrPattern(self._generate_pattern(abbr), title) + else: + new_text.append(line) + return new_text + + def _generate_pattern(self, text): + ''' + Given a string, returns an regex pattern to match that string. + + 'HTML' -> r'(?P<abbr>[H][T][M][L])' + + Note: we force each char as a literal match (in brackets) as we don't + know what they will be beforehand. + + ''' + chars = list(text) + for i in range(len(chars)): + chars[i] = r'[%s]' % chars[i] + return r'(?P<abbr>\b%s\b)' % (r''.join(chars)) + + +class AbbrPattern(markdown.inlinepatterns.Pattern): + """ Abbreviation inline pattern. """ + + def __init__(self, pattern, title): + markdown.inlinepatterns.Pattern.__init__(self, pattern) + self.title = title + + def handleMatch(self, m): + abbr = etree.Element('abbr') + abbr.text = m.group('abbr') + abbr.set('title', self.title) + return abbr + +def makeExtension(configs=None): + return AbbrExtension(configs=configs) + +if __name__ == "__main__": + import doctest + doctest.testmod() diff --git a/src/contrib/markdown/extensions/codehilite.py b/src/contrib/markdown/extensions/codehilite.py new file mode 100644 index 0000000..c5d496b --- /dev/null +++ b/src/contrib/markdown/extensions/codehilite.py @@ -0,0 +1,224 @@ +#!/usr/bin/python + +""" +CodeHilite Extension for Python-Markdown +======================================== + +Adds code/syntax highlighting to standard Python-Markdown code blocks. + +Copyright 2006-2008 [Waylan Limberg](http://achinghead.com/). + +Project website: <http://www.freewisdom.org/project/python-markdown/CodeHilite> +Contact: markdown@freewisdom.org + +License: BSD (see ../docs/LICENSE for details) + +Dependencies: +* [Python 2.3+](http://python.org/) +* [Markdown 2.0+](http://www.freewisdom.org/projects/python-markdown/) +* [Pygments](http://pygments.org/) + +""" + +import markdown + +# --------------- CONSTANTS YOU MIGHT WANT TO MODIFY ----------------- + +try: + TAB_LENGTH = markdown.TAB_LENGTH +except AttributeError: + TAB_LENGTH = 4 + + +# ------------------ The Main CodeHilite Class ---------------------- +class CodeHilite: + """ + Determine language of source code, and pass it into the pygments hilighter. + + Basic Usage: + >>> code = CodeHilite(src = 'some text') + >>> html = code.hilite() + + * src: Source string or any object with a .readline attribute. + + * linenos: (Boolen) Turn line numbering 'on' or 'off' (off by default). + + * css_class: Set class name of wrapper div ('codehilite' by default). + + Low Level Usage: + >>> code = CodeHilite() + >>> code.src = 'some text' # String or anything with a .readline attr. + >>> code.linenos = True # True or False; Turns line numbering on or of. + >>> html = code.hilite() + + """ + + def __init__(self, src=None, linenos=False, css_class="codehilite"): + self.src = src + self.lang = None + self.linenos = linenos + self.css_class = css_class + + def hilite(self): + """ + Pass code to the [Pygments](http://pygments.pocoo.org/) highliter with + optional line numbers. The output should then be styled with css to + your liking. No styles are applied by default - only styling hooks + (i.e.: <span class="k">). + + returns : A string of html. + + """ + + self.src = self.src.strip('\n') + + self._getLang() + + try: + from pygments import highlight + from pygments.lexers import get_lexer_by_name, guess_lexer, \ + TextLexer + from pygments.formatters import HtmlFormatter + except ImportError: + # just escape and pass through + txt = self._escape(self.src) + if self.linenos: + txt = self._number(txt) + else : + txt = '<div class="%s"><pre>%s</pre></div>\n'% \ + (self.css_class, txt) + return txt + else: + try: + lexer = get_lexer_by_name(self.lang) + except ValueError: + try: + lexer = guess_lexer(self.src) + except ValueError: + lexer = TextLexer() + formatter = HtmlFormatter(linenos=self.linenos, + cssclass=self.css_class) + return highlight(self.src, lexer, formatter) + + def _escape(self, txt): + """ basic html escaping """ + txt = txt.replace('&', '&') + txt = txt.replace('<', '<') + txt = txt.replace('>', '>') + txt = txt.replace('"', '"') + return txt + + def _number(self, txt): + """ Use <ol> for line numbering """ + # Fix Whitespace + txt = txt.replace('\t', ' '*TAB_LENGTH) + txt = txt.replace(" "*4, " ") + txt = txt.replace(" "*3, " ") + txt = txt.replace(" "*2, " ") + + # Add line numbers + lines = txt.splitlines() + txt = '<div class="codehilite"><pre><ol>\n' + for line in lines: + txt += '\t<li>%s</li>\n'% line + txt += '</ol></pre></div>\n' + return txt + + + def _getLang(self): + """ + Determines language of a code block from shebang lines and whether said + line should be removed or left in place. If the sheband line contains a + path (even a single /) then it is assumed to be a real shebang lines and + left alone. However, if no path is given (e.i.: #!python or :::python) + then it is assumed to be a mock shebang for language identifitation of a + code fragment and removed from the code block prior to processing for + code highlighting. When a mock shebang (e.i: #!python) is found, line + numbering is turned on. When colons are found in place of a shebang + (e.i.: :::python), line numbering is left in the current state - off + by default. + + """ + + import re + + #split text into lines + lines = self.src.split("\n") + #pull first line to examine + fl = lines.pop(0) + + c = re.compile(r''' + (?:(?:::+)|(?P<shebang>[#]!)) # Shebang or 2 or more colons. + (?P<path>(?:/\w+)*[/ ])? # Zero or 1 path + (?P<lang>[\w+-]*) # The language + ''', re.VERBOSE) + # search first line for shebang + m = c.search(fl) + if m: + # we have a match + try: + self.lang = m.group('lang').lower() + except IndexError: + self.lang = None + if m.group('path'): + # path exists - restore first line + lines.insert(0, fl) + if m.group('shebang'): + # shebang exists - use line numbers + self.linenos = True + else: + # No match + lines.insert(0, fl) + + self.src = "\n".join(lines).strip("\n") + + + +# ------------------ The Markdown Extension ------------------------------- +class HiliteTreeprocessor(markdown.treeprocessors.Treeprocessor): + """ Hilight source code in code blocks. """ + + def run(self, root): + """ Find code blocks and store in htmlStash. """ + blocks = root.getiterator('pre') + for block in blocks: + children = block.getchildren() + if len(children) == 1 and children[0].tag == 'code': + code = CodeHilite(children[0].text, + linenos=self.config['force_linenos'][0], + css_class=self.config['css_class'][0]) + placeholder = self.markdown.htmlStash.store(code.hilite(), + safe=True) + # Clear codeblock in etree instance + block.clear() + # Change to p element which will later + # be removed when inserting raw html + block.tag = 'p' + block.text = placeholder + + +class CodeHiliteExtension(markdown.Extension): + """ Add source code hilighting to markdown codeblocks. """ + + def __init__(self, configs): + # define default configs + self.config = { + 'force_linenos' : [False, "Force line numbers - Default: False"], + 'css_class' : ["codehilite", + "Set class name for wrapper <div> - Default: codehilite"], + } + + # Override defaults with user settings + for key, value in configs: + self.setConfig(key, value) + + def extendMarkdown(self, md, md_globals): + """ Add HilitePostprocessor to Markdown instance. """ + hiliter = HiliteTreeprocessor(md) + hiliter.config = self.config + md.treeprocessors.add("hilite", hiliter, "_begin") + + +def makeExtension(configs={}): + return CodeHiliteExtension(configs=configs) + diff --git a/src/contrib/markdown/extensions/def_list.py b/src/contrib/markdown/extensions/def_list.py new file mode 100644 index 0000000..73a1c85 --- /dev/null +++ b/src/contrib/markdown/extensions/def_list.py @@ -0,0 +1,104 @@ +#!/usr/bin/env Python +""" +Definition List Extension for Python-Markdown +============================================= + +Added parsing of Definition Lists to Python-Markdown. + +A simple example: + + Apple + : Pomaceous fruit of plants of the genus Malus in + the family Rosaceae. + : An american computer company. + + Orange + : The fruit of an evergreen tree of the genus Citrus. + +Copyright 2008 - [Waylan Limberg](http://achinghead.com) + +""" + +import markdown, re +from markdown import etree + + +class DefListProcessor(markdown.blockprocessors.BlockProcessor): + """ Process Definition Lists. """ + + RE = re.compile(r'(^|\n)[ ]{0,3}:[ ]{1,3}(.*?)(\n|$)') + + def test(self, parent, block): + return bool(self.RE.search(block)) + + def run(self, parent, blocks): + block = blocks.pop(0) + m = self.RE.search(block) + terms = [l.strip() for l in block[:m.start()].split('\n') if l.strip()] + d, theRest = self.detab(block[m.end():]) + if d: + d = '%s\n%s' % (m.group(2), d) + else: + d = m.group(2) + #import ipdb; ipdb.set_trace() + sibling = self.lastChild(parent) + if not terms and sibling.tag == 'p': + # The previous paragraph contains the terms + state = 'looselist' + terms = sibling.text.split('\n') + parent.remove(sibling) + # Aquire new sibling + sibling = self.lastChild(parent) + else: + state = 'list' + + if sibling and sibling.tag == 'dl': + # This is another item on an existing list + dl = sibling + if len(dl) and dl[-1].tag == 'dd' and len(dl[-1]): + state = 'looselist' + else: + # This is a new list + dl = etree.SubElement(parent, 'dl') + # Add terms + for term in terms: + dt = etree.SubElement(dl, 'dt') + dt.text = term + # Add definition + self.parser.state.set(state) + dd = etree.SubElement(dl, 'dd') + self.parser.parseBlocks(dd, [d]) + self.parser.state.reset() + + if theRest: + blocks.insert(0, theRest) + +class DefListIndentProcessor(markdown.blockprocessors.ListIndentProcessor): + """ Process indented children of definition list items. """ + + ITEM_TYPES = ['dd'] + LIST_TYPES = ['dl'] + + def create_item(parent, block): + """ Create a new dd and parse the block with it as the parent. """ + dd = markdown.etree.SubElement(parent, 'dd') + self.parser.parseBlocks(dd, [block]) + + + +class DefListExtension(markdown.Extension): + """ Add definition lists to Markdown. """ + + def extendMarkdown(self, md, md_globals): + """ Add an instance of DefListProcessor to BlockParser. """ + md.parser.blockprocessors.add('defindent', + DefListIndentProcessor(md.parser), + '>indent') + md.parser.blockprocessors.add('deflist', + DefListProcessor(md.parser), + '>ulist') + + +def makeExtension(configs={}): + return DefListExtension(configs=configs) + diff --git a/src/contrib/markdown/extensions/extra.py b/src/contrib/markdown/extensions/extra.py new file mode 100644 index 0000000..4a2ffbf --- /dev/null +++ b/src/contrib/markdown/extensions/extra.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +""" +Python-Markdown Extra Extension +=============================== + +A compilation of various Python-Markdown extensions that imitates +[PHP Markdown Extra](http://michelf.com/projects/php-markdown/extra/). + +Note that each of the individual extensions still need to be available +on your PYTHONPATH. This extension simply wraps them all up as a +convenience so that only one extension needs to be listed when +initiating Markdown. See the documentation for each individual +extension for specifics about that extension. + +In the event that one or more of the supported extensions are not +available for import, Markdown will issue a warning and simply continue +without that extension. + +There may be additional extensions that are distributed with +Python-Markdown that are not included here in Extra. Those extensions +are not part of PHP Markdown Extra, and therefore, not part of +Python-Markdown Extra. If you really would like Extra to include +additional extensions, we suggest creating your own clone of Extra +under a differant name. You could also edit the `extensions` global +variable defined below, but be aware that such changes may be lost +when you upgrade to any future version of Python-Markdown. + +""" + +import markdown + +extensions = ['fenced_code', + 'footnotes', + 'headerid', + 'def_list', + 'tables', + 'abbr', + ] + + +class ExtraExtension(markdown.Extension): + """ Add various extensions to Markdown class.""" + + def extendMarkdown(self, md, md_globals): + """ Register extension instances. """ + md.registerExtensions(extensions, self.config) + +def makeExtension(configs={}): + return ExtraExtension(configs=dict(configs)) diff --git a/src/contrib/markdown/extensions/fenced_code.py b/src/contrib/markdown/extensions/fenced_code.py new file mode 100644 index 0000000..307b1dc --- /dev/null +++ b/src/contrib/markdown/extensions/fenced_code.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python + +""" +Fenced Code Extension for Python Markdown +========================================= + +This extension adds Fenced Code Blocks to Python-Markdown. + + >>> import markdown + >>> text = ''' + ... A paragraph before a fenced code block: + ... + ... ~~~ + ... Fenced code block + ... ~~~ + ... ''' + >>> html = markdown.markdown(text, extensions=['fenced_code']) + >>> html + u'<p>A paragraph before a fenced code block:</p>\\n<pre><code>Fenced code block\\n</code></pre>' + +Works with safe_mode also (we check this because we are using the HtmlStash): + + >>> markdown.markdown(text, extensions=['fenced_code'], safe_mode='replace') + u'<p>A paragraph before a fenced code block:</p>\\n<pre><code>Fenced code block\\n</code></pre>' + +Include tilde's in a code block and wrap with blank lines: + + >>> text = ''' + ... ~~~~~~~~ + ... + ... ~~~~ + ... + ... ~~~~~~~~''' + >>> markdown.markdown(text, extensions=['fenced_code']) + u'<pre><code>\\n~~~~\\n\\n</code></pre>' + +Multiple blocks and language tags: + + >>> text = ''' + ... ~~~~{.python} + ... block one + ... ~~~~ + ... + ... ~~~~.html + ... <p>block two</p> + ... ~~~~''' + >>> markdown.markdown(text, extensions=['fenced_code']) + u'<pre><code class="python">block one\\n</code></pre>\\n\\n<pre><code class="html"><p>block two</p>\\n</code></pre>' + +Copyright 2007-2008 [Waylan Limberg](http://achinghead.com/). + +Project website: <http://www.freewisdom.org/project/python-markdown/Fenced__Code__Blocks> +Contact: markdown@freewisdom.org + +License: BSD (see ../docs/LICENSE for details) + +Dependencies: +* [Python 2.3+](http://python.org) +* [Markdown 2.0+](http://www.freewisdom.org/projects/python-markdown/) + +""" + +import markdown, re + +# Global vars +FENCED_BLOCK_RE = re.compile( \ + r'(?P<fence>^~{3,})[ ]*(\{?\.(?P<lang>[a-zA-Z0-9_-]*)\}?)?[ ]*\n(?P<code>.*?)(?P=fence)[ ]*$', + re.MULTILINE|re.DOTALL + ) +CODE_WRAP = '<pre><code%s>%s</code></pre>' +LANG_TAG = ' class="%s"' + + +class FencedCodeExtension(markdown.Extension): + + def extendMarkdown(self, md, md_globals): + """ Add FencedBlockPreprocessor to the Markdown instance. """ + + md.preprocessors.add('fenced_code_block', + FencedBlockPreprocessor(md), + "_begin") + + +class FencedBlockPreprocessor(markdown.preprocessors.Preprocessor): + + def run(self, lines): + """ Match and store Fenced Code Blocks in the HtmlStash. """ + text = "\n".join(lines) + while 1: + m = FENCED_BLOCK_RE.search(text) + if m: + lang = '' + if m.group('lang'): + lang = LANG_TAG % m.group('lang') + code = CODE_WRAP % (lang, self._escape(m.group('code'))) + placeholder = self.markdown.htmlStash.store(code, safe=True) + text = '%s\n%s\n%s'% (text[:m.start()], placeholder, text[m.end():]) + else: + break + return text.split("\n") + + def _escape(self, txt): + """ basic html escaping """ + txt = txt.replace('&', '&') + txt = txt.replace('<', '<') + txt = txt.replace('>', '>') + txt = txt.replace('"', '"') + return txt + + +def makeExtension(configs=None): + return FencedCodeExtension() + + +if __name__ == "__main__": + import doctest + doctest.testmod() diff --git a/src/contrib/markdown/extensions/footnotes.py b/src/contrib/markdown/extensions/footnotes.py new file mode 100644 index 0000000..6dacab7 --- /dev/null +++ b/src/contrib/markdown/extensions/footnotes.py @@ -0,0 +1,293 @@ +""" +========================= FOOTNOTES ================================= + +This section adds footnote handling to markdown. It can be used as +an example for extending python-markdown with relatively complex +functionality. While in this case the extension is included inside +the module itself, it could just as easily be added from outside the +module. Not that all markdown classes above are ignorant about +footnotes. All footnote functionality is provided separately and +then added to the markdown instance at the run time. + +Footnote functionality is attached by calling extendMarkdown() +method of FootnoteExtension. The method also registers the +extension to allow it's state to be reset by a call to reset() +method. + +Example: + Footnotes[^1] have a label[^label] and a definition[^!DEF]. + + [^1]: This is a footnote + [^label]: A footnote on "label" + [^!DEF]: The footnote for definition + +""" + +import re, markdown +from markdown import etree + +FN_BACKLINK_TEXT = "zz1337820767766393qq" +NBSP_PLACEHOLDER = "qq3936677670287331zz" +DEF_RE = re.compile(r'(\ ?\ ?\ ?)\[\^([^\]]*)\]:\s*(.*)') +TABBED_RE = re.compile(r'((\t)|( ))(.*)') + +class FootnoteExtension(markdown.Extension): + """ Footnote Extension. """ + + def __init__ (self, configs): + """ Setup configs. """ + self.config = {'PLACE_MARKER': + ["///Footnotes Go Here///", + "The text string that marks where the footnotes go"]} + + for key, value in configs: + self.config[key][0] = value + + self.reset() + + def extendMarkdown(self, md, md_globals): + """ Add pieces to Markdown. """ + md.registerExtension(self) + self.parser = md.parser + # Insert a preprocessor before ReferencePreprocessor + md.preprocessors.add("footnote", FootnotePreprocessor(self), + "<reference") + # Insert an inline pattern before ImageReferencePattern + FOOTNOTE_RE = r'\[\^([^\]]*)\]' # blah blah [^1] blah + md.inlinePatterns.add("footnote", FootnotePattern(FOOTNOTE_RE, self), + "<reference") + # Insert a tree-processor that would actually add the footnote div + # This must be before the inline treeprocessor so inline patterns + # run on the contents of the div. + md.treeprocessors.add("footnote", FootnoteTreeprocessor(self), + "<inline") + # Insert a postprocessor after amp_substitute oricessor + md.postprocessors.add("footnote", FootnotePostprocessor(self), + ">amp_substitute") + + def reset(self): + """ Clear the footnotes on reset. """ + self.footnotes = markdown.odict.OrderedDict() + + def findFootnotesPlaceholder(self, root): + """ Return ElementTree Element that contains Footnote placeholder. """ + def finder(element): + for child in element: + if child.text: + if child.text.find(self.getConfig("PLACE_MARKER")) > -1: + return child, True + if child.tail: + if child.tail.find(self.getConfig("PLACE_MARKER")) > -1: + return (child, element), False + finder(child) + return None + + res = finder(root) + return res + + def setFootnote(self, id, text): + """ Store a footnote for later retrieval. """ + self.footnotes[id] = text + + def makeFootnoteId(self, id): + """ Return footnote link id. """ + return 'fn:%s' % id + + def makeFootnoteRefId(self, id): + """ Return footnote back-link id. """ + return 'fnref:%s' % id + + def makeFootnotesDiv(self, root): + """ Return div of footnotes as et Element. """ + + if not self.footnotes.keys(): + return None + + div = etree.Element("div") + div.set('class', 'footnote') + hr = etree.SubElement(div, "hr") + ol = etree.SubElement(div, "ol") + + for id in self.footnotes.keys(): + li = etree.SubElement(ol, "li") + li.set("id", self.makeFootnoteId(id)) + self.parser.parseChunk(li, self.footnotes[id]) + backlink = etree.Element("a") + backlink.set("href", "#" + self.makeFootnoteRefId(id)) + backlink.set("rev", "footnote") + backlink.set("title", "Jump back to footnote %d in the text" % \ + (self.footnotes.index(id)+1)) + backlink.text = FN_BACKLINK_TEXT + + if li.getchildren(): + node = li[-1] + if node.tag == "p": + node.text = node.text + NBSP_PLACEHOLDER + node.append(backlink) + else: + p = etree.SubElement(li, "p") + p.append(backlink) + return div + + +class FootnotePreprocessor(markdown.preprocessors.Preprocessor): + """ Find all footnote references and store for later use. """ + + def __init__ (self, footnotes): + self.footnotes = footnotes + + def run(self, lines): + lines = self._handleFootnoteDefinitions(lines) + text = "\n".join(lines) + return text.split("\n") + + def _handleFootnoteDefinitions(self, lines): + """ + Recursively find all footnote definitions in lines. + + Keywords: + + * lines: A list of lines of text + + Return: A list of lines with footnote definitions removed. + + """ + i, id, footnote = self._findFootnoteDefinition(lines) + + if id : + plain = lines[:i] + detabbed, theRest = self.detectTabbed(lines[i+1:]) + self.footnotes.setFootnote(id, + footnote + "\n" + + "\n".join(detabbed)) + more_plain = self._handleFootnoteDefinitions(theRest) + return plain + [""] + more_plain + else : + return lines + + def _findFootnoteDefinition(self, lines): + """ + Find the parts of a footnote definition. + + Keywords: + + * lines: A list of lines of text. + + Return: A three item tuple containing the index of the first line of a + footnote definition, the id of the definition and the body of the + definition. + + """ + counter = 0 + for line in lines: + m = DEF_RE.match(line) + if m: + return counter, m.group(2), m.group(3) + counter += 1 + return counter, None, None + + def detectTabbed(self, lines): + """ Find indented text and remove indent before further proccesing. + + Keyword arguments: + + * lines: an array of strings + + Returns: a list of post processed items and the unused + remainder of the original list + + """ + items = [] + item = -1 + i = 0 # to keep track of where we are + + def detab(line): + match = TABBED_RE.match(line) + if match: + return match.group(4) + + for line in lines: + if line.strip(): # Non-blank line + line = detab(line) + if line: + items.append(line) + i += 1 + continue + else: + return items, lines[i:] + + else: # Blank line: _maybe_ we are done. + i += 1 # advance + + # Find the next non-blank line + for j in range(i, len(lines)): + if lines[j].strip(): + next_line = lines[j]; break + else: + break # There is no more text; we are done. + + # Check if the next non-blank line is tabbed + if detab(next_line): # Yes, more work to do. + items.append("") + continue + else: + break # No, we are done. + else: + i += 1 + + return items, lines[i:] + + +class FootnotePattern(markdown.inlinepatterns.Pattern): + """ InlinePattern for footnote markers in a document's body text. """ + + def __init__(self, pattern, footnotes): + markdown.inlinepatterns.Pattern.__init__(self, pattern) + self.footnotes = footnotes + + def handleMatch(self, m): + sup = etree.Element("sup") + a = etree.SubElement(sup, "a") + id = m.group(2) + sup.set('id', self.footnotes.makeFootnoteRefId(id)) + a.set('href', '#' + self.footnotes.makeFootnoteId(id)) + a.set('rel', 'footnote') + a.text = str(self.footnotes.footnotes.index(id) + 1) + return sup + + +class FootnoteTreeprocessor(markdown.treeprocessors.Treeprocessor): + """ Build and append footnote div to end of document. """ + + def __init__ (self, footnotes): + self.footnotes = footnotes + + def run(self, root): + footnotesDiv = self.footnotes.makeFootnotesDiv(root) + if footnotesDiv: + result = self.footnotes.findFootnotesPlaceholder(root) + if result: + node, isText = result + if isText: + node.text = None + node.getchildren().insert(0, footnotesDiv) + else: + child, element = node + ind = element.getchildren().find(child) + element.getchildren().insert(ind + 1, footnotesDiv) + child.tail = None + fnPlaceholder.parent.replaceChild(fnPlaceholder, footnotesDiv) + else: + root.append(footnotesDiv) + +class FootnotePostprocessor(markdown.postprocessors.Postprocessor): + """ Replace placeholders with html entities. """ + + def run(self, text): + text = text.replace(FN_BACKLINK_TEXT, "↩") + return text.replace(NBSP_PLACEHOLDER, " ") + +def makeExtension(configs=[]): + """ Return an instance of the FootnoteExtension """ + return FootnoteExtension(configs=configs) + diff --git a/src/contrib/markdown/extensions/headerid.py b/src/contrib/markdown/extensions/headerid.py new file mode 100644 index 0000000..f70a7a9 --- /dev/null +++ b/src/contrib/markdown/extensions/headerid.py @@ -0,0 +1,195 @@ +#!/usr/bin/python + +""" +HeaderID Extension for Python-Markdown +====================================== + +Adds ability to set HTML IDs for headers. + +Basic usage: + + >>> import markdown + >>> text = "# Some Header # {#some_id}" + >>> md = markdown.markdown(text, ['headerid']) + >>> md + u'<h1 id="some_id">Some Header</h1>' + +All header IDs are unique: + + >>> text = ''' + ... #Header + ... #Another Header {#header} + ... #Third Header {#header}''' + >>> md = markdown.markdown(text, ['headerid']) + >>> md + u'<h1 id="header">Header</h1>\\n<h1 id="header_1">Another Header</h1>\\n<h1 id="header_2">Third Header</h1>' + +To fit within a html template's hierarchy, set the header base level: + + >>> text = ''' + ... #Some Header + ... ## Next Level''' + >>> md = markdown.markdown(text, ['headerid(level=3)']) + >>> md + u'<h3 id="some_header">Some Header</h3>\\n<h4 id="next_level">Next Level</h4>' + +Turn off auto generated IDs: + + >>> text = ''' + ... # Some Header + ... # Header with ID # { #foo }''' + >>> md = markdown.markdown(text, ['headerid(forceid=False)']) + >>> md + u'<h1>Some Header</h1>\\n<h1 id="foo">Header with ID</h1>' + +Use with MetaData extension: + + >>> text = '''header_level: 2 + ... header_forceid: Off + ... + ... # A Header''' + >>> md = markdown.markdown(text, ['headerid', 'meta']) + >>> md + u'<h2>A Header</h2>' + +Copyright 2007-2008 [Waylan Limberg](http://achinghead.com/). + +Project website: <http://www.freewisdom.org/project/python-markdown/HeaderId> +Contact: markdown@freewisdom.org + +License: BSD (see ../docs/LICENSE for details) + +Dependencies: +* [Python 2.3+](http://python.org) +* [Markdown 2.0+](http://www.freewisdom.org/projects/python-markdown/) + +""" + +import markdown +from markdown import etree +import re +from string import ascii_lowercase, digits, punctuation + +ID_CHARS = ascii_lowercase + digits + '-_' +IDCOUNT_RE = re.compile(r'^(.*)_([0-9]+)$') + + +class HeaderIdProcessor(markdown.blockprocessors.BlockProcessor): + """ Replacement BlockProcessor for Header IDs. """ + + # Detect a header at start of any line in block + RE = re.compile(r"""(^|\n) + (?P<level>\#{1,6}) # group('level') = string of hashes + (?P<header>.*?) # group('header') = Header text + \#* # optional closing hashes + (?:[ \t]*\{[ \t]*\#(?P<id>[-_:a-zA-Z0-9]+)[ \t]*\})? + (\n|$) # ^^ group('id') = id attribute + """, + re.VERBOSE) + + IDs = [] + + def test(self, parent, block): + return bool(self.RE.search(block)) + + def run(self, parent, blocks): + block = blocks.pop(0) + m = self.RE.search(block) + if m: + before = block[:m.start()] # All lines before header + after = block[m.end():] # All lines after header + if before: + # As the header was not the first line of the block and the + # lines before the header must be parsed first, + # recursively parse this lines as a block. + self.parser.parseBlocks(parent, [before]) + # Create header using named groups from RE + start_level, force_id = self._get_meta() + level = len(m.group('level')) + start_level + if level > 6: + level = 6 + h = markdown.etree.SubElement(parent, 'h%d' % level) + h.text = m.group('header').strip() + if m.group('id'): + h.set('id', self._unique_id(m.group('id'))) + elif force_id: + h.set('id', self._create_id(m.group('header').strip())) + if after: + # Insert remaining lines as first block for future parsing. + blocks.insert(0, after) + else: + # This should never happen, but just in case... + message(CRITICAL, "We've got a problem header!") + + def _get_meta(self): + """ Return meta data suported by this ext as a tuple """ + level = int(self.config['level'][0]) - 1 + force = self._str2bool(self.config['forceid'][0]) + if hasattr(self.md, 'Meta'): + if self.md.Meta.has_key('header_level'): + level = int(self.md.Meta['header_level'][0]) - 1 + if self.md.Meta.has_key('header_forceid'): + force = self._str2bool(self.md.Meta['header_forceid'][0]) + return level, force + + def _str2bool(self, s, default=False): + """ Convert a string to a booleen value. """ + s = str(s) + if s.lower() in ['0', 'f', 'false', 'off', 'no', 'n']: + return False + elif s.lower() in ['1', 't', 'true', 'on', 'yes', 'y']: + return True + return default + + def _unique_id(self, id): + """ Ensure ID is unique. Append '_1', '_2'... if not """ + while id in self.IDs: + m = IDCOUNT_RE.match(id) + if m: + id = '%s_%d'% (m.group(1), int(m.group(2))+1) + else: + id = '%s_%d'% (id, 1) + self.IDs.append(id) + return id + + def _create_id(self, header): + """ Return ID from Header text. """ + h = '' + for c in header.lower().replace(' ', '_'): + if c in ID_CHARS: + h += c + elif c not in punctuation: + h += '+' + return self._unique_id(h) + + +class HeaderIdExtension (markdown.Extension): + def __init__(self, configs): + # set defaults + self.config = { + 'level' : ['1', 'Base level for headers.'], + 'forceid' : ['True', 'Force all headers to have an id.'] + } + + for key, value in configs: + self.setConfig(key, value) + + def extendMarkdown(self, md, md_globals): + md.registerExtension(self) + self.processor = HeaderIdProcessor(md.parser) + self.processor.md = md + self.processor.config = self.config + # Replace existing hasheader in place. + md.parser.blockprocessors['hashheader'] = self.processor + + def reset(self): + self.processor.IDs = [] + + +def makeExtension(configs=None): + return HeaderIdExtension(configs=configs) + +if __name__ == "__main__": + import doctest + doctest.testmod() + diff --git a/src/contrib/markdown/extensions/html_tidy.py b/src/contrib/markdown/extensions/html_tidy.py new file mode 100644 index 0000000..5105e33 --- /dev/null +++ b/src/contrib/markdown/extensions/html_tidy.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +""" +HTML Tidy Extension for Python-Markdown +======================================= + +Runs [HTML Tidy][] on the output of Python-Markdown using the [uTidylib][] +Python wrapper. Both libtidy and uTidylib must be installed on your system. + +Note than any Tidy [options][] can be passed in as extension configs. So, +for example, to output HTML rather than XHTML, set ``output_xhtml=0``. To +indent the output, set ``indent=auto`` and to have Tidy wrap the output in +``<html>`` and ``<body>`` tags, set ``show_body_only=0``. + +[HTML Tidy]: http://tidy.sourceforge.net/ +[uTidylib]: http://utidylib.berlios.de/ +[options]: http://tidy.sourceforge.net/docs/quickref.html + +Copyright (c)2008 [Waylan Limberg](http://achinghead.com) + +License: [BSD](http://www.opensource.org/licenses/bsd-license.php) + +Dependencies: +* [Python2.3+](http://python.org) +* [Markdown 2.0+](http://www.freewisdom.org/projects/python-markdown/) +* [HTML Tidy](http://utidylib.berlios.de/) +* [uTidylib](http://utidylib.berlios.de/) + +""" + +import markdown +import tidy + +class TidyExtension(markdown.Extension): + + def __init__(self, configs): + # Set defaults to match typical markdown behavior. + self.config = dict(output_xhtml=1, + show_body_only=1, + ) + # Merge in user defined configs overriding any present if nessecary. + for c in configs: + self.config[c[0]] = c[1] + + def extendMarkdown(self, md, md_globals): + # Save options to markdown instance + md.tidy_options = self.config + # Add TidyProcessor to postprocessors + md.postprocessors['tidy'] = TidyProcessor(md) + + +class TidyProcessor(markdown.postprocessors.Postprocessor): + + def run(self, text): + # Pass text to Tidy. As Tidy does not accept unicode we need to encode + # it and decode its return value. + return unicode(tidy.parseString(text.encode('utf-8'), + **self.markdown.tidy_options)) + + +def makeExtension(configs=None): + return TidyExtension(configs=configs) diff --git a/src/contrib/markdown/extensions/imagelinks.py b/src/contrib/markdown/extensions/imagelinks.py new file mode 100644 index 0000000..ee0b708 --- /dev/null +++ b/src/contrib/markdown/extensions/imagelinks.py @@ -0,0 +1,119 @@ +""" +========================= IMAGE LINKS ================================= + + +Turns paragraphs like + +<~~~~~~~~~~~~~~~~~~~~~~~~ +dir/subdir +dir/subdir +dir/subdir +~~~~~~~~~~~~~~ +dir/subdir +dir/subdir +dir/subdir +~~~~~~~~~~~~~~~~~~~> + +Into mini-photo galleries. + +""" + +import re, markdown +import url_manager + + +IMAGE_LINK = """<a href="%s"><img src="%s" title="%s"/></a>""" +SLIDESHOW_LINK = """<a href="%s" target="_blank">[slideshow]</a>""" +ALBUM_LINK = """ <a href="%s">[%s]</a>""" + + +class ImageLinksExtension(markdown.Extension): + + def extendMarkdown(self, md, md_globals): + + md.preprocessors.add("imagelink", ImageLinkPreprocessor(md), "_begin") + + +class ImageLinkPreprocessor(markdown.preprocessors.Preprocessor): + + def run(self, lines): + + url = url_manager.BlogEntryUrl(url_manager.BlogUrl("all"), + "2006/08/29/the_rest_of_our") + + + all_images = [] + blocks = [] + in_image_block = False + + new_lines = [] + + for line in lines: + + if line.startswith("<~~~~~~~"): + albums = [] + rows = [] + in_image_block = True + + if not in_image_block: + + new_lines.append(line) + + else: + + line = line.strip() + + if line.endswith("~~~~~~>") or not line: + in_image_block = False + new_block = "<div><br/><center><span class='image-links'>\n" + + album_url_hash = {} + + for row in rows: + for photo_url, title in row: + new_block += " " + new_block += IMAGE_LINK % (photo_url, + photo_url.get_thumbnail(), + title) + + album_url_hash[str(photo_url.get_album())] = 1 + + new_block += "<br/>" + + new_block += "</span>" + new_block += SLIDESHOW_LINK % url.get_slideshow() + + album_urls = album_url_hash.keys() + album_urls.sort() + + if len(album_urls) == 1: + new_block += ALBUM_LINK % (album_urls[0], "complete album") + else : + for i in range(len(album_urls)) : + new_block += ALBUM_LINK % (album_urls[i], + "album %d" % (i + 1) ) + + new_lines.append(new_block + "</center><br/></div>") + + elif line[1:6] == "~~~~~" : + rows.append([]) # start a new row + else : + parts = line.split() + line = parts[0] + title = " ".join(parts[1:]) + + album, photo = line.split("/") + photo_url = url.get_photo(album, photo, + len(all_images)+1) + all_images.append(photo_url) + rows[-1].append((photo_url, title)) + + if not album in albums : + albums.append(album) + + return new_lines + + +def makeExtension(configs): + return ImageLinksExtension(configs) + diff --git a/src/contrib/markdown/extensions/legacy.py b/src/contrib/markdown/extensions/legacy.py new file mode 100644 index 0000000..5b3a9c2 --- /dev/null +++ b/src/contrib/markdown/extensions/legacy.py @@ -0,0 +1,468 @@ +""" +Legacy Extension for Python-Markdown +==================================== + +Replaces the core parser with the old one. + +""" + +import markdown, re +from markdown import etree + +"""Basic and reusable regular expressions.""" + +def wrapRe(raw_re) : return re.compile("^%s$" % raw_re, re.DOTALL) +CORE_RE = { + 'header': wrapRe(r'(#{1,6})[ \t]*(.*?)[ \t]*(#*)'), # # A title + 'reference-def': wrapRe(r'(\ ?\ ?\ ?)\[([^\]]*)\]:\s*([^ ]*)(.*)'), + # [Google]: http://www.google.com/ + 'containsline': wrapRe(r'([-]*)$|^([=]*)'), # -----, =====, etc. + 'ol': wrapRe(r'[ ]{0,3}[\d]*\.\s+(.*)'), # 1. text + 'ul': wrapRe(r'[ ]{0,3}[*+-]\s+(.*)'), # "* text" + 'isline1': wrapRe(r'(\**)'), # *** + 'isline2': wrapRe(r'(\-*)'), # --- + 'isline3': wrapRe(r'(\_*)'), # ___ + 'tabbed': wrapRe(r'((\t)|( ))(.*)'), # an indented line + 'quoted': wrapRe(r'[ ]{0,2}> ?(.*)'), # a quoted block ("> ...") + 'containsline': re.compile(r'^([-]*)$|^([=]*)$', re.M), + 'attr': re.compile("\{@([^\}]*)=([^\}]*)}") # {@id=123} +} + +class MarkdownParser: + """Parser Markdown into a ElementTree.""" + + def __init__(self): + pass + + def parseDocument(self, lines): + """Parse a markdown string into an ElementTree.""" + # Create a ElementTree from the lines + root = etree.Element("div") + buffer = [] + for line in lines: + if line.startswith("#"): + self.parseChunk(root, buffer) + buffer = [line] + else: + buffer.append(line) + + self.parseChunk(root, buffer) + + return etree.ElementTree(root) + + def parseChunk(self, parent_elem, lines, inList=0, looseList=0): + """Process a chunk of markdown-formatted text and attach the parse to + an ElementTree node. + + Process a section of a source document, looking for high + level structural elements like lists, block quotes, code + segments, html blocks, etc. Some those then get stripped + of their high level markup (e.g. get unindented) and the + lower-level markup is processed recursively. + + Keyword arguments: + + * parent_elem: The ElementTree element to which the content will be + added. + * lines: a list of lines + * inList: a level + + Returns: None + + """ + # Loop through lines until none left. + while lines: + # Skipping empty line + if not lines[0]: + lines = lines[1:] + continue + + # Check if this section starts with a list, a blockquote or + # a code block. If so, process them. + processFn = { 'ul': self.__processUList, + 'ol': self.__processOList, + 'quoted': self.__processQuote, + 'tabbed': self.__processCodeBlock} + for regexp in ['ul', 'ol', 'quoted', 'tabbed']: + m = CORE_RE[regexp].match(lines[0]) + if m: + processFn[regexp](parent_elem, lines, inList) + return + + # We are NOT looking at one of the high-level structures like + # lists or blockquotes. So, it's just a regular paragraph + # (though perhaps nested inside a list or something else). If + # we are NOT inside a list, we just need to look for a blank + # line to find the end of the block. If we ARE inside a + # list, however, we need to consider that a sublist does not + # need to be separated by a blank line. Rather, the following + # markup is legal: + # + # * The top level list item + # + # Another paragraph of the list. This is where we are now. + # * Underneath we might have a sublist. + # + + if inList: + start, lines = self.__linesUntil(lines, (lambda line: + CORE_RE['ul'].match(line) + or CORE_RE['ol'].match(line) + or not line.strip())) + self.parseChunk(parent_elem, start, inList-1, + looseList=looseList) + inList = inList-1 + + else: # Ok, so it's just a simple block + test = lambda line: not line.strip() or line[0] == '>' + paragraph, lines = self.__linesUntil(lines, test) + if len(paragraph) and paragraph[0].startswith('#'): + self.__processHeader(parent_elem, paragraph) + elif len(paragraph) and CORE_RE["isline3"].match(paragraph[0]): + self.__processHR(parent_elem) + lines = paragraph[1:] + lines + elif paragraph: + self.__processParagraph(parent_elem, paragraph, + inList, looseList) + + if lines and not lines[0].strip(): + lines = lines[1:] # skip the first (blank) line + + def __processHR(self, parentElem): + hr = etree.SubElement(parentElem, "hr") + + def __processHeader(self, parentElem, paragraph): + m = CORE_RE['header'].match(paragraph[0]) + if m: + level = len(m.group(1)) + h = etree.SubElement(parentElem, "h%d" % level) + h.text = m.group(2).strip() + else: + message(CRITICAL, "We've got a problem header!") + + def __processParagraph(self, parentElem, paragraph, inList, looseList): + + if ( parentElem.tag == 'li' + and not (looseList or parentElem.getchildren())): + + # If this is the first paragraph inside "li", don't + # put <p> around it - append the paragraph bits directly + # onto parentElem + el = parentElem + else: + # Otherwise make a "p" element + el = etree.SubElement(parentElem, "p") + + dump = [] + + # Searching for hr or header + for line in paragraph: + # it's hr + if CORE_RE["isline3"].match(line): + el.text = "\n".join(dump) + self.__processHR(el) + dump = [] + # it's header + elif line.startswith("#"): + el.text = "\n".join(dump) + self.__processHeader(parentElem, [line]) + dump = [] + else: + dump.append(line) + if dump: + text = "\n".join(dump) + el.text = text + + def __processUList(self, parentElem, lines, inList): + self.__processList(parentElem, lines, inList, listexpr='ul', tag='ul') + + def __processOList(self, parentElem, lines, inList): + self.__processList(parentElem, lines, inList, listexpr='ol', tag='ol') + + def __processList(self, parentElem, lines, inList, listexpr, tag): + """ + Given a list of document lines starting with a list item, + finds the end of the list, breaks it up, and recursively + processes each list item and the remainder of the text file. + + Keyword arguments: + + * parentElem: A ElementTree element to which the content will be added + * lines: a list of lines + * inList: a level + + Returns: None + + """ + ul = etree.SubElement(parentElem, tag) # ul might actually be '<ol>' + + looseList = 0 + + # Make a list of list items + items = [] + item = -1 + + i = 0 # a counter to keep track of where we are + for line in lines: + loose = 0 + if not line.strip(): + # If we see a blank line, this _might_ be the end of the list + i += 1 + loose = 1 + + # Find the next non-blank line + for j in range(i, len(lines)): + if lines[j].strip(): + next = lines[j] + break + else: + # There is no more text => end of the list + break + + # Check if the next non-blank line is still a part of the list + + if ( CORE_RE[listexpr].match(next) or + CORE_RE['tabbed'].match(next) ): + # get rid of any white space in the line + items[item].append(line.strip()) + looseList = loose or looseList + continue + else: + break # found end of the list + + # Now we need to detect list items (at the current level) + # while also detabing child elements if necessary + + for expr in ['ul', 'ol', 'tabbed']: + m = CORE_RE[expr].match(line) + if m: + if expr in ['ul', 'ol']: # We are looking at a new item + #if m.group(1) : + # Removed the check to allow for a blank line + # at the beginning of the list item + items.append([m.group(1)]) + item += 1 + elif expr == 'tabbed': # This line needs to be detabbed + items[item].append(m.group(4)) #after the 'tab' + i += 1 + break + else: + items[item].append(line) # Just regular continuation + i += 1 # added on 2006.02.25 + else: + i += 1 + + # Add the ElementTree elements + for item in items: + li = etree.SubElement(ul, "li") + self.parseChunk(li, item, inList + 1, looseList = looseList) + + # Process the remaining part of the section + self.parseChunk(parentElem, lines[i:], inList) + + def __linesUntil(self, lines, condition): + """ + A utility function to break a list of lines upon the + first line that satisfied a condition. The condition + argument should be a predicate function. + + """ + i = -1 + for line in lines: + i += 1 + if condition(line): + break + else: + i += 1 + return lines[:i], lines[i:] + + def __processQuote(self, parentElem, lines, inList): + """ + Given a list of document lines starting with a quote finds + the end of the quote, unindents it and recursively + processes the body of the quote and the remainder of the + text file. + + Keyword arguments: + + * parentElem: ElementTree element to which the content will be added + * lines: a list of lines + * inList: a level + + Returns: None + + """ + dequoted = [] + i = 0 + blank_line = False # allow one blank line between paragraphs + for line in lines: + m = CORE_RE['quoted'].match(line) + if m: + dequoted.append(m.group(1)) + i += 1 + blank_line = False + elif not blank_line and line.strip() != '': + dequoted.append(line) + i += 1 + elif not blank_line and line.strip() == '': + dequoted.append(line) + i += 1 + blank_line = True + else: + break + + blockquote = etree.SubElement(parentElem, "blockquote") + + self.parseChunk(blockquote, dequoted, inList) + self.parseChunk(parentElem, lines[i:], inList) + + def __processCodeBlock(self, parentElem, lines, inList): + """ + Given a list of document lines starting with a code block + finds the end of the block, puts it into the ElementTree verbatim + wrapped in ("<pre><code>") and recursively processes the + the remainder of the text file. + + Keyword arguments: + + * parentElem: ElementTree element to which the content will be added + * lines: a list of lines + * inList: a level + + Returns: None + + """ + detabbed, theRest = self.detectTabbed(lines) + pre = etree.SubElement(parentElem, "pre") + code = etree.SubElement(pre, "code") + text = "\n".join(detabbed).rstrip()+"\n" + code.text = markdown.AtomicString(text) + self.parseChunk(parentElem, theRest, inList) + + def detectTabbed(self, lines): + """ Find indented text and remove indent before further proccesing. + + Keyword arguments: + + * lines: an array of strings + + Returns: a list of post processed items and the unused + remainder of the original list + + """ + items = [] + item = -1 + i = 0 # to keep track of where we are + + def detab(line): + match = CORE_RE['tabbed'].match(line) + if match: + return match.group(4) + + for line in lines: + if line.strip(): # Non-blank line + line = detab(line) + if line: + items.append(line) + i += 1 + continue + else: + return items, lines[i:] + + else: # Blank line: _maybe_ we are done. + i += 1 # advance + + # Find the next non-blank line + for j in range(i, len(lines)): + if lines[j].strip(): + next_line = lines[j]; break + else: + break # There is no more text; we are done. + + # Check if the next non-blank line is tabbed + if detab(next_line): # Yes, more work to do. + items.append("") + continue + else: + break # No, we are done. + else: + i += 1 + + return items, lines[i:] + +class HeaderPreprocessor(markdown.Preprocessor): + + """Replace underlined headers with hashed headers. + + (To avoid the need for lookahead later.) + + """ + + def run (self, lines): + i = -1 + while i+1 < len(lines): + i = i+1 + if not lines[i].strip(): + continue + + if lines[i].startswith("#"): + lines.insert(i+1, "\n") + + if (i+1 <= len(lines) + and lines[i+1] + and lines[i+1][0] in ['-', '=']): + + underline = lines[i+1].strip() + + if underline == "="*len(underline): + lines[i] = "# " + lines[i].strip() + lines[i+1] = "" + elif underline == "-"*len(underline): + lines[i] = "## " + lines[i].strip() + lines[i+1] = "" + + return lines + + +class LinePreprocessor(markdown.Preprocessor): + """Convert HR lines to "___" format.""" + blockquote_re = re.compile(r'^(> )+') + + def run (self, lines): + for i in range(len(lines)): + prefix = '' + m = self.blockquote_re.search(lines[i]) + if m: + prefix = m.group(0) + if self._isLine(lines[i][len(prefix):]): + lines[i] = prefix + "___" + return lines + + def _isLine(self, block): + """Determine if a block should be replaced with an <HR>""" + if block.startswith(" "): + return False # a code block + text = "".join([x for x in block if not x.isspace()]) + if len(text) <= 2: + return False + for pattern in ['isline1', 'isline2', 'isline3']: + m = CORE_RE[pattern].match(text) + if (m and m.group(1)): + return True + else: + return False + + +class LegacyExtension(markdown.Extension): + """ Replace Markdown's core parser. """ + + def extendMarkdown(self, md, md_globals): + """ Set the core parser to an instance of MarkdownParser. """ + md.parser = MarkdownParser() + md.preprocessors.add ("header", HeaderPreprocessor(self), "<reference") + md.preprocessors.add("line", LinePreprocessor(self), "<reference") + + +def makeExtension(configs={}): + return LegacyExtension(configs=configs) + diff --git a/src/contrib/markdown/extensions/meta.py b/src/contrib/markdown/extensions/meta.py new file mode 100644 index 0000000..1b555b2 --- /dev/null +++ b/src/contrib/markdown/extensions/meta.py @@ -0,0 +1,90 @@ +#!usr/bin/python + +""" +Meta Data Extension for Python-Markdown +======================================= + +This extension adds Meta Data handling to markdown. + +Basic Usage: + + >>> import markdown + >>> text = '''Title: A Test Doc. + ... Author: Waylan Limberg + ... John Doe + ... Blank_Data: + ... + ... The body. This is paragraph one. + ... ''' + >>> md = markdown.Markdown(['meta']) + >>> md.convert(text) + u'<p>The body. This is paragraph one.</p>' + >>> md.Meta + {u'blank_data': [u''], u'author': [u'Waylan Limberg', u'John Doe'], u'title': [u'A Test Doc.']} + +Make sure text without Meta Data still works (markdown < 1.6b returns a <p>). + + >>> text = ' Some Code - not extra lines of meta data.' + >>> md = markdown.Markdown(['meta']) + >>> md.convert(text) + u'<pre><code>Some Code - not extra lines of meta data.\\n</code></pre>' + >>> md.Meta + {} + +Copyright 2007-2008 [Waylan Limberg](http://achinghead.com). + +Project website: <http://www.freewisdom.org/project/python-markdown/Meta-Data> +Contact: markdown@freewisdom.org + +License: BSD (see ../docs/LICENSE for details) + +""" + +import markdown, re + +# Global Vars +META_RE = re.compile(r'^[ ]{0,3}(?P<key>[A-Za-z0-9_-]+):\s*(?P<value>.*)') +META_MORE_RE = re.compile(r'^[ ]{4,}(?P<value>.*)') + +class MetaExtension (markdown.Extension): + """ Meta-Data extension for Python-Markdown. """ + + def extendMarkdown(self, md, md_globals): + """ Add MetaPreprocessor to Markdown instance. """ + + md.preprocessors.add("meta", MetaPreprocessor(md), "_begin") + + +class MetaPreprocessor(markdown.preprocessors.Preprocessor): + """ Get Meta-Data. """ + + def run(self, lines): + """ Parse Meta-Data and store in Markdown.Meta. """ + meta = {} + key = None + while 1: + line = lines.pop(0) + if line.strip() == '': + break # blank line - done + m1 = META_RE.match(line) + if m1: + key = m1.group('key').lower().strip() + meta[key] = [m1.group('value').strip()] + else: + m2 = META_MORE_RE.match(line) + if m2 and key: + # Add another line to existing key + meta[key].append(m2.group('value').strip()) + else: + lines.insert(0, line) + break # no meta data - done + self.markdown.Meta = meta + return lines + + +def makeExtension(configs={}): + return MetaExtension(configs=configs) + +if __name__ == "__main__": + import doctest + doctest.testmod() diff --git a/src/contrib/markdown/extensions/rss.py b/src/contrib/markdown/extensions/rss.py new file mode 100644 index 0000000..1274da2 --- /dev/null +++ b/src/contrib/markdown/extensions/rss.py @@ -0,0 +1,114 @@ +import markdown +from markdown import etree + +DEFAULT_URL = "http://www.freewisdom.org/projects/python-markdown/" +DEFAULT_CREATOR = "Yuri Takhteyev" +DEFAULT_TITLE = "Markdown in Python" +GENERATOR = "http://www.freewisdom.org/projects/python-markdown/markdown2rss" + +month_map = { "Jan" : "01", + "Feb" : "02", + "March" : "03", + "April" : "04", + "May" : "05", + "June" : "06", + "July" : "07", + "August" : "08", + "September" : "09", + "October" : "10", + "November" : "11", + "December" : "12" } + +def get_time(heading): + + heading = heading.split("-")[0] + heading = heading.strip().replace(",", " ").replace(".", " ") + + month, date, year = heading.split() + month = month_map[month] + + return rdftime(" ".join((month, date, year, "12:00:00 AM"))) + +def rdftime(time): + + time = time.replace(":", " ") + time = time.replace("/", " ") + time = time.split() + return "%s-%s-%sT%s:%s:%s-08:00" % (time[0], time[1], time[2], + time[3], time[4], time[5]) + + +def get_date(text): + return "date" + +class RssExtension (markdown.Extension): + + def extendMarkdown(self, md, md_globals): + + self.config = { 'URL' : [DEFAULT_URL, "Main URL"], + 'CREATOR' : [DEFAULT_CREATOR, "Feed creator's name"], + 'TITLE' : [DEFAULT_TITLE, "Feed title"] } + + md.xml_mode = True + + # Insert a tree-processor that would actually add the title tag + treeprocessor = RssTreeProcessor(md) + treeprocessor.ext = self + md.treeprocessors['rss'] = treeprocessor + md.stripTopLevelTags = 0 + md.docType = '<?xml version="1.0" encoding="utf-8"?>\n' + +class RssTreeProcessor(markdown.treeprocessors.Treeprocessor): + + def run (self, root): + + rss = etree.Element("rss") + rss.set("version", "2.0") + + channel = etree.SubElement(rss, "channel") + + for tag, text in (("title", self.ext.getConfig("TITLE")), + ("link", self.ext.getConfig("URL")), + ("description", None)): + + element = etree.SubElement(channel, tag) + element.text = text + + for child in root: + + if child.tag in ["h1", "h2", "h3", "h4", "h5"]: + + heading = child.text.strip() + item = etree.SubElement(channel, "item") + link = etree.SubElement(item, "link") + link.text = self.ext.getConfig("URL") + title = etree.SubElement(item, "title") + title.text = heading + + guid = ''.join([x for x in heading if x.isalnum()]) + guidElem = etree.SubElement(item, "guid") + guidElem.text = guid + guidElem.set("isPermaLink", "false") + + elif child.tag in ["p"]: + try: + description = etree.SubElement(item, "description") + except UnboundLocalError: + # Item not defined - moving on + pass + else: + if len(child): + content = "\n".join([etree.tostring(node) + for node in child]) + else: + content = child.text + pholder = self.markdown.htmlStash.store( + "<![CDATA[ %s]]>" % content) + description.text = pholder + + return rss + + +def makeExtension(configs): + + return RssExtension(configs) diff --git a/src/contrib/markdown/extensions/tables.py b/src/contrib/markdown/extensions/tables.py new file mode 100644 index 0000000..1d3c920 --- /dev/null +++ b/src/contrib/markdown/extensions/tables.py @@ -0,0 +1,97 @@ +#!/usr/bin/env Python +""" +Tables Extension for Python-Markdown +==================================== + +Added parsing of tables to Python-Markdown. + +A simple example: + + First Header | Second Header + ------------- | ------------- + Content Cell | Content Cell + Content Cell | Content Cell + +Copyright 2009 - [Waylan Limberg](http://achinghead.com) +""" +import markdown +from markdown import etree + + +class TableProcessor(markdown.blockprocessors.BlockProcessor): + """ Process Tables. """ + + def test(self, parent, block): + rows = block.split('\n') + return (len(rows) > 2 and '|' in rows[0] and + '|' in rows[1] and '-' in rows[1] and + rows[1][0] in ['|', ':', '-']) + + def run(self, parent, blocks): + """ Parse a table block and build table. """ + block = blocks.pop(0).split('\n') + header = block[:2] + rows = block[2:] + # Get format type (bordered by pipes or not) + border = False + if header[0].startswith('|'): + border = True + # Get alignment of columns + align = [] + for c in self._split_row(header[1], border): + if c.startswith(':') and c.endswith(':'): + align.append('center') + elif c.startswith(':'): + align.append('left') + elif c.endswith(':'): + align.append('right') + else: + align.append(None) + # Build table + table = etree.SubElement(parent, 'table') + thead = etree.SubElement(table, 'thead') + self._build_row(header[0], thead, align, border) + tbody = etree.SubElement(table, 'tbody') + for row in rows: + self._build_row(row, tbody, align, border) + + def _build_row(self, row, parent, align, border): + """ Given a row of text, build table cells. """ + tr = etree.SubElement(parent, 'tr') + tag = 'td' + if parent.tag == 'thead': + tag = 'th' + cells = self._split_row(row, border) + # We use align here rather than cells to ensure every row + # contains the same number of columns. + for i, a in enumerate(align): + c = etree.SubElement(tr, tag) + try: + c.text = cells[i].strip() + except IndexError: + c.text = "" + if a: + c.set('align', a) + + def _split_row(self, row, border): + """ split a row of text into list of cells. """ + if border: + if row.startswith('|'): + row = row[1:] + if row.endswith('|'): + row = row[:-1] + return row.split('|') + + +class TableExtension(markdown.Extension): + """ Add tables to Markdown. """ + + def extendMarkdown(self, md, md_globals): + """ Add an instance of TableProcessor to BlockParser. """ + md.parser.blockprocessors.add('table', + TableProcessor(md.parser), + '<hashheader') + + +def makeExtension(configs={}): + return TableExtension(configs=configs) diff --git a/src/contrib/markdown/extensions/toc.py b/src/contrib/markdown/extensions/toc.py new file mode 100644 index 0000000..1624ccf --- /dev/null +++ b/src/contrib/markdown/extensions/toc.py @@ -0,0 +1,140 @@ +""" +Table of Contents Extension for Python-Markdown +* * * + +(c) 2008 [Jack Miller](http://codezen.org) + +Dependencies: +* [Markdown 2.0+](http://www.freewisdom.org/projects/python-markdown/) + +""" +import markdown +from markdown import etree +import re + +class TocTreeprocessor(markdown.treeprocessors.Treeprocessor): + # Iterator wrapper to get parent and child all at once + def iterparent(self, root): + for parent in root.getiterator(): + for child in parent: + yield parent, child + + def run(self, doc): + div = etree.Element("div") + div.attrib["class"] = "toc" + last_li = None + + # Add title to the div + if self.config["title"][0]: + header = etree.SubElement(div, "span") + header.attrib["class"] = "toctitle" + header.text = self.config["title"][0] + + level = 0 + list_stack=[div] + header_rgx = re.compile("[Hh][123456]") + + # Get a list of id attributes + used_ids = [] + for c in doc.getiterator(): + if "id" in c.attrib: + used_ids.append(c.attrib["id"]) + + for (p, c) in self.iterparent(doc): + if not c.text: + continue + + # To keep the output from screwing up the + # validation by putting a <div> inside of a <p> + # we actually replace the <p> in its entirety. + # We do not allow the marker inside a header as that + # would causes an enless loop of placing a new TOC + # inside previously generated TOC. + + if c.text.find(self.config["marker"][0]) > -1 and not header_rgx.match(c.tag): + for i in range(len(p)): + if p[i] == c: + p[i] = div + break + + if header_rgx.match(c.tag): + tag_level = int(c.tag[-1]) + + # Regardless of how many levels we jumped + # only one list should be created, since + # empty lists containing lists are illegal. + + if tag_level < level: + list_stack.pop() + level = tag_level + + if tag_level > level: + newlist = etree.Element("ul") + if last_li: + last_li.append(newlist) + else: + list_stack[-1].append(newlist) + list_stack.append(newlist) + level = tag_level + + # Do not override pre-existing ids + if not "id" in c.attrib: + id = self.config["slugify"][0](c.text) + if id in used_ids: + ctr = 1 + while "%s_%d" % (id, ctr) in used_ids: + ctr += 1 + id = "%s_%d" % (id, ctr) + used_ids.append(id) + c.attrib["id"] = id + else: + id = c.attrib["id"] + + # List item link, to be inserted into the toc div + last_li = etree.Element("li") + link = etree.SubElement(last_li, "a") + link.text = c.text + link.attrib["href"] = '#' + id + + if int(self.config["anchorlink"][0]): + anchor = etree.SubElement(c, "a") + anchor.text = c.text + anchor.attrib["href"] = "#" + id + anchor.attrib["class"] = "toclink" + c.text = "" + + list_stack[-1].append(last_li) + +class TocExtension(markdown.Extension): + def __init__(self, configs): + self.config = { "marker" : ["[TOC]", + "Text to find and replace with Table of Contents -" + "Defaults to \"[TOC]\""], + "slugify" : [self.slugify, + "Function to generate anchors based on header text-" + "Defaults to a built in slugify function."], + "title" : [None, + "Title to insert into TOC <div> - " + "Defaults to None"], + "anchorlink" : [0, + "1 if header should be a self link" + "Defaults to 0"]} + + for key, value in configs: + self.setConfig(key, value) + + # This is exactly the same as Django's slugify + def slugify(self, value): + """ Slugify a string, to make it URL friendly. """ + import unicodedata + value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore') + value = unicode(re.sub('[^\w\s-]', '', value).strip().lower()) + return re.sub('[-\s]+','-',value) + + def extendMarkdown(self, md, md_globals): + tocext = TocTreeprocessor(md) + tocext.config = self.config + md.treeprocessors.add("toc", tocext, "_begin") + +def makeExtension(configs={}): + return TocExtension(configs=configs) diff --git a/src/contrib/markdown/extensions/wikilinks.py b/src/contrib/markdown/extensions/wikilinks.py new file mode 100644 index 0000000..df44e1c --- /dev/null +++ b/src/contrib/markdown/extensions/wikilinks.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python + +''' +WikiLinks Extension for Python-Markdown +====================================== + +Converts [[WikiLinks]] to relative links. Requires Python-Markdown 2.0+ + +Basic usage: + + >>> import markdown + >>> text = "Some text with a [[WikiLink]]." + >>> html = markdown.markdown(text, ['wikilinks']) + >>> html + u'<p>Some text with a <a class="wikilink" href="/WikiLink/">WikiLink</a>.</p>' + +Whitespace behavior: + + >>> markdown.markdown('[[ foo bar_baz ]]', ['wikilinks']) + u'<p><a class="wikilink" href="/foo_bar_baz/">foo bar_baz</a></p>' + >>> markdown.markdown('foo [[ ]] bar', ['wikilinks']) + u'<p>foo bar</p>' + +To define custom settings the simple way: + + >>> markdown.markdown(text, + ... ['wikilinks(base_url=/wiki/,end_url=.html,html_class=foo)'] + ... ) + u'<p>Some text with a <a class="foo" href="/wiki/WikiLink.html">WikiLink</a>.</p>' + +Custom settings the complex way: + + >>> md = markdown.Markdown( + ... extensions = ['wikilinks'], + ... extension_configs = {'wikilinks': [ + ... ('base_url', 'http://example.com/'), + ... ('end_url', '.html'), + ... ('html_class', '') ]}, + ... safe_mode = True) + >>> md.convert(text) + u'<p>Some text with a <a href="http://example.com/WikiLink.html">WikiLink</a>.</p>' + +Use MetaData with mdx_meta.py (Note the blank html_class in MetaData): + + >>> text = """wiki_base_url: http://example.com/ + ... wiki_end_url: .html + ... wiki_html_class: + ... + ... Some text with a [[WikiLink]].""" + >>> md = markdown.Markdown(extensions=['meta', 'wikilinks']) + >>> md.convert(text) + u'<p>Some text with a <a href="http://example.com/WikiLink.html">WikiLink</a>.</p>' + +MetaData should not carry over to next document: + + >>> md.convert("No [[MetaData]] here.") + u'<p>No <a class="wikilink" href="/MetaData/">MetaData</a> here.</p>' + +Define a custom URL builder: + + >>> def my_url_builder(label, base, end): + ... return '/bar/' + >>> md = markdown.Markdown(extensions=['wikilinks'], + ... extension_configs={'wikilinks' : [('build_url', my_url_builder)]}) + >>> md.convert('[[foo]]') + u'<p><a class="wikilink" href="/bar/">foo</a></p>' + +From the command line: + + python markdown.py -x wikilinks(base_url=http://example.com/,end_url=.html,html_class=foo) src.txt + +By [Waylan Limberg](http://achinghead.com/). + +License: [BSD](http://www.opensource.org/licenses/bsd-license.php) + +Dependencies: +* [Python 2.3+](http://python.org) +* [Markdown 2.0+](http://www.freewisdom.org/projects/python-markdown/) +''' + +import markdown +import re + +def build_url(label, base, end): + """ Build a url from the label, a base, and an end. """ + clean_label = re.sub(r'([ ]+_)|(_[ ]+)|([ ]+)', '_', label) + return '%s%s%s'% (base, clean_label, end) + + +class WikiLinkExtension(markdown.Extension): + def __init__(self, configs): + # set extension defaults + self.config = { + 'base_url' : ['/', 'String to append to beginning or URL.'], + 'end_url' : ['/', 'String to append to end of URL.'], + 'html_class' : ['wikilink', 'CSS hook. Leave blank for none.'], + 'build_url' : [build_url, 'Callable formats URL from label.'], + } + + # Override defaults with user settings + for key, value in configs : + self.setConfig(key, value) + + def extendMarkdown(self, md, md_globals): + self.md = md + + # append to end of inline patterns + WIKILINK_RE = r'\[\[([A-Za-z0-9_ -]+)\]\]' + wikilinkPattern = WikiLinks(WIKILINK_RE, self.config) + wikilinkPattern.md = md + md.inlinePatterns.add('wikilink', wikilinkPattern, "<not_strong") + + +class WikiLinks(markdown.inlinepatterns.Pattern): + def __init__(self, pattern, config): + markdown.inlinepatterns.Pattern.__init__(self, pattern) + self.config = config + + def handleMatch(self, m): + if m.group(2).strip(): + base_url, end_url, html_class = self._getMeta() + label = m.group(2).strip() + url = self.config['build_url'][0](label, base_url, end_url) + a = markdown.etree.Element('a') + a.text = label + a.set('href', url) + if html_class: + a.set('class', html_class) + else: + a = '' + return a + + def _getMeta(self): + """ Return meta data or config data. """ + base_url = self.config['base_url'][0] + end_url = self.config['end_url'][0] + html_class = self.config['html_class'][0] + if hasattr(self.md, 'Meta'): + if self.md.Meta.has_key('wiki_base_url'): + base_url = self.md.Meta['wiki_base_url'][0] + if self.md.Meta.has_key('wiki_end_url'): + end_url = self.md.Meta['wiki_end_url'][0] + if self.md.Meta.has_key('wiki_html_class'): + html_class = self.md.Meta['wiki_html_class'][0] + return base_url, end_url, html_class + + +def makeExtension(configs=None) : + return WikiLinkExtension(configs=configs) + + +if __name__ == "__main__": + import doctest + doctest.testmod() + diff --git a/src/contrib/markdown/html4.py b/src/contrib/markdown/html4.py new file mode 100644 index 0000000..08f241d --- /dev/null +++ b/src/contrib/markdown/html4.py @@ -0,0 +1,274 @@ +# markdown/html4.py +# +# Add html4 serialization to older versions of Elementree +# Taken from ElementTree 1.3 preview with slight modifications +# +# Copyright (c) 1999-2007 by Fredrik Lundh. All rights reserved. +# +# fredrik@pythonware.com +# http://www.pythonware.com +# +# -------------------------------------------------------------------- +# The ElementTree toolkit is +# +# Copyright (c) 1999-2007 by Fredrik Lundh +# +# By obtaining, using, and/or copying this software and/or its +# associated documentation, you agree that you have read, understood, +# and will comply with the following terms and conditions: +# +# Permission to use, copy, modify, and distribute this software and +# its associated documentation for any purpose and without fee is +# hereby granted, provided that the above copyright notice appears in +# all copies, and that both that copyright notice and this permission +# notice appear in supporting documentation, and that the name of +# Secret Labs AB or the author not be used in advertising or publicity +# pertaining to distribution of the software without specific, written +# prior permission. +# +# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD +# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT- +# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR +# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY +# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS +# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE +# OF THIS SOFTWARE. +# -------------------------------------------------------------------- + + +import markdown +ElementTree = markdown.etree.ElementTree +QName = markdown.etree.QName +Comment = markdown.etree.Comment +PI = markdown.etree.PI +ProcessingInstruction = markdown.etree.ProcessingInstruction + +HTML_EMPTY = ("area", "base", "basefont", "br", "col", "frame", "hr", + "img", "input", "isindex", "link", "meta" "param") + +try: + HTML_EMPTY = set(HTML_EMPTY) +except NameError: + pass + +_namespace_map = { + # "well-known" namespace prefixes + "http://www.w3.org/XML/1998/namespace": "xml", + "http://www.w3.org/1999/xhtml": "html", + "http://www.w3.org/1999/02/22-rdf-syntax-ns#": "rdf", + "http://schemas.xmlsoap.org/wsdl/": "wsdl", + # xml schema + "http://www.w3.org/2001/XMLSchema": "xs", + "http://www.w3.org/2001/XMLSchema-instance": "xsi", + # dublic core + "http://purl.org/dc/elements/1.1/": "dc", +} + + +def _raise_serialization_error(text): + raise TypeError( + "cannot serialize %r (type %s)" % (text, type(text).__name__) + ) + +def _encode(text, encoding): + try: + return text.encode(encoding, "xmlcharrefreplace") + except (TypeError, AttributeError): + _raise_serialization_error(text) + +def _escape_cdata(text, encoding): + # escape character data + try: + # it's worth avoiding do-nothing calls for strings that are + # shorter than 500 character, or so. assume that's, by far, + # the most common case in most applications. + if "&" in text: + text = text.replace("&", "&") + if "<" in text: + text = text.replace("<", "<") + if ">" in text: + text = text.replace(">", ">") + return text.encode(encoding, "xmlcharrefreplace") + except (TypeError, AttributeError): + _raise_serialization_error(text) + + +def _escape_attrib(text, encoding): + # escape attribute value + try: + if "&" in text: + text = text.replace("&", "&") + if "<" in text: + text = text.replace("<", "<") + if ">" in text: + text = text.replace(">", ">") + if "\"" in text: + text = text.replace("\"", """) + if "\n" in text: + text = text.replace("\n", " ") + return text.encode(encoding, "xmlcharrefreplace") + except (TypeError, AttributeError): + _raise_serialization_error(text) + +def _escape_attrib_html(text, encoding): + # escape attribute value + try: + if "&" in text: + text = text.replace("&", "&") + if ">" in text: + text = text.replace(">", ">") + if "\"" in text: + text = text.replace("\"", """) + return text.encode(encoding, "xmlcharrefreplace") + except (TypeError, AttributeError): + _raise_serialization_error(text) + + +def _serialize_html(write, elem, encoding, qnames, namespaces): + tag = elem.tag + text = elem.text + if tag is Comment: + write("<!--%s-->" % _escape_cdata(text, encoding)) + elif tag is ProcessingInstruction: + write("<?%s?>" % _escape_cdata(text, encoding)) + else: + tag = qnames[tag] + if tag is None: + if text: + write(_escape_cdata(text, encoding)) + for e in elem: + _serialize_html(write, e, encoding, qnames, None) + else: + write("<" + tag) + items = elem.items() + if items or namespaces: + items.sort() # lexical order + for k, v in items: + if isinstance(k, QName): + k = k.text + if isinstance(v, QName): + v = qnames[v.text] + else: + v = _escape_attrib_html(v, encoding) + # FIXME: handle boolean attributes + write(" %s=\"%s\"" % (qnames[k], v)) + if namespaces: + items = namespaces.items() + items.sort(key=lambda x: x[1]) # sort on prefix + for v, k in items: + if k: + k = ":" + k + write(" xmlns%s=\"%s\"" % ( + k.encode(encoding), + _escape_attrib(v, encoding) + )) + write(">") + tag = tag.lower() + if text: + if tag == "script" or tag == "style": + write(_encode(text, encoding)) + else: + write(_escape_cdata(text, encoding)) + for e in elem: + _serialize_html(write, e, encoding, qnames, None) + if tag not in HTML_EMPTY: + write("</" + tag + ">") + if elem.tail: + write(_escape_cdata(elem.tail, encoding)) + +def write_html(root, f, + # keyword arguments + encoding="us-ascii", + default_namespace=None): + assert root is not None + if not hasattr(f, "write"): + f = open(f, "wb") + write = f.write + if not encoding: + encoding = "us-ascii" + qnames, namespaces = _namespaces( + root, encoding, default_namespace + ) + _serialize_html( + write, root, encoding, qnames, namespaces + ) + +# -------------------------------------------------------------------- +# serialization support + +def _namespaces(elem, encoding, default_namespace=None): + # identify namespaces used in this tree + + # maps qnames to *encoded* prefix:local names + qnames = {None: None} + + # maps uri:s to prefixes + namespaces = {} + if default_namespace: + namespaces[default_namespace] = "" + + def encode(text): + return text.encode(encoding) + + def add_qname(qname): + # calculate serialized qname representation + try: + if qname[:1] == "{": + uri, tag = qname[1:].split("}", 1) + prefix = namespaces.get(uri) + if prefix is None: + prefix = _namespace_map.get(uri) + if prefix is None: + prefix = "ns%d" % len(namespaces) + if prefix != "xml": + namespaces[uri] = prefix + if prefix: + qnames[qname] = encode("%s:%s" % (prefix, tag)) + else: + qnames[qname] = encode(tag) # default element + else: + if default_namespace: + # FIXME: can this be handled in XML 1.0? + raise ValueError( + "cannot use non-qualified names with " + "default_namespace option" + ) + qnames[qname] = encode(qname) + except TypeError: + _raise_serialization_error(qname) + + # populate qname and namespaces table + try: + iterate = elem.iter + except AttributeError: + iterate = elem.getiterator # cET compatibility + for elem in iterate(): + tag = elem.tag + if isinstance(tag, QName) and tag.text not in qnames: + add_qname(tag.text) + elif isinstance(tag, basestring): + if tag not in qnames: + add_qname(tag) + elif tag is not None and tag is not Comment and tag is not PI: + _raise_serialization_error(tag) + for key, value in elem.items(): + if isinstance(key, QName): + key = key.text + if key not in qnames: + add_qname(key) + if isinstance(value, QName) and value.text not in qnames: + add_qname(value.text) + text = elem.text + if isinstance(text, QName) and text.text not in qnames: + add_qname(text.text) + return qnames, namespaces + +def to_html_string(element, encoding=None): + class dummy: + pass + data = [] + file = dummy() + file.write = data.append + write_html(ElementTree(element).getroot(),file,encoding) + return "".join(data) diff --git a/src/contrib/markdown/inlinepatterns.py b/src/contrib/markdown/inlinepatterns.py new file mode 100644 index 0000000..89fa3b2 --- /dev/null +++ b/src/contrib/markdown/inlinepatterns.py @@ -0,0 +1,371 @@ +""" +INLINE PATTERNS +============================================================================= + +Inline patterns such as *emphasis* are handled by means of auxiliary +objects, one per pattern. Pattern objects must be instances of classes +that extend markdown.Pattern. Each pattern object uses a single regular +expression and needs support the following methods: + + pattern.getCompiledRegExp() # returns a regular expression + + pattern.handleMatch(m) # takes a match object and returns + # an ElementTree element or just plain text + +All of python markdown's built-in patterns subclass from Pattern, +but you can add additional patterns that don't. + +Also note that all the regular expressions used by inline must +capture the whole block. For this reason, they all start with +'^(.*)' and end with '(.*)!'. In case with built-in expression +Pattern takes care of adding the "^(.*)" and "(.*)!". + +Finally, the order in which regular expressions are applied is very +important - e.g. if we first replace http://.../ links with <a> tags +and _then_ try to replace inline html, we would end up with a mess. +So, we apply the expressions in the following order: + +* escape and backticks have to go before everything else, so + that we can preempt any markdown patterns by escaping them. + +* then we handle auto-links (must be done before inline html) + +* then we handle inline HTML. At this point we will simply + replace all inline HTML strings with a placeholder and add + the actual HTML to a hash. + +* then inline images (must be done before links) + +* then bracketed links, first regular then reference-style + +* finally we apply strong and emphasis +""" + +import markdown +import re +from urlparse import urlparse, urlunparse +import sys +if sys.version >= "3.0": + from html import entities as htmlentitydefs +else: + import htmlentitydefs + +""" +The actual regular expressions for patterns +----------------------------------------------------------------------------- +""" + +NOBRACKET = r'[^\]\[]*' +BRK = ( r'\[(' + + (NOBRACKET + r'(\[')*6 + + (NOBRACKET+ r'\])*')*6 + + NOBRACKET + r')\]' ) +NOIMG = r'(?<!\!)' + +BACKTICK_RE = r'(?<!\\)(`+)(.+?)(?<!`)\2(?!`)' # `e=f()` or ``e=f("`")`` +ESCAPE_RE = r'\\(.)' # \< +EMPHASIS_RE = r'(\*)([^\*]*)\2' # *emphasis* +STRONG_RE = r'(\*{2}|_{2})(.*?)\2' # **strong** +STRONG_EM_RE = r'(\*{3}|_{3})(.*?)\2' # ***strong*** + +if markdown.SMART_EMPHASIS: + EMPHASIS_2_RE = r'(?<!\S)(_)(\S.*?)\2' # _emphasis_ +else: + EMPHASIS_2_RE = r'(_)(.*?)\2' # _emphasis_ + +LINK_RE = NOIMG + BRK + \ +r'''\(\s*(<.*?>|((?:(?:\(.*?\))|[^\(\)]))*?)\s*((['"])(.*)\12)?\)''' +# [text](url) or [text](<url>) + +IMAGE_LINK_RE = r'\!' + BRK + r'\s*\((<.*?>|([^\)]*))\)' +#  or  +REFERENCE_RE = NOIMG + BRK+ r'\s*\[([^\]]*)\]' # [Google][3] +IMAGE_REFERENCE_RE = r'\!' + BRK + '\s*\[([^\]]*)\]' # ![alt text][2] +NOT_STRONG_RE = r'( \* )' # stand-alone * or _ +AUTOLINK_RE = r'<((?:f|ht)tps?://[^>]*)>' # <http://www.123.com> +AUTOMAIL_RE = r'<([^> \!]*@[^> ]*)>' # <me@example.com> + +HTML_RE = r'(\<([a-zA-Z/][^\>]*?|\!--.*?--)\>)' # <...> +ENTITY_RE = r'(&[\#a-zA-Z0-9]*;)' # & +LINE_BREAK_RE = r' \n' # two spaces at end of line +LINE_BREAK_2_RE = r' $' # two spaces at end of text + + +def dequote(string): + """Remove quotes from around a string.""" + if ( ( string.startswith('"') and string.endswith('"')) + or (string.startswith("'") and string.endswith("'")) ): + return string[1:-1] + else: + return string + +ATTR_RE = re.compile("\{@([^\}]*)=([^\}]*)}") # {@id=123} + +def handleAttributes(text, parent): + """Set values of an element based on attribute definitions ({@id=123}).""" + def attributeCallback(match): + parent.set(match.group(1), match.group(2).replace('\n', ' ')) + return ATTR_RE.sub(attributeCallback, text) + + +""" +The pattern classes +----------------------------------------------------------------------------- +""" + +class Pattern: + """Base class that inline patterns subclass. """ + + def __init__ (self, pattern, markdown_instance=None): + """ + Create an instant of an inline pattern. + + Keyword arguments: + + * pattern: A regular expression that matches a pattern + + """ + self.pattern = pattern + self.compiled_re = re.compile("^(.*?)%s(.*?)$" % pattern, re.DOTALL) + + # Api for Markdown to pass safe_mode into instance + self.safe_mode = False + if markdown_instance: + self.markdown = markdown_instance + + def getCompiledRegExp (self): + """ Return a compiled regular expression. """ + return self.compiled_re + + def handleMatch(self, m): + """Return a ElementTree element from the given match. + + Subclasses should override this method. + + Keyword arguments: + + * m: A re match object containing a match of the pattern. + + """ + pass + + def type(self): + """ Return class name, to define pattern type """ + return self.__class__.__name__ + +BasePattern = Pattern # for backward compatibility + +class SimpleTextPattern (Pattern): + """ Return a simple text of group(2) of a Pattern. """ + def handleMatch(self, m): + text = m.group(2) + if text == markdown.INLINE_PLACEHOLDER_PREFIX: + return None + return text + +class SimpleTagPattern (Pattern): + """ + Return element of type `tag` with a text attribute of group(3) + of a Pattern. + + """ + def __init__ (self, pattern, tag): + Pattern.__init__(self, pattern) + self.tag = tag + + def handleMatch(self, m): + el = markdown.etree.Element(self.tag) + el.text = m.group(3) + return el + + +class SubstituteTagPattern (SimpleTagPattern): + """ Return a eLement of type `tag` with no children. """ + def handleMatch (self, m): + return markdown.etree.Element(self.tag) + + +class BacktickPattern (Pattern): + """ Return a `<code>` element containing the matching text. """ + def __init__ (self, pattern): + Pattern.__init__(self, pattern) + self.tag = "code" + + def handleMatch(self, m): + el = markdown.etree.Element(self.tag) + el.text = markdown.AtomicString(m.group(3).strip()) + return el + + +class DoubleTagPattern (SimpleTagPattern): + """Return a ElementTree element nested in tag2 nested in tag1. + + Useful for strong emphasis etc. + + """ + def handleMatch(self, m): + tag1, tag2 = self.tag.split(",") + el1 = markdown.etree.Element(tag1) + el2 = markdown.etree.SubElement(el1, tag2) + el2.text = m.group(3) + return el1 + + +class HtmlPattern (Pattern): + """ Store raw inline html and return a placeholder. """ + def handleMatch (self, m): + rawhtml = m.group(2) + inline = True + place_holder = self.markdown.htmlStash.store(rawhtml) + return place_holder + + +class LinkPattern (Pattern): + """ Return a link element from the given match. """ + def handleMatch(self, m): + el = markdown.etree.Element("a") + el.text = m.group(2) + title = m.group(11) + href = m.group(9) + + if href: + if href[0] == "<": + href = href[1:-1] + el.set("href", self.sanitize_url(href.strip())) + else: + el.set("href", "") + + if title: + title = dequote(title) #.replace('"', """) + el.set("title", title) + return el + + def sanitize_url(self, url): + """ + Sanitize a url against xss attacks in "safe_mode". + + Rather than specifically blacklisting `javascript:alert("XSS")` and all + its aliases (see <http://ha.ckers.org/xss.html>), we whitelist known + safe url formats. Most urls contain a network location, however some + are known not to (i.e.: mailto links). Script urls do not contain a + location. Additionally, for `javascript:...`, the scheme would be + "javascript" but some aliases will appear to `urlparse()` to have no + scheme. On top of that relative links (i.e.: "foo/bar.html") have no + scheme. Therefore we must check "path", "parameters", "query" and + "fragment" for any literal colons. We don't check "scheme" for colons + because it *should* never have any and "netloc" must allow the form: + `username:password@host:port`. + + """ + locless_schemes = ['', 'mailto', 'news'] + scheme, netloc, path, params, query, fragment = url = urlparse(url) + safe_url = False + if netloc != '' or scheme in locless_schemes: + safe_url = True + + for part in url[2:]: + if ":" in part: + safe_url = False + + if self.markdown.safeMode and not safe_url: + return '' + else: + return urlunparse(url) + +class ImagePattern(LinkPattern): + """ Return a img element from the given match. """ + def handleMatch(self, m): + el = markdown.etree.Element("img") + src_parts = m.group(9).split() + if src_parts: + src = src_parts[0] + if src[0] == "<" and src[-1] == ">": + src = src[1:-1] + el.set('src', self.sanitize_url(src)) + else: + el.set('src', "") + if len(src_parts) > 1: + el.set('title', dequote(" ".join(src_parts[1:]))) + + if markdown.ENABLE_ATTRIBUTES: + truealt = handleAttributes(m.group(2), el) + else: + truealt = m.group(2) + + el.set('alt', truealt) + return el + +class ReferencePattern(LinkPattern): + """ Match to a stored reference and return link element. """ + def handleMatch(self, m): + if m.group(9): + id = m.group(9).lower() + else: + # if we got something like "[Google][]" + # we'll use "google" as the id + id = m.group(2).lower() + + if not id in self.markdown.references: # ignore undefined refs + return None + href, title = self.markdown.references[id] + + text = m.group(2) + return self.makeTag(href, title, text) + + def makeTag(self, href, title, text): + el = markdown.etree.Element('a') + + el.set('href', self.sanitize_url(href)) + if title: + el.set('title', title) + + el.text = text + return el + + +class ImageReferencePattern (ReferencePattern): + """ Match to a stored reference and return img element. """ + def makeTag(self, href, title, text): + el = markdown.etree.Element("img") + el.set("src", self.sanitize_url(href)) + if title: + el.set("title", title) + el.set("alt", text) + return el + + +class AutolinkPattern (Pattern): + """ Return a link Element given an autolink (`<http://example/com>`). """ + def handleMatch(self, m): + el = markdown.etree.Element("a") + el.set('href', m.group(2)) + el.text = markdown.AtomicString(m.group(2)) + return el + +class AutomailPattern (Pattern): + """ + Return a mailto link Element given an automail link (`<foo@example.com>`). + """ + def handleMatch(self, m): + el = markdown.etree.Element('a') + email = m.group(2) + if email.startswith("mailto:"): + email = email[len("mailto:"):] + + def codepoint2name(code): + """Return entity definition by code, or the code if not defined.""" + entity = htmlentitydefs.codepoint2name.get(code) + if entity: + return "%s%s;" % (markdown.AMP_SUBSTITUTE, entity) + else: + return "%s#%d;" % (markdown.AMP_SUBSTITUTE, code) + + letters = [codepoint2name(ord(letter)) for letter in email] + el.text = markdown.AtomicString(''.join(letters)) + + mailto = "mailto:" + email + mailto = "".join([markdown.AMP_SUBSTITUTE + '#%d;' % + ord(letter) for letter in mailto]) + el.set('href', mailto) + return el + diff --git a/src/contrib/markdown/odict.py b/src/contrib/markdown/odict.py new file mode 100644 index 0000000..bf3ef07 --- /dev/null +++ b/src/contrib/markdown/odict.py @@ -0,0 +1,162 @@ +class OrderedDict(dict): + """ + A dictionary that keeps its keys in the order in which they're inserted. + + Copied from Django's SortedDict with some modifications. + + """ + def __new__(cls, *args, **kwargs): + instance = super(OrderedDict, cls).__new__(cls, *args, **kwargs) + instance.keyOrder = [] + return instance + + def __init__(self, data=None): + if data is None: + data = {} + super(OrderedDict, self).__init__(data) + if isinstance(data, dict): + self.keyOrder = data.keys() + else: + self.keyOrder = [] + for key, value in data: + if key not in self.keyOrder: + self.keyOrder.append(key) + + def __deepcopy__(self, memo): + from copy import deepcopy + return self.__class__([(key, deepcopy(value, memo)) + for key, value in self.iteritems()]) + + def __setitem__(self, key, value): + super(OrderedDict, self).__setitem__(key, value) + if key not in self.keyOrder: + self.keyOrder.append(key) + + def __delitem__(self, key): + super(OrderedDict, self).__delitem__(key) + self.keyOrder.remove(key) + + def __iter__(self): + for k in self.keyOrder: + yield k + + def pop(self, k, *args): + result = super(OrderedDict, self).pop(k, *args) + try: + self.keyOrder.remove(k) + except ValueError: + # Key wasn't in the dictionary in the first place. No problem. + pass + return result + + def popitem(self): + result = super(OrderedDict, self).popitem() + self.keyOrder.remove(result[0]) + return result + + def items(self): + return zip(self.keyOrder, self.values()) + + def iteritems(self): + for key in self.keyOrder: + yield key, super(OrderedDict, self).__getitem__(key) + + def keys(self): + return self.keyOrder[:] + + def iterkeys(self): + return iter(self.keyOrder) + + def values(self): + return [super(OrderedDict, self).__getitem__(k) for k in self.keyOrder] + + def itervalues(self): + for key in self.keyOrder: + yield super(OrderedDict, self).__getitem__(key) + + def update(self, dict_): + for k, v in dict_.items(): + self.__setitem__(k, v) + + def setdefault(self, key, default): + if key not in self.keyOrder: + self.keyOrder.append(key) + return super(OrderedDict, self).setdefault(key, default) + + def value_for_index(self, index): + """Return the value of the item at the given zero-based index.""" + return self[self.keyOrder[index]] + + def insert(self, index, key, value): + """Insert the key, value pair before the item with the given index.""" + if key in self.keyOrder: + n = self.keyOrder.index(key) + del self.keyOrder[n] + if n < index: + index -= 1 + self.keyOrder.insert(index, key) + super(OrderedDict, self).__setitem__(key, value) + + def copy(self): + """Return a copy of this object.""" + # This way of initializing the copy means it works for subclasses, too. + obj = self.__class__(self) + obj.keyOrder = self.keyOrder[:] + return obj + + def __repr__(self): + """ + Replace the normal dict.__repr__ with a version that returns the keys + in their sorted order. + """ + return '{%s}' % ', '.join(['%r: %r' % (k, v) for k, v in self.items()]) + + def clear(self): + super(OrderedDict, self).clear() + self.keyOrder = [] + + def index(self, key): + """ Return the index of a given key. """ + return self.keyOrder.index(key) + + def index_for_location(self, location): + """ Return index or None for a given location. """ + if location == '_begin': + i = 0 + elif location == '_end': + i = None + elif location.startswith('<') or location.startswith('>'): + i = self.index(location[1:]) + if location.startswith('>'): + if i >= len(self): + # last item + i = None + else: + i += 1 + else: + raise ValueError('Not a valid location: "%s". Location key ' + 'must start with a ">" or "<".' % location) + return i + + def add(self, key, value, location): + """ Insert by key location. """ + i = self.index_for_location(location) + if i is not None: + self.insert(i, key, value) + else: + self.__setitem__(key, value) + + def link(self, key, location): + """ Change location of an existing item. """ + n = self.keyOrder.index(key) + del self.keyOrder[n] + i = self.index_for_location(location) + try: + if i is not None: + self.keyOrder.insert(i, key) + else: + self.keyOrder.append(key) + except Error: + # restore to prevent data loss and reraise + self.keyOrder.insert(n, key) + raise Error diff --git a/src/contrib/markdown/postprocessors.py b/src/contrib/markdown/postprocessors.py new file mode 100644 index 0000000..80227bb --- /dev/null +++ b/src/contrib/markdown/postprocessors.py @@ -0,0 +1,77 @@ +""" +POST-PROCESSORS +============================================================================= + +Markdown also allows post-processors, which are similar to preprocessors in +that they need to implement a "run" method. However, they are run after core +processing. + +""" + + +import markdown + +class Processor: + def __init__(self, markdown_instance=None): + if markdown_instance: + self.markdown = markdown_instance + +class Postprocessor(Processor): + """ + Postprocessors are run after the ElementTree it converted back into text. + + Each Postprocessor implements a "run" method that takes a pointer to a + text string, modifies it as necessary and returns a text string. + + Postprocessors must extend markdown.Postprocessor. + + """ + + def run(self, text): + """ + Subclasses of Postprocessor should implement a `run` method, which + takes the html document as a single text string and returns a + (possibly modified) string. + + """ + pass + + +class RawHtmlPostprocessor(Postprocessor): + """ Restore raw html to the document. """ + + def run(self, text): + """ Iterate over html stash and restore "safe" html. """ + for i in range(self.markdown.htmlStash.html_counter): + html, safe = self.markdown.htmlStash.rawHtmlBlocks[i] + if self.markdown.safeMode and not safe: + if str(self.markdown.safeMode).lower() == 'escape': + html = self.escape(html) + elif str(self.markdown.safeMode).lower() == 'remove': + html = '' + else: + html = markdown.HTML_REMOVED_TEXT + if safe or not self.markdown.safeMode: + text = text.replace("<p>%s</p>" % + (markdown.preprocessors.HTML_PLACEHOLDER % i), + html + "\n") + text = text.replace(markdown.preprocessors.HTML_PLACEHOLDER % i, + html) + return text + + def escape(self, html): + """ Basic html escaping """ + html = html.replace('&', '&') + html = html.replace('<', '<') + html = html.replace('>', '>') + return html.replace('"', '"') + + +class AndSubstitutePostprocessor(Postprocessor): + """ Restore valid entities """ + def __init__(self): + pass + + def run(self, text): + text = text.replace(markdown.AMP_SUBSTITUTE, "&") + return text diff --git a/src/contrib/markdown/preprocessors.py b/src/contrib/markdown/preprocessors.py new file mode 100644 index 0000000..712a1e8 --- /dev/null +++ b/src/contrib/markdown/preprocessors.py @@ -0,0 +1,214 @@ + +""" +PRE-PROCESSORS +============================================================================= + +Preprocessors work on source text before we start doing anything too +complicated. +""" + +import re +import markdown + +HTML_PLACEHOLDER_PREFIX = markdown.STX+"wzxhzdk:" +HTML_PLACEHOLDER = HTML_PLACEHOLDER_PREFIX + "%d" + markdown.ETX + +class Processor: + def __init__(self, markdown_instance=None): + if markdown_instance: + self.markdown = markdown_instance + +class Preprocessor (Processor): + """ + Preprocessors are run after the text is broken into lines. + + Each preprocessor implements a "run" method that takes a pointer to a + list of lines of the document, modifies it as necessary and returns + either the same pointer or a pointer to a new list. + + Preprocessors must extend markdown.Preprocessor. + + """ + def run(self, lines): + """ + Each subclass of Preprocessor should override the `run` method, which + takes the document as a list of strings split by newlines and returns + the (possibly modified) list of lines. + + """ + pass + +class HtmlStash: + """ + This class is used for stashing HTML objects that we extract + in the beginning and replace with place-holders. + """ + + def __init__ (self): + """ Create a HtmlStash. """ + self.html_counter = 0 # for counting inline html segments + self.rawHtmlBlocks=[] + + def store(self, html, safe=False): + """ + Saves an HTML segment for later reinsertion. Returns a + placeholder string that needs to be inserted into the + document. + + Keyword arguments: + + * html: an html segment + * safe: label an html segment as safe for safemode + + Returns : a placeholder string + + """ + self.rawHtmlBlocks.append((html, safe)) + placeholder = HTML_PLACEHOLDER % self.html_counter + self.html_counter += 1 + return placeholder + + def reset(self): + self.html_counter = 0 + self.rawHtmlBlocks = [] + + +class HtmlBlockPreprocessor(Preprocessor): + """Remove html blocks from the text and store them for later retrieval.""" + + right_tag_patterns = ["</%s>", "%s>"] + + def _get_left_tag(self, block): + return block[1:].replace(">", " ", 1).split()[0].lower() + + def _get_right_tag(self, left_tag, block): + for p in self.right_tag_patterns: + tag = p % left_tag + i = block.rfind(tag) + if i > 2: + return tag.lstrip("<").rstrip(">"), i + len(p)-2 + len(left_tag) + return block.rstrip()[-len(left_tag)-2:-1].lower(), len(block) + + def _equal_tags(self, left_tag, right_tag): + if left_tag == 'div' or left_tag[0] in ['?', '@', '%']: # handle PHP, etc. + return True + if ("/" + left_tag) == right_tag: + return True + if (right_tag == "--" and left_tag == "--"): + return True + elif left_tag == right_tag[1:] \ + and right_tag[0] != "<": + return True + else: + return False + + def _is_oneliner(self, tag): + return (tag in ['hr', 'hr/']) + + def run(self, lines): + text = "\n".join(lines) + new_blocks = [] + text = text.split("\n\n") + items = [] + left_tag = '' + right_tag = '' + in_tag = False # flag + + while text: + block = text[0] + if block.startswith("\n"): + block = block[1:] + text = text[1:] + + if block.startswith("\n"): + block = block[1:] + + if not in_tag: + if block.startswith("<"): + left_tag = self._get_left_tag(block) + right_tag, data_index = self._get_right_tag(left_tag, block) + + if data_index < len(block): + text.insert(0, block[data_index:]) + block = block[:data_index] + + if not (markdown.isBlockLevel(left_tag) \ + or block[1] in ["!", "?", "@", "%"]): + new_blocks.append(block) + continue + + if self._is_oneliner(left_tag): + new_blocks.append(block.strip()) + continue + + if block[1] == "!": + # is a comment block + left_tag = "--" + right_tag, data_index = self._get_right_tag(left_tag, block) + # keep checking conditions below and maybe just append + + if block.rstrip().endswith(">") \ + and self._equal_tags(left_tag, right_tag): + new_blocks.append( + self.markdown.htmlStash.store(block.strip())) + continue + else: #if not block[1] == "!": + # if is block level tag and is not complete + + if markdown.isBlockLevel(left_tag) or left_tag == "--" \ + and not block.rstrip().endswith(">"): + items.append(block.strip()) + in_tag = True + else: + new_blocks.append( + self.markdown.htmlStash.store(block.strip())) + + continue + + new_blocks.append(block) + + else: + items.append(block.strip()) + + right_tag, data_index = self._get_right_tag(left_tag, block) + + if self._equal_tags(left_tag, right_tag): + # if find closing tag + in_tag = False + new_blocks.append( + self.markdown.htmlStash.store('\n\n'.join(items))) + items = [] + + if items: + new_blocks.append(self.markdown.htmlStash.store('\n\n'.join(items))) + new_blocks.append('\n') + + new_text = "\n\n".join(new_blocks) + return new_text.split("\n") + + +class ReferencePreprocessor(Preprocessor): + """ Remove reference definitions from text and store for later use. """ + + RE = re.compile(r'^(\ ?\ ?\ ?)\[([^\]]*)\]:\s*([^ ]*)(.*)$', re.DOTALL) + + def run (self, lines): + new_text = []; + for line in lines: + m = self.RE.match(line) + if m: + id = m.group(2).strip().lower() + t = m.group(4).strip() # potential title + if not t: + self.markdown.references[id] = (m.group(3), t) + elif (len(t) >= 2 + and (t[0] == t[-1] == "\"" + or t[0] == t[-1] == "\'" + or (t[0] == "(" and t[-1] == ")") ) ): + self.markdown.references[id] = (m.group(3), t[1:-1]) + else: + new_text.append(line) + else: + new_text.append(line) + + return new_text #+ "\n" diff --git a/src/contrib/markdown/treeprocessors.py b/src/contrib/markdown/treeprocessors.py new file mode 100644 index 0000000..1dc612a --- /dev/null +++ b/src/contrib/markdown/treeprocessors.py @@ -0,0 +1,329 @@ +import markdown +import re + +def isString(s): + """ Check if it's string """ + return isinstance(s, unicode) or isinstance(s, str) + +class Processor: + def __init__(self, markdown_instance=None): + if markdown_instance: + self.markdown = markdown_instance + +class Treeprocessor(Processor): + """ + Treeprocessors are run on the ElementTree object before serialization. + + Each Treeprocessor implements a "run" method that takes a pointer to an + ElementTree, modifies it as necessary and returns an ElementTree + object. + + Treeprocessors must extend markdown.Treeprocessor. + + """ + def run(self, root): + """ + Subclasses of Treeprocessor should implement a `run` method, which + takes a root ElementTree. This method can return another ElementTree + object, and the existing root ElementTree will be replaced, or it can + modify the current tree and return None. + """ + pass + + +class InlineProcessor(Treeprocessor): + """ + A Treeprocessor that traverses a tree, applying inline patterns. + """ + + def __init__ (self, md): + self.__placeholder_prefix = markdown.INLINE_PLACEHOLDER_PREFIX + self.__placeholder_suffix = markdown.ETX + self.__placeholder_length = 4 + len(self.__placeholder_prefix) \ + + len(self.__placeholder_suffix) + self.__placeholder_re = re.compile(markdown.INLINE_PLACEHOLDER % r'([0-9]{4})') + self.markdown = md + + def __makePlaceholder(self, type): + """ Generate a placeholder """ + id = "%04d" % len(self.stashed_nodes) + hash = markdown.INLINE_PLACEHOLDER % id + return hash, id + + def __findPlaceholder(self, data, index): + """ + Extract id from data string, start from index + + Keyword arguments: + + * data: string + * index: index, from which we start search + + Returns: placeholder id and string index, after the found placeholder. + """ + + m = self.__placeholder_re.search(data, index) + if m: + return m.group(1), m.end() + else: + return None, index + 1 + + def __stashNode(self, node, type): + """ Add node to stash """ + placeholder, id = self.__makePlaceholder(type) + self.stashed_nodes[id] = node + return placeholder + + def __handleInline(self, data, patternIndex=0): + """ + Process string with inline patterns and replace it + with placeholders + + Keyword arguments: + + * data: A line of Markdown text + * patternIndex: The index of the inlinePattern to start with + + Returns: String with placeholders. + + """ + if not isinstance(data, markdown.AtomicString): + startIndex = 0 + while patternIndex < len(self.markdown.inlinePatterns): + data, matched, startIndex = self.__applyPattern( + self.markdown.inlinePatterns.value_for_index(patternIndex), + data, patternIndex, startIndex) + if not matched: + patternIndex += 1 + return data + + def __processElementText(self, node, subnode, isText=True): + """ + Process placeholders in Element.text or Element.tail + of Elements popped from self.stashed_nodes. + + Keywords arguments: + + * node: parent node + * subnode: processing node + * isText: bool variable, True - it's text, False - it's tail + + Returns: None + + """ + if isText: + text = subnode.text + subnode.text = None + else: + text = subnode.tail + subnode.tail = None + + childResult = self.__processPlaceholders(text, subnode) + + if not isText and node is not subnode: + pos = node.getchildren().index(subnode) + node.remove(subnode) + else: + pos = 0 + + childResult.reverse() + for newChild in childResult: + node.insert(pos, newChild) + + def __processPlaceholders(self, data, parent): + """ + Process string with placeholders and generate ElementTree tree. + + Keyword arguments: + + * data: string with placeholders instead of ElementTree elements. + * parent: Element, which contains processing inline data + + Returns: list with ElementTree elements with applied inline patterns. + """ + def linkText(text): + if text: + if result: + if result[-1].tail: + result[-1].tail += text + else: + result[-1].tail = text + else: + if parent.text: + parent.text += text + else: + parent.text = text + + result = [] + strartIndex = 0 + while data: + index = data.find(self.__placeholder_prefix, strartIndex) + if index != -1: + id, phEndIndex = self.__findPlaceholder(data, index) + + if id in self.stashed_nodes: + node = self.stashed_nodes.get(id) + + if index > 0: + text = data[strartIndex:index] + linkText(text) + + if not isString(node): # it's Element + for child in [node] + node.getchildren(): + if child.tail: + if child.tail.strip(): + self.__processElementText(node, child, False) + if child.text: + if child.text.strip(): + self.__processElementText(child, child) + else: # it's just a string + linkText(node) + strartIndex = phEndIndex + continue + + strartIndex = phEndIndex + result.append(node) + + else: # wrong placeholder + end = index + len(prefix) + linkText(data[strartIndex:end]) + strartIndex = end + else: + text = data[strartIndex:] + linkText(text) + data = "" + + return result + + def __applyPattern(self, pattern, data, patternIndex, startIndex=0): + """ + Check if the line fits the pattern, create the necessary + elements, add it to stashed_nodes. + + Keyword arguments: + + * data: the text to be processed + * pattern: the pattern to be checked + * patternIndex: index of current pattern + * startIndex: string index, from which we starting search + + Returns: String with placeholders instead of ElementTree elements. + + """ + match = pattern.getCompiledRegExp().match(data[startIndex:]) + leftData = data[:startIndex] + + if not match: + return data, False, 0 + + node = pattern.handleMatch(match) + + if node is None: + return data, True, len(leftData) + match.span(len(match.groups()))[0] + + if not isString(node): + if not isinstance(node.text, markdown.AtomicString): + # We need to process current node too + for child in [node] + node.getchildren(): + if not isString(node): + if child.text: + child.text = self.__handleInline(child.text, + patternIndex + 1) + if child.tail: + child.tail = self.__handleInline(child.tail, + patternIndex) + + placeholder = self.__stashNode(node, pattern.type()) + + return "%s%s%s%s" % (leftData, + match.group(1), + placeholder, match.groups()[-1]), True, 0 + + def run(self, tree): + """Apply inline patterns to a parsed Markdown tree. + + Iterate over ElementTree, find elements with inline tag, apply inline + patterns and append newly created Elements to tree. If you don't + want process your data with inline paterns, instead of normal string, + use subclass AtomicString: + + node.text = markdown.AtomicString("data won't be processed with inline patterns") + + Arguments: + + * markdownTree: ElementTree object, representing Markdown tree. + + Returns: ElementTree object with applied inline patterns. + + """ + self.stashed_nodes = {} + + stack = [tree] + + while stack: + currElement = stack.pop() + insertQueue = [] + for child in currElement.getchildren(): + if child.text and not isinstance(child.text, markdown.AtomicString): + text = child.text + child.text = None + lst = self.__processPlaceholders(self.__handleInline( + text), child) + stack += lst + insertQueue.append((child, lst)) + + if child.getchildren(): + stack.append(child) + + for element, lst in insertQueue: + if element.text: + element.text = \ + markdown.inlinepatterns.handleAttributes(element.text, + element) + i = 0 + for newChild in lst: + # Processing attributes + if newChild.tail: + newChild.tail = \ + markdown.inlinepatterns.handleAttributes(newChild.tail, + element) + if newChild.text: + newChild.text = \ + markdown.inlinepatterns.handleAttributes(newChild.text, + newChild) + element.insert(i, newChild) + i += 1 + return tree + + +class PrettifyTreeprocessor(Treeprocessor): + """ Add linebreaks to the html document. """ + + def _prettifyETree(self, elem): + """ Recursively add linebreaks to ElementTree children. """ + + i = "\n" + if markdown.isBlockLevel(elem.tag) and elem.tag not in ['code', 'pre']: + if (not elem.text or not elem.text.strip()) \ + and len(elem) and markdown.isBlockLevel(elem[0].tag): + elem.text = i + for e in elem: + if markdown.isBlockLevel(e.tag): + self._prettifyETree(e) + if not elem.tail or not elem.tail.strip(): + elem.tail = i + if not elem.tail or not elem.tail.strip(): + elem.tail = i + + def run(self, root): + """ Add linebreaks to ElementTree root object. """ + + self._prettifyETree(root) + # Do <br />'s seperately as they are often in the middle of + # inline content and missed by _prettifyETree. + brs = root.getiterator('br') + for br in brs: + if not br.tail or not br.tail.strip(): + br.tail = '\n' + else: + br.tail = '\n%s' % br.tail |