diff options
| -rw-r--r-- | AUTHORS | 1 | ||||
| -rw-r--r-- | pygments/lexers/_mapping.py | 1 | ||||
| -rw-r--r-- | pygments/lexers/agile.py | 112 | ||||
| -rw-r--r-- | tests/examplefiles/language.hy | 165 | ||||
| -rw-r--r-- | tests/test_examplefiles.py | 5 |
5 files changed, 282 insertions, 2 deletions
@@ -136,6 +136,7 @@ Other contributors, listed alphabetically, are: * Tiberius Teng -- default style overhaul * Jeremy Thurgood -- Erlang, Squid config lexers * Brian Tiffin -- OpenCOBOL lexer +# Bob Tolbert -- Hy lexer * Erick Tryzelaar -- Felix lexer * Daniele Varrazzo -- PostgreSQL lexers * Abe Voelker -- OpenEdge ABL lexer diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index 969bdba5..c96001e3 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -136,6 +136,7 @@ LEXERS = { 'HttpLexer': ('pygments.lexers.text', 'HTTP', ('http',), (), ()), 'HxmlLexer': ('pygments.lexers.text', 'Hxml', ('haxeml', 'hxml'), ('*.hxml',), ()), 'HybrisLexer': ('pygments.lexers.other', 'Hybris', ('hybris', 'hy'), ('*.hy', '*.hyb'), ('text/x-hybris', 'application/x-hybris')), + 'HyLexer': ('pygments.lexers.agile', 'Hy', ('hylang',), ('*.hy',), ('text/x-hy', 'application/x-hy')), 'IDLLexer': ('pygments.lexers.math', 'IDL', ('idl',), ('*.pro',), ('text/idl',)), 'IgorLexer': ('pygments.lexers.math', 'Igor', ('igor', 'igorpro'), ('*.ipf',), ('text/ipf',)), 'IniLexer': ('pygments.lexers.text', 'INI', ('ini', 'cfg', 'dosini'), ('*.ini', '*.cfg'), ('text/x-ini',)), diff --git a/pygments/lexers/agile.py b/pygments/lexers/agile.py index 1f81365e..bc0cf4c0 100644 --- a/pygments/lexers/agile.py +++ b/pygments/lexers/agile.py @@ -23,7 +23,7 @@ __all__ = ['PythonLexer', 'PythonConsoleLexer', 'PythonTracebackLexer', 'Python3Lexer', 'Python3TracebackLexer', 'RubyLexer', 'RubyConsoleLexer', 'PerlLexer', 'LuaLexer', 'MoonScriptLexer', 'CrocLexer', 'MiniDLexer', 'IoLexer', 'TclLexer', 'FactorLexer', - 'FancyLexer', 'DgLexer', 'Perl6Lexer'] + 'FancyLexer', 'DgLexer', 'Perl6Lexer', 'HyLexer'] # b/w compatibility from pygments.lexers.functional import SchemeLexer @@ -2288,3 +2288,113 @@ class Perl6Lexer(ExtendedRegexLexer): def __init__(self, **options): super(Perl6Lexer, self).__init__(**options) self.encoding = options.get('encoding', 'utf-8') + + +class HyLexer(RegexLexer): + """ + Lexer for `Hy <http://hylang.org/>`_ source code. + """ + name = 'Hy' + aliases = ['hy'] + filenames = ['*.hy'] + mimetypes = ['text/x-hy', 'application/x-hy'] + + special_forms = [ + 'cond', 'for', '->', '->>', 'car', + 'cdr', 'first', 'rest', 'let', 'when', 'unless', + 'import', 'do', 'progn', 'get', 'slice', 'assoc', 'with-decorator', + ',', 'list_comp', 'kwapply', '~', 'is', 'in', 'is-not', 'not-in', + 'quasiquote', 'unquote', 'unquote-splice', 'quote', '|', '<<=', '>>=', + 'foreach', 'while', + 'eval-and-compile', 'eval-when-compile' + ] + + declarations = [ + 'def' 'defn', 'defun', 'defmacro', 'defclass', 'lambda', 'fn', 'setv' + ] + + hy_builtins = [] + + hy_core = [ + 'cycle', 'dec', 'distinct', 'drop', 'even?', 'filter', 'inc', + 'instance?', 'iterable?', 'iterate', 'iterator?', 'neg?', + 'none?', 'nth', 'numeric?', 'odd?', 'pos?', 'remove', 'repeat', + 'repeatedly', 'take', 'take_nth', 'take_while', 'zero?' + ] + + builtins = hy_builtins + hy_core + + # valid names for identifiers + # well, names can only not consist fully of numbers + # but this should be good enough for now + valid_name = r'(?!#)[\w!$%*+<=>?/.#-]+' + + def _multi_escape(entries): + return '(%s)' % ('|'.join(re.escape(entry) + ' ' for entry in entries)) + + tokens = { + 'root': [ + # the comments - always starting with semicolon + # and going to the end of the line + (r';.*$', Comment.Single), + + # whitespaces - usually not relevant + (r'[,\s]+', Text), + + # numbers + (r'-?\d+\.\d+', Number.Float), + (r'-?\d+', Number.Integer), + (r'0[0-7]+j?', Number.Oct), + (r'0[xX][a-fA-F0-9]+', Number.Hex), + + # strings, symbols and characters + (r'"(\\\\|\\"|[^"])*"', String), + (r"'" + valid_name, String.Symbol), + (r"\\(.|[a-z]+)", String.Char), + (r'^(\s*)([rRuU]{,2}"""(?:.|\n)*?""")', bygroups(Text, String.Doc)), + (r"^(\s*)([rRuU]{,2}'''(?:.|\n)*?''')", bygroups(Text, String.Doc)), + + # keywords + (r'::?' + valid_name, String.Symbol), + + # special operators + (r'~@|[`\'#^~&@]', Operator), + + include('py-keywords'), + include('py-builtins'), + + # highlight the special forms + (_multi_escape(special_forms), Keyword), + + # Technically, only the special forms are 'keywords'. The problem + # is that only treating them as keywords means that things like + # 'defn' and 'ns' need to be highlighted as builtins. This is ugly + # and weird for most styles. So, as a compromise we're going to + # highlight them as Keyword.Declarations. + (_multi_escape(declarations), Keyword.Declaration), + + # highlight the builtins + (_multi_escape(builtins), Name.Builtin), + + # the remaining functions + (r'(?<=\()' + valid_name, Name.Function), + + # find the remaining variables + (valid_name, Name.Variable), + + # Hy accepts vector notation + (r'(\[|\])', Punctuation), + + # Hy accepts map notation + (r'(\{|\})', Punctuation), + + # the famous parentheses! + (r'(\(|\))', Punctuation), + + ], + 'py-keywords': PythonLexer.tokens['keywords'], + 'py-builtins': PythonLexer.tokens['builtins'], + } + + def analyse_text(text): + return '(import' in text or '(defn' in text diff --git a/tests/examplefiles/language.hy b/tests/examplefiles/language.hy new file mode 100644 index 00000000..9768c39c --- /dev/null +++ b/tests/examplefiles/language.hy @@ -0,0 +1,165 @@ +;;;; This contains some of the core Hy functions used +;;;; to make functional programming slightly easier. +;;;; + + +(defn _numeric-check [x] + (if (not (numeric? x)) + (raise (TypeError (.format "{0!r} is not a number" x))))) + +(defn cycle [coll] + "Yield an infinite repetition of the items in coll" + (setv seen []) + (for [x coll] + (yield x) + (.append seen x)) + (while seen + (for [x seen] + (yield x)))) + +(defn dec [n] + "Decrement n by 1" + (_numeric-check n) + (- n 1)) + +(defn distinct [coll] + "Return a generator from the original collection with duplicates + removed" + (let [[seen []] [citer (iter coll)]] + (for [val citer] + (if (not_in val seen) + (do + (yield val) + (.append seen val)))))) + +(defn drop [count coll] + "Drop `count` elements from `coll` and yield back the rest" + (let [[citer (iter coll)]] + (try (for [i (range count)] + (next citer)) + (catch [StopIteration])) + citer)) + +(defn even? [n] + "Return true if n is an even number" + (_numeric-check n) + (= (% n 2) 0)) + +(defn filter [pred coll] + "Return all elements from `coll` that pass `pred`" + (let [[citer (iter coll)]] + (for [val citer] + (if (pred val) + (yield val))))) + +(defn inc [n] + "Increment n by 1" + (_numeric-check n) + (+ n 1)) + +(defn instance? [klass x] + (isinstance x klass)) + +(defn iterable? [x] + "Return true if x is iterable" + (try (do (iter x) true) + (catch [Exception] false))) + +(defn iterate [f x] + (setv val x) + (while true + (yield val) + (setv val (f val)))) + +(defn iterator? [x] + "Return true if x is an iterator" + (try (= x (iter x)) + (catch [TypeError] false))) + +(defn neg? [n] + "Return true if n is < 0" + (_numeric-check n) + (< n 0)) + +(defn none? [x] + "Return true if x is None" + (is x None)) + +(defn numeric? [x] + (import numbers) + (instance? numbers.Number x)) + +(defn nth [coll index] + "Return nth item in collection or sequence, counting from 0" + (if (not (neg? index)) + (if (iterable? coll) + (try (first (list (take 1 (drop index coll)))) + (catch [IndexError] None)) + (try (get coll index) + (catch [IndexError] None))) + None)) + +(defn odd? [n] + "Return true if n is an odd number" + (_numeric-check n) + (= (% n 2) 1)) + +(defn pos? [n] + "Return true if n is > 0" + (_numeric_check n) + (> n 0)) + +(defn remove [pred coll] + "Return coll with elements removed that pass `pred`" + (let [[citer (iter coll)]] + (for [val citer] + (if (not (pred val)) + (yield val))))) + +(defn repeat [x &optional n] + "Yield x forever or optionally n times" + (if (none? n) + (setv dispatch (fn [] (while true (yield x)))) + (setv dispatch (fn [] (for [_ (range n)] (yield x))))) + (dispatch)) + +(defn repeatedly [func] + "Yield result of running func repeatedly" + (while true + (yield (func)))) + +(defn take [count coll] + "Take `count` elements from `coll`, or the whole set if the total + number of entries in `coll` is less than `count`." + (let [[citer (iter coll)]] + (for [_ (range count)] + (yield (next citer))))) + +(defn take-nth [n coll] + "Return every nth member of coll + raises ValueError for (not (pos? n))" + (if (pos? n) + (let [[citer (iter coll)] [skip (dec n)]] + (for [val citer] + (yield val) + (for [_ (range skip)] + (next citer)))) + (raise (ValueError "n must be positive")))) + +(defn take-while [pred coll] + "Take all elements while `pred` is true" + (let [[citer (iter coll)]] + (for [val citer] + (if (pred val) + (yield val) + (break))))) + +(defn zero? [n] + "Return true if n is 0" + (_numeric_check n) + (= n 0)) + +(def *exports* ["cycle" "dec" "distinct" "drop" "even?" "filter" "inc" + "instance?" "iterable?" "iterate" "iterator?" "neg?" + "none?" "nth" "numeric?" "odd?" "pos?" "remove" "repeat" + "repeatedly" "take" "take_nth" "take_while" "zero?"]) diff --git a/tests/test_examplefiles.py b/tests/test_examplefiles.py index d785cf3b..c3e3b2f7 100644 --- a/tests/test_examplefiles.py +++ b/tests/test_examplefiles.py @@ -31,10 +31,13 @@ def test_example_files(): absfn = os.path.join(testdir, 'examplefiles', fn) if not os.path.isfile(absfn): continue + + code = open(absfn).read() + outfn = os.path.join(outdir, fn) try: - lx = get_lexer_for_filename(absfn) + lx = get_lexer_for_filename(absfn, code=code) except ClassNotFound: if "_" not in fn: raise AssertionError('file %r has no registered extension, ' |