diff options
| author | Georg Brandl <georg@python.org> | 2013-01-09 13:24:33 +0100 |
|---|---|---|
| committer | Georg Brandl <georg@python.org> | 2013-01-09 13:24:33 +0100 |
| commit | c7baf27b4058f53f8d26be23e45fb3e7772656eb (patch) | |
| tree | cbe61af680e678990e108760272381501a80edbb | |
| parent | d3f74b1897d2cc5887527bf5a3faefcdcc20cb08 (diff) | |
| parent | 07dc045f5ad2b94cb8c99313184e7306a62813ed (diff) | |
| download | pygments-c7baf27b4058f53f8d26be23e45fb3e7772656eb.tar.gz | |
Merged in andyli/pygments-main/ExtendedRegexLexer-tuple-newstate (pull request #131: ExtendedRegexLexer handles tuple new_state the same way as RegexLexer)
114 files changed, 2907 insertions, 1297 deletions
@@ -51,11 +51,13 @@ Other contributors, listed alphabetically, are: * Varun Hiremath -- Debian control lexer * Doug Hogan -- Mscgen lexer * Ben Hollis -- Mason lexer +* Alastair Houghton -- Lexer inheritance facility * Tim Howard -- BlitzMax lexer * Ivan Inozemtsev -- Fantom lexer * Brian R. Jackson -- Tea lexer * Dennis Kaarsemaker -- sources.list lexer * Igor Kalnitsky -- vhdl lexer +* Pekka Klärck -- Robot Framework lexer * Eric Knibbe -- Lasso lexer * Adam Koprowski -- Opa lexer * Benjamin Kowarsch -- Modula-2 lexer @@ -68,15 +70,18 @@ Other contributors, listed alphabetically, are: * Sylvestre Ledru -- Scilab lexer * Mark Lee -- Vala lexer * Ben Mabey -- Gherkin lexer +* Angus MacArthur -- QML lexer * Simone Margaritelli -- Hybris lexer * Kirk McDonald -- D lexer * Gordon McGregor -- SystemVerilog lexer * Stephen McKamey -- Duel/JBST lexer * Brian McKenna -- F# lexer +* Charles McLaughlin -- Puppet lexer * Lukas Meuser -- BBCode formatter, Lua lexer * Paul Miller -- LiveScript lexer * Hong Minhee -- HTTP lexer * Michael Mior -- Awk lexer +* Bruce Mitchener -- Dylan lexer rewrite * Reuben Morais -- SourcePawn lexer * Jon Morton -- Rust lexer * Paulo Moura -- Logtalk lexer @@ -6,7 +6,7 @@ http://bitbucket.org/birkenfeld/pygments-main/issues. Version 1.6 ----------- -(in development, to be released xx November 2012) +(in development, to be released xx January 2013) - Lexers added: @@ -19,12 +19,17 @@ Version 1.6 * Jags (PR#89) * Julia (PR#61) * Kconfig (#711) - * Lasso (PR#95) + * Lasso (PR#95, PR#113) * LiveScript (PR#84) * Monkey (PR#117) * Mscgen (PR#80) + * NSIS scripts (PR#136) + * QML (PR#123) + * Puppet (PR#133) * Racket (PR#94) * Rdoc (PR#99) + * Robot Framework (PR#137) + * RPM spec files (PR#124) * Rust (PR#67) * Smali (Dalvik assembly) * SourcePawn (PR#39) @@ -34,12 +39,26 @@ Version 1.6 * Windows Registry (#819) * Xtend (PR#68) +- When deriving a lexer from another lexer with token definitions, definitions + for states not in the child lexer are now inherited. If you override a state + in the child lexer, an "inherit" keyword has been added to insert the base + state at that position (PR#141) + +- The C family lexers now inherit token definitions from a common base class, + removing code duplication (PR#141) + +- Use "colorama" on Windows for console color output (PR#142) + - Fix Template Haskell highlighting (PR#63) - Fix some S/R lexer errors (PR#91) - Fix a bug in the Prolog lexer with names that start with 'is' (#810) +- Rewrite Dylan lexer, add Dylan LID lexer (PR#147) + +- Add a Java quickstart document (PR#146) + Version 1.5 ----------- @@ -1,4 +1,4 @@ -Copyright (c) 2006-2012 by the respective authors (see AUTHORS file). +Copyright (c) 2006-2013 by the respective authors (see AUTHORS file). All rights reserved. Redistribution and use in source and binary forms, with or without @@ -4,7 +4,7 @@ # # Combines scripts for common tasks. # -# :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. +# :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. # :license: BSD, see LICENSE for details. # diff --git a/docs/generate.py b/docs/generate.py index 0ceeb58d..f5405074 100755 --- a/docs/generate.py +++ b/docs/generate.py @@ -6,7 +6,7 @@ Generates a bunch of html files containing the documentation. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/docs/src/api.txt b/docs/src/api.txt index b8159379..4276eea2 100644 --- a/docs/src/api.txt +++ b/docs/src/api.txt @@ -64,7 +64,7 @@ def `guess_lexer(text, **options):` def `guess_lexer_for_filename(filename, text, **options):` As `guess_lexer()`, but only lexers which have a pattern in `filenames` or `alias_filenames` that matches `filename` are taken into consideration. - + `pygments.util.ClassNotFound` is raised if no lexer thinks it can handle the content. diff --git a/docs/src/index.txt b/docs/src/index.txt index 1fad0f03..b1e099c7 100644 --- a/docs/src/index.txt +++ b/docs/src/index.txt @@ -37,7 +37,7 @@ Welcome to the Pygments documentation. - `Write your own lexer <lexerdevelopment.txt>`_ - `Write your own formatter <formatterdevelopment.txt>`_ - + - `Write your own filter <filterdevelopment.txt>`_ - `Register plugins <plugins.txt>`_ diff --git a/docs/src/integrate.txt b/docs/src/integrate.txt index 51a3dac4..6f8c1253 100644 --- a/docs/src/integrate.txt +++ b/docs/src/integrate.txt @@ -41,3 +41,8 @@ Bash completion The source distribution contains a file ``external/pygments.bashcomp`` that sets up completion for the ``pygmentize`` command in bash. + +Java +---- + +See the `Java quickstart <java.txt>`_ document. diff --git a/docs/src/java.txt b/docs/src/java.txt new file mode 100644 index 00000000..f40a52d2 --- /dev/null +++ b/docs/src/java.txt @@ -0,0 +1,70 @@ +===================== +Use Pygments in Java +===================== + +Thanks to `Jython <http://www.jython.org>`__ it is possible to use Pygments in +Java. + +This page is a simple tutorial to get an idea of how this is working. You can +then look at the `Jython documentation <http://www.jython.org/docs/>`__ for more +advanced use. + +Since version 1.5, Pygments is deployed on `Maven Central +<http://repo1.maven.org/maven2/org/pygments/pygments/>`__ as a JAR so is Jython +which make it a lot easier to create the Java project. + +Here is an example of a `Maven <http://www.maven.org>`__ ``pom.xml`` file for a +project running Pygments: + +.. sourcecode:: xml + + <?xml version="1.0" encoding="UTF-8"?> + + <project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <groupId>example</groupId> + <artifactId>example</artifactId> + <version>1.0-SNAPSHOT</version> + <dependencies> + <dependency> + <groupId>org.python</groupId> + <artifactId>jython-standalone</artifactId> + <version>2.5.3</version> + </dependency> + <dependency> + <groupId>org.pygments</groupId> + <artifactId>pygments</artifactId> + <version>1.5</version> + <scope>runtime</scope> + </dependency> + </dependencies> + </project> + +The following Java example: + +.. sourcecode:: java + + PythonInterpreter interpreter = new PythonInterpreter(); + + // Set a variable with the content you want to work with + interpreter.set("code", code); + + // Simple use Pygments as you would in Python + interpreter.exec("from pygments import highlight\n" + + "from pygments.lexers import PythonLexer\n" + + "from pygments.formatters import HtmlFormatter\n" + + "\nresult = highlight(code, PythonLexer(), HtmlFormatter())"); + + // Get the result that has been set in a variable + System.out.println(interpreter.get("result", String.class)); + +will print something like: + +.. sourcecode:: html + + <div class="highlight"> + <pre><span class="k">print</span> <span class="s">"Hello World"</span></pre> + </div> diff --git a/external/markdown-processor.py b/external/markdown-processor.py index 0d939b67..12e64680 100644 --- a/external/markdown-processor.py +++ b/external/markdown-processor.py @@ -27,7 +27,7 @@ .. _Markdown: http://www.freewisdom.org/projects/python-markdown/ - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -64,4 +64,4 @@ class CodeBlockPreprocessor(TextPreprocessor): code = code.replace('\n\n', '\n \n').replace('\n', '<br />') return '\n\n<div class="code">%s</div>\n\n' % code return self.pattern.sub( - repl, lines)
\ No newline at end of file + repl, lines) diff --git a/external/moin-parser.py b/external/moin-parser.py index 0bb9791c..6544da1b 100644 --- a/external/moin-parser.py +++ b/external/moin-parser.py @@ -31,7 +31,7 @@ If you do not want to do that and are willing to accept larger HTML output, you can set the INLINESTYLES option below to True. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/external/rst-directive-old.py b/external/rst-directive-old.py index 8e703faa..a074536f 100644 --- a/external/rst-directive-old.py +++ b/external/rst-directive-old.py @@ -31,7 +31,7 @@ .. _directive documentation: http://docutils.sourceforge.net/docs/howto/rst-directives.html - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/external/rst-directive.py b/external/rst-directive.py index b54b03ef..5c04038d 100644 --- a/external/rst-directive.py +++ b/external/rst-directive.py @@ -31,7 +31,7 @@ .. _directive documentation: http://docutils.sourceforge.net/docs/howto/rst-directives.html - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/__init__.py b/pygments/__init__.py index 0ecc167d..cd7bea3a 100644 --- a/pygments/__init__.py +++ b/pygments/__init__.py @@ -22,7 +22,7 @@ .. _Pygments tip: http://bitbucket.org/birkenfeld/pygments-main/get/tip.zip#egg=Pygments-dev - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/cmdline.py b/pygments/cmdline.py index 1f14cf5d..3bf081d4 100644 --- a/pygments/cmdline.py +++ b/pygments/cmdline.py @@ -5,7 +5,7 @@ Command line interface. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import sys @@ -192,6 +192,14 @@ def main(args=sys.argv): usage = USAGE % ((args[0],) * 6) + if sys.platform in ['win32', 'cygwin']: + try: + # Provide coloring under Windows, if possible + import colorama + colorama.init() + except ImportError: + pass + try: popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:hVHg") except getopt.GetoptError, err: @@ -219,7 +227,7 @@ def main(args=sys.argv): return 0 if opts.pop('-V', None) is not None: - print 'Pygments version %s, (c) 2006-2012 by Georg Brandl.' % __version__ + print 'Pygments version %s, (c) 2006-2013 by Georg Brandl.' % __version__ return 0 # handle ``pygmentize -L`` diff --git a/pygments/console.py b/pygments/console.py index 3901bd18..c8dfbd1f 100644 --- a/pygments/console.py +++ b/pygments/console.py @@ -5,7 +5,7 @@ Format colored console output. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/filter.py b/pygments/filter.py index de1bf79c..0b9224f2 100644 --- a/pygments/filter.py +++ b/pygments/filter.py @@ -5,7 +5,7 @@ Module that implements the default filter. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/filters/__init__.py b/pygments/filters/__init__.py index 93e97563..f12d025c 100644 --- a/pygments/filters/__init__.py +++ b/pygments/filters/__init__.py @@ -6,7 +6,7 @@ Module containing filter lookup functions and default filters. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatter.py b/pygments/formatter.py index 73124846..4b69f2a4 100644 --- a/pygments/formatter.py +++ b/pygments/formatter.py @@ -5,7 +5,7 @@ Base formatter class. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/__init__.py b/pygments/formatters/__init__.py index da0fd51e..d842b96b 100644 --- a/pygments/formatters/__init__.py +++ b/pygments/formatters/__init__.py @@ -5,7 +5,7 @@ Pygments formatters. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import os.path diff --git a/pygments/formatters/_mapping.py b/pygments/formatters/_mapping.py index 9d3d7823..a423ba50 100755 --- a/pygments/formatters/_mapping.py +++ b/pygments/formatters/_mapping.py @@ -9,7 +9,7 @@ Do not alter the FORMATTERS dictionary by hand. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/bbcode.py b/pygments/formatters/bbcode.py index a0d50553..15faff6c 100644 --- a/pygments/formatters/bbcode.py +++ b/pygments/formatters/bbcode.py @@ -5,7 +5,7 @@ BBcode formatter. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/html.py b/pygments/formatters/html.py index 85a16ff8..075ad022 100644 --- a/pygments/formatters/html.py +++ b/pygments/formatters/html.py @@ -5,7 +5,7 @@ Formatter for HTML output. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -596,7 +596,8 @@ class HtmlFormatter(Formatter): def _wrap_lineanchors(self, inner): s = self.lineanchors - i = self.linenostart - 1 # subtract 1 since we have to increment i *before* yielding + i = self.linenostart - 1 # subtract 1 since we have to increment i + # *before* yielding for t, line in inner: if t: i += 1 diff --git a/pygments/formatters/img.py b/pygments/formatters/img.py index 609aa54c..394c3b6a 100644 --- a/pygments/formatters/img.py +++ b/pygments/formatters/img.py @@ -5,7 +5,7 @@ Formatter for Pixmap output. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/latex.py b/pygments/formatters/latex.py index fcd5f379..47fd1239 100644 --- a/pygments/formatters/latex.py +++ b/pygments/formatters/latex.py @@ -5,7 +5,7 @@ Formatter for LaTeX fancyvrb output. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/other.py b/pygments/formatters/other.py index 4b6d4c38..1029a7a7 100644 --- a/pygments/formatters/other.py +++ b/pygments/formatters/other.py @@ -5,7 +5,7 @@ Other formatters: NullFormatter, RawTokenFormatter. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/rtf.py b/pygments/formatters/rtf.py index 5627d070..3efda284 100644 --- a/pygments/formatters/rtf.py +++ b/pygments/formatters/rtf.py @@ -5,7 +5,7 @@ A formatter that generates RTF files. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/svg.py b/pygments/formatters/svg.py index f08e6570..271f22a7 100644 --- a/pygments/formatters/svg.py +++ b/pygments/formatters/svg.py @@ -5,7 +5,7 @@ Formatter for SVG output. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/terminal.py b/pygments/formatters/terminal.py index dae00157..94e078f2 100644 --- a/pygments/formatters/terminal.py +++ b/pygments/formatters/terminal.py @@ -5,7 +5,7 @@ Formatter for terminal output with ANSI sequences. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/formatters/terminal256.py b/pygments/formatters/terminal256.py index cbd30be1..772ed423 100644 --- a/pygments/formatters/terminal256.py +++ b/pygments/formatters/terminal256.py @@ -11,7 +11,7 @@ Formatter version 1. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexer.py b/pygments/lexer.py index 134471f6..b2af789b 100644 --- a/pygments/lexer.py +++ b/pygments/lexer.py @@ -5,10 +5,10 @@ Base lexer classes. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -import re +import re, itertools from pygments.filter import apply_filters, Filter from pygments.filters import get_filter_by_name @@ -18,7 +18,7 @@ from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \ __all__ = ['Lexer', 'RegexLexer', 'ExtendedRegexLexer', 'DelegatingLexer', - 'LexerContext', 'include', 'bygroups', 'using', 'this'] + 'LexerContext', 'include', 'inherit', 'bygroups', 'using', 'this'] _encoding_map = [('\xef\xbb\xbf', 'utf-8'), @@ -72,15 +72,18 @@ class Lexer(object): #: Shortcuts for the lexer aliases = [] - #: fn match rules + #: File name globs filenames = [] - #: fn alias filenames + #: Secondary file name globs alias_filenames = [] - #: mime types + #: MIME types mimetypes = [] + #: Priority, should multiple lexers match and no content is provided + priority = 0 + __metaclass__ = LexerMeta def __init__(self, **options): @@ -163,6 +166,10 @@ class Lexer(object): text = decoded else: text = text.decode(self.encoding) + else: + if text.startswith(u'\ufeff'): + text = text[len(u'\ufeff'):] + # text now *is* a unicode string text = text.replace('\r\n', '\n') text = text.replace('\r', '\n') @@ -238,6 +245,16 @@ class include(str): pass +class _inherit(object): + """ + Indicates the a state should inherit from its superclass. + """ + def __repr__(self): + return 'inherit' + +inherit = _inherit() + + class combined(tuple): """ Indicates a state combined from multiple states. @@ -428,6 +445,9 @@ class RegexLexerMeta(LexerMeta): tokens.extend(cls._process_state(unprocessed, processed, str(tdef))) continue + if isinstance(tdef, _inherit): + # processed already + continue assert type(tdef) is tuple, "wrong rule def %r" % tdef @@ -456,6 +476,49 @@ class RegexLexerMeta(LexerMeta): cls._process_state(tokendefs, processed, state) return processed + def get_tokendefs(cls): + """ + Merge tokens from superclasses in MRO order, returning a single tokendef + dictionary. + + Any state that is not defined by a subclass will be inherited + automatically. States that *are* defined by subclasses will, by + default, override that state in the superclass. If a subclass wishes to + inherit definitions from a superclass, it can use the special value + "inherit", which will cause the superclass' state definition to be + included at that point in the state. + """ + tokens = {} + inheritable = {} + for c in itertools.chain((cls,), cls.__mro__): + toks = c.__dict__.get('tokens', {}) + + for state, items in toks.iteritems(): + curitems = tokens.get(state) + if curitems is None: + tokens[state] = items + try: + inherit_ndx = items.index(inherit) + except ValueError: + continue + inheritable[state] = inherit_ndx + continue + + inherit_ndx = inheritable.pop(state, None) + if inherit_ndx is None: + continue + + # Replace the "inherit" value with the items + curitems[inherit_ndx:inherit_ndx+1] = items + try: + new_inh_ndx = items.index(inherit) + except ValueError: + pass + else: + inheritable[state] = inherit_ndx + new_inh_ndx + + return tokens + def __call__(cls, *args, **kwds): """Instantiate cls after preprocessing its token definitions.""" if '_tokens' not in cls.__dict__: @@ -465,7 +528,7 @@ class RegexLexerMeta(LexerMeta): # don't process yet pass else: - cls._tokens = cls.process_tokendef('', cls.tokens) + cls._tokens = cls.process_tokendef('', cls.get_tokendefs()) return type.__call__(cls, *args, **kwds) @@ -700,4 +763,3 @@ def do_insertions(insertions, tokens): except StopIteration: insleft = False break # not strictly necessary - diff --git a/pygments/lexers/__init__.py b/pygments/lexers/__init__.py index eb8609d4..9af6ce68 100644 --- a/pygments/lexers/__init__.py +++ b/pygments/lexers/__init__.py @@ -5,7 +5,7 @@ Pygments lexers. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -113,7 +113,7 @@ def get_lexer_for_filename(_fn, code=None, **options): # to find lexers which need it overridden. if code: return cls.analyse_text(code) + bonus - return bonus + return cls.priority + bonus if matches: matches.sort(key=get_rating) diff --git a/pygments/lexers/_asybuiltins.py b/pygments/lexers/_asybuiltins.py index d2edace6..108fa199 100644 --- a/pygments/lexers/_asybuiltins.py +++ b/pygments/lexers/_asybuiltins.py @@ -10,7 +10,7 @@ TODO: perl/python script in Asymptote SVN similar to asy-list.pl but only for function and variable names. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_clbuiltins.py b/pygments/lexers/_clbuiltins.py index 3a3b3dfc..59f948ba 100644 --- a/pygments/lexers/_clbuiltins.py +++ b/pygments/lexers/_clbuiltins.py @@ -5,7 +5,7 @@ ANSI Common Lisp builtins. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_lassobuiltins.py b/pygments/lexers/_lassobuiltins.py index e609daf7..08b65f37 100644 --- a/pygments/lexers/_lassobuiltins.py +++ b/pygments/lexers/_lassobuiltins.py @@ -1,9 +1,12 @@ # -*- coding: utf-8 -*- """ pygments.lexers._lassobuiltins - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Built-in Lasso types, traits, and methods. + + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. """ BUILTINS = { diff --git a/pygments/lexers/_luabuiltins.py b/pygments/lexers/_luabuiltins.py index 27b39822..069c44fd 100644 --- a/pygments/lexers/_luabuiltins.py +++ b/pygments/lexers/_luabuiltins.py @@ -9,7 +9,7 @@ Do not edit the MODULES dict by hand. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index c535a889..63e53dd8 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -9,7 +9,7 @@ Do not alter the LEXERS dictionary by hand. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -83,7 +83,8 @@ LEXERS = { 'DjangoLexer': ('pygments.lexers.templates', 'Django/Jinja', ('django', 'jinja'), (), ('application/x-django-templating', 'application/x-jinja')), 'DtdLexer': ('pygments.lexers.web', 'DTD', ('dtd',), ('*.dtd',), ('application/xml-dtd',)), 'DuelLexer': ('pygments.lexers.web', 'Duel', ('duel', 'Duel Engine', 'Duel View', 'JBST', 'jbst', 'JsonML+BST'), ('*.duel', '*.jbst'), ('text/x-duel', 'text/x-jbst')), - 'DylanLexer': ('pygments.lexers.compiled', 'Dylan', ('dylan',), ('*.dylan', '*.dyl'), ('text/x-dylan',)), + 'DylanLexer': ('pygments.lexers.compiled', 'Dylan', ('dylan',), ('*.dylan', '*.dyl', '*.intr'), ('text/x-dylan',)), + 'DylanLidLexer': ('pygments.lexers.compiled', 'DylanLID', ('dylan-lid', 'lid'), ('*.lid', '*.hdp'), ('text/x-dylan-lid',)), 'ECLLexer': ('pygments.lexers.other', 'ECL', ('ecl',), ('*.ecl',), ('application/x-ecl',)), 'ECLexer': ('pygments.lexers.compiled', 'eC', ('ec',), ('*.ec', '*.eh'), ('text/x-echdr', 'text/x-ecsrc')), 'ElixirConsoleLexer': ('pygments.lexers.functional', 'Elixir iex session', ('iex',), (), ('text/x-elixir-shellsession',)), @@ -122,6 +123,7 @@ LEXERS = { 'HtmlPhpLexer': ('pygments.lexers.templates', 'HTML+PHP', ('html+php',), ('*.phtml',), ('application/x-php', 'application/x-httpd-php', 'application/x-httpd-php3', 'application/x-httpd-php4', 'application/x-httpd-php5')), 'HtmlSmartyLexer': ('pygments.lexers.templates', 'HTML+Smarty', ('html+smarty',), (), ('text/html+smarty',)), 'HttpLexer': ('pygments.lexers.text', 'HTTP', ('http',), (), ()), + 'HxmlLexer': ('pygments.lexers.text', 'Hxml', ('haxeml', 'hxml'), ('*.hxml',), ()), 'HybrisLexer': ('pygments.lexers.other', 'Hybris', ('hybris', 'hy'), ('*.hy', '*.hyb'), ('text/x-hybris', 'application/x-hybris')), 'IniLexer': ('pygments.lexers.text', 'INI', ('ini', 'cfg'), ('*.ini', '*.cfg'), ('text/x-ini',)), 'IoLexer': ('pygments.lexers.agile', 'Io', ('io',), ('*.io',), ('text/x-iosrc',)), @@ -180,6 +182,7 @@ LEXERS = { 'MyghtyJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Myghty', ('js+myghty', 'javascript+myghty'), (), ('application/x-javascript+myghty', 'text/x-javascript+myghty', 'text/javascript+mygthy')), 'MyghtyLexer': ('pygments.lexers.templates', 'Myghty', ('myghty',), ('*.myt', 'autodelegate'), ('application/x-myghty',)), 'MyghtyXmlLexer': ('pygments.lexers.templates', 'XML+Myghty', ('xml+myghty',), (), ('application/xml+myghty',)), + 'NSISLexer': ('pygments.lexers.other', 'NSIS', ('nsis', 'nsi', 'nsh'), ('*.nsi', '*.nsh'), ('text/x-nsis',)), 'NasmLexer': ('pygments.lexers.asm', 'NASM', ('nasm',), ('*.asm', '*.ASM'), ('text/x-nasm',)), 'NemerleLexer': ('pygments.lexers.dotnet', 'Nemerle', ('nemerle',), ('*.n',), ('text/x-nemerle',)), 'NewLispLexer': ('pygments.lexers.functional', 'NewLisp', ('newlisp',), ('*.lsp', '*.nl'), ('text/x-newlisp', 'application/x-newlisp')), @@ -188,7 +191,8 @@ LEXERS = { 'NimrodLexer': ('pygments.lexers.compiled', 'Nimrod', ('nimrod', 'nim'), ('*.nim', '*.nimrod'), ('text/x-nimrod',)), 'NumPyLexer': ('pygments.lexers.math', 'NumPy', ('numpy',), (), ()), 'ObjdumpLexer': ('pygments.lexers.asm', 'objdump', ('objdump',), ('*.objdump',), ('text/x-objdump',)), - 'ObjectiveCLexer': ('pygments.lexers.compiled', 'Objective-C', ('objective-c', 'objectivec', 'obj-c', 'objc'), ('*.m',), ('text/x-objective-c',)), + 'ObjectiveCLexer': ('pygments.lexers.compiled', 'Objective-C', ('objective-c', 'objectivec', 'obj-c', 'objc'), ('*.m', '*.h'), ('text/x-objective-c',)), + 'ObjectiveCppLexer': ('pygments.lexers.compiled', 'Objective-C++', ('objective-c++', 'objectivec++', 'obj-c++', 'objc++'), ('*.mm', '*.hh'), ('text/x-objective-c++',)), 'ObjectiveJLexer': ('pygments.lexers.web', 'Objective-J', ('objective-j', 'objectivej', 'obj-j', 'objj'), ('*.j',), ('text/x-objective-j',)), 'OcamlLexer': ('pygments.lexers.functional', 'OCaml', ('ocaml',), ('*.ml', '*.mli', '*.mll', '*.mly'), ('text/x-ocaml',)), 'OctaveLexer': ('pygments.lexers.math', 'Octave', ('octave',), ('*.m',), ('text/octave',)), @@ -206,13 +210,16 @@ LEXERS = { 'PrologLexer': ('pygments.lexers.compiled', 'Prolog', ('prolog',), ('*.prolog', '*.pro', '*.pl'), ('text/x-prolog',)), 'PropertiesLexer': ('pygments.lexers.text', 'Properties', ('properties',), ('*.properties',), ('text/x-java-properties',)), 'ProtoBufLexer': ('pygments.lexers.other', 'Protocol Buffer', ('protobuf',), ('*.proto',), ()), + 'PuppetLexer': ('pygments.lexers.other', 'Puppet', ('puppet',), ('*.pp',), ()), 'PyPyLogLexer': ('pygments.lexers.text', 'PyPy Log', ('pypylog', 'pypy'), ('*.pypylog',), ('application/x-pypylog',)), 'Python3Lexer': ('pygments.lexers.agile', 'Python 3', ('python3', 'py3'), (), ('text/x-python3', 'application/x-python3')), 'Python3TracebackLexer': ('pygments.lexers.agile', 'Python 3.0 Traceback', ('py3tb',), ('*.py3tb',), ('text/x-python3-traceback',)), 'PythonConsoleLexer': ('pygments.lexers.agile', 'Python console session', ('pycon',), (), ('text/x-python-doctest',)), 'PythonLexer': ('pygments.lexers.agile', 'Python', ('python', 'py', 'sage'), ('*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript', '*.tac', '*.sage'), ('text/x-python', 'application/x-python')), 'PythonTracebackLexer': ('pygments.lexers.agile', 'Python Traceback', ('pytb',), ('*.pytb',), ('text/x-python-traceback',)), + 'QmlLexer': ('pygments.lexers.web', 'QML', ('qml', 'Qt Meta Language', 'Qt modeling Language'), ('*.qml',), ('application/x-qml',)), 'RConsoleLexer': ('pygments.lexers.math', 'RConsole', ('rconsole', 'rout'), ('*.Rout',), ()), + 'RPMSpecLexer': ('pygments.lexers.other', 'RPMSpec', ('spec',), ('*.spec',), ('text/x-rpm-spec',)), 'RacketLexer': ('pygments.lexers.functional', 'Racket', ('racket', 'rkt'), ('*.rkt', '*.rktl'), ('text/x-racket', 'application/x-racket')), 'RagelCLexer': ('pygments.lexers.parsers', 'Ragel in C Host', ('ragel-c',), ('*.rl',), ()), 'RagelCppLexer': ('pygments.lexers.parsers', 'Ragel in CPP Host', ('ragel-cpp',), ('*.rl',), ()), @@ -228,6 +235,7 @@ LEXERS = { 'RedcodeLexer': ('pygments.lexers.other', 'Redcode', ('redcode',), ('*.cw',), ()), 'RegeditLexer': ('pygments.lexers.text', 'reg', (), ('*.reg',), ('text/x-windows-registry',)), 'RhtmlLexer': ('pygments.lexers.templates', 'RHTML', ('rhtml', 'html+erb', 'html+ruby'), ('*.rhtml',), ('text/html+ruby',)), + 'RobotFrameworkLexer': ('pygments.lexers.other', 'RobotFramework', ('RobotFramework', 'robotframework'), ('*.txt',), ('text/x-robotframework',)), 'RstLexer': ('pygments.lexers.text', 'reStructuredText', ('rst', 'rest', 'restructuredtext'), ('*.rst', '*.rest'), ('text/x-rst', 'text/prs.fallenstein.rst')), 'RubyConsoleLexer': ('pygments.lexers.agile', 'Ruby irb session', ('rbcon', 'irb'), (), ('text/x-ruby-shellsession',)), 'RubyLexer': ('pygments.lexers.agile', 'Ruby', ('rb', 'ruby', 'duby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby'), ('text/x-ruby', 'application/x-ruby')), @@ -319,4 +327,3 @@ if __name__ == '__main__': f.write('LEXERS = {\n %s,\n}\n\n' % ',\n '.join(found_lexers)) f.write(footer) f.close() - diff --git a/pygments/lexers/_openedgebuiltins.py b/pygments/lexers/_openedgebuiltins.py index 81cc17c9..4561b07b 100644 --- a/pygments/lexers/_openedgebuiltins.py +++ b/pygments/lexers/_openedgebuiltins.py @@ -1,3 +1,14 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers._openedgebuiltins + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Builtin list for the OpenEdgeLexer. + + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + OPENEDGEKEYWORDS = [ 'ABSOLUTE', 'ABS', 'ABSO', 'ABSOL', 'ABSOLU', 'ABSOLUT', 'ACCELERATOR', 'ACCUM', 'ACCUMULATE', 'ACCUM', 'ACCUMU', 'ACCUMUL', 'ACCUMULA', diff --git a/pygments/lexers/_phpbuiltins.py b/pygments/lexers/_phpbuiltins.py index cd1608f1..08eaaf2e 100644 --- a/pygments/lexers/_phpbuiltins.py +++ b/pygments/lexers/_phpbuiltins.py @@ -12,7 +12,7 @@ internet connection. don't run that at home, use a server ;-) - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -3784,4 +3784,4 @@ if __name__ == '__main__': regenerate(__file__, modules) shutil.rmtree(PHP_MANUAL_DIR) - run()
\ No newline at end of file + run() diff --git a/pygments/lexers/_postgres_builtins.py b/pygments/lexers/_postgres_builtins.py index 1f9918fe..b2322137 100644 --- a/pygments/lexers/_postgres_builtins.py +++ b/pygments/lexers/_postgres_builtins.py @@ -1,10 +1,11 @@ +# -*- coding: utf-8 -*- """ pygments.lexers._postgres_builtins ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Self-updating data files for PostgreSQL lexer. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -56,7 +57,8 @@ def parse_datatypes(f): line = re.sub("<[^>]+>", "", line) # Drop the parts containing braces - for tmp in [ t for tmp in line.split('[') for t in tmp.split(']') if "(" not in t ]: + for tmp in [t for tmp in line.split('[') + for t in tmp.split(']') if "(" not in t]: for t in tmp.split(','): t = t.strip() if not t: continue diff --git a/pygments/lexers/_robotframeworklexer.py b/pygments/lexers/_robotframeworklexer.py new file mode 100644 index 00000000..64d832f3 --- /dev/null +++ b/pygments/lexers/_robotframeworklexer.py @@ -0,0 +1,557 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers._robotframeworklexer + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Lexer for Robot Framework. + + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +# Copyright 2012 Nokia Siemens Networks Oyj +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +from pygments.lexer import Lexer +from pygments.token import Token + + +HEADING = Token.Generic.Heading +SETTING = Token.Keyword.Namespace +IMPORT = Token.Name.Namespace +TC_KW_NAME = Token.Generic.Subheading +KEYWORD = Token.Name.Function +ARGUMENT = Token.String +VARIABLE = Token.Name.Variable +COMMENT = Token.Comment +SEPARATOR = Token.Punctuation +SYNTAX = Token.Punctuation +GHERKIN = Token.Generic.Emph +ERROR = Token.Error + + +def normalize(string, remove=''): + string = string.lower() + for char in remove + ' ': + if char in string: + string = string.replace(char, '') + return string + + +class RobotFrameworkLexer(Lexer): + """ + For `Robot Framework <http://robotframework.org>`_ test data. + + Supports both space and pipe separated plain text formats. + + *New in Pygments 1.6.* + """ + name = 'RobotFramework' + aliases = ['RobotFramework', 'robotframework'] + filenames = ['*.txt'] + mimetypes = ['text/x-robotframework'] + + def __init__(self, **options): + options['tabsize'] = 2 + options['encoding'] = 'UTF-8' + Lexer.__init__(self, **options) + + def get_tokens_unprocessed(self, text): + row_tokenizer = RowTokenizer() + var_tokenizer = VariableTokenizer() + index = 0 + for row in text.splitlines(): + for value, token in row_tokenizer.tokenize(row): + for value, token in var_tokenizer.tokenize(value, token): + if value: + yield index, token, unicode(value) + index += len(value) + + +class VariableTokenizer(object): + + def tokenize(self, string, token): + var = VariableSplitter(string, identifiers='$@%') + if var.start < 0 or token in (COMMENT, ERROR): + yield string, token + return + for value, token in self._tokenize(var, string, token): + if value: + yield value, token + + def _tokenize(self, var, string, orig_token): + before = string[:var.start] + yield before, orig_token + yield var.identifier + '{', SYNTAX + for value, token in self.tokenize(var.base, VARIABLE): + yield value, token + yield '}', SYNTAX + if var.index: + yield '[', SYNTAX + for value, token in self.tokenize(var.index, VARIABLE): + yield value, token + yield ']', SYNTAX + for value, token in self.tokenize(string[var.end:], orig_token): + yield value, token + + +class RowTokenizer(object): + + def __init__(self): + self._table = UnknownTable() + self._splitter = RowSplitter() + testcases = TestCaseTable() + settings = SettingTable(testcases.set_default_template) + variables = VariableTable() + keywords = KeywordTable() + self._tables = {'settings': settings, 'setting': settings, + 'metadata': settings, + 'variables': variables, 'variable': variables, + 'testcases': testcases, 'testcase': testcases, + 'keywords': keywords, 'keyword': keywords, + 'userkeywords': keywords, 'userkeyword': keywords} + + def tokenize(self, row): + commented = False + heading = False + for index, value in enumerate(self._splitter.split(row)): + # First value, and every second after that, is a separator. + index, separator = divmod(index-1, 2) + if value.startswith('#'): + commented = True + elif index == 0 and value.startswith('*'): + self._table = self._start_table(value) + heading = True + for value, token in self._tokenize(value, index, commented, + separator, heading): + yield value, token + self._table.end_row() + + def _start_table(self, header): + name = normalize(header, remove='*') + return self._tables.get(name, UnknownTable()) + + def _tokenize(self, value, index, commented, separator, heading): + if commented: + yield value, COMMENT + elif separator: + yield value, SEPARATOR + elif heading: + yield value, HEADING + else: + for value, token in self._table.tokenize(value, index): + yield value, token + + +class RowSplitter(object): + _space_splitter = re.compile('( {2,})') + _pipe_splitter = re.compile('((?:^| +)\|(?: +|$))') + + def split(self, row): + splitter = (row.startswith('| ') and self._split_from_pipes + or self._split_from_spaces) + for value in splitter(row.rstrip()): + yield value + yield '\n' + + def _split_from_spaces(self, row): + yield '' # Start with (pseudo)separator similarly as with pipes + for value in self._space_splitter.split(row): + yield value + + def _split_from_pipes(self, row): + _, separator, rest = self._pipe_splitter.split(row, 1) + yield separator + while self._pipe_splitter.search(rest): + cell, separator, rest = self._pipe_splitter.split(rest, 1) + yield cell + yield separator + yield rest + + +class Tokenizer(object): + _tokens = None + + def __init__(self): + self._index = 0 + + def tokenize(self, value): + values_and_tokens = self._tokenize(value, self._index) + self._index += 1 + if isinstance(values_and_tokens, type(Token)): + values_and_tokens = [(value, values_and_tokens)] + return values_and_tokens + + def _tokenize(self, value, index): + index = min(index, len(self._tokens) - 1) + return self._tokens[index] + + def _is_assign(self, value): + if value.endswith('='): + value = value[:-1].strip() + var = VariableSplitter(value, identifiers='$@') + return var.start == 0 and var.end == len(value) + + +class Comment(Tokenizer): + _tokens = (COMMENT,) + + +class Setting(Tokenizer): + _tokens = (SETTING, ARGUMENT) + _keyword_settings = ('suitesetup', 'suiteprecondition', 'suiteteardown', + 'suitepostcondition', 'testsetup', 'testprecondition', + 'testteardown', 'testpostcondition', 'testtemplate') + _import_settings = ('library', 'resource', 'variables') + _other_settings = ('documentation', 'metadata', 'forcetags', 'defaulttags', + 'testtimeout') + _custom_tokenizer = None + + def __init__(self, template_setter=None): + Tokenizer.__init__(self) + self._template_setter = template_setter + + def _tokenize(self, value, index): + if index == 1 and self._template_setter: + self._template_setter(value) + if index == 0: + normalized = normalize(value) + if normalized in self._keyword_settings: + self._custom_tokenizer = KeywordCall(support_assign=False) + elif normalized in self._import_settings: + self._custom_tokenizer = ImportSetting() + elif normalized not in self._other_settings: + return ERROR + elif self._custom_tokenizer: + return self._custom_tokenizer.tokenize(value) + return Tokenizer._tokenize(self, value, index) + + +class ImportSetting(Tokenizer): + _tokens = (IMPORT, ARGUMENT) + + +class TestCaseSetting(Setting): + _keyword_settings = ('setup', 'precondition', 'teardown', 'postcondition', + 'template') + _import_settings = () + _other_settings = ('documentation', 'tags', 'timeout') + + def _tokenize(self, value, index): + if index == 0: + type = Setting._tokenize(self, value[1:-1], index) + return [('[', SYNTAX), (value[1:-1], type), (']', SYNTAX)] + return Setting._tokenize(self, value, index) + + +class KeywordSetting(TestCaseSetting): + _keyword_settings = ('teardown',) + _other_settings = ('documentation', 'arguments', 'return', 'timeout') + + +class Variable(Tokenizer): + _tokens = (SYNTAX, ARGUMENT) + + def _tokenize(self, value, index): + if index == 0 and not self._is_assign(value): + return ERROR + return Tokenizer._tokenize(self, value, index) + + +class KeywordCall(Tokenizer): + _tokens = (KEYWORD, ARGUMENT) + + def __init__(self, support_assign=True): + Tokenizer.__init__(self) + self._keyword_found = not support_assign + self._assigns = 0 + + def _tokenize(self, value, index): + if not self._keyword_found and self._is_assign(value): + self._assigns += 1 + return SYNTAX # VariableTokenizer tokenizes this later. + if self._keyword_found: + return Tokenizer._tokenize(self, value, index - self._assigns) + self._keyword_found = True + return GherkinTokenizer().tokenize(value, KEYWORD) + + +class GherkinTokenizer(object): + _gherkin_prefix = re.compile('^(Given|When|Then|And) ', re.IGNORECASE) + + def tokenize(self, value, token): + match = self._gherkin_prefix.match(value) + if not match: + return [(value, token)] + end = match.end() + return [(value[:end], GHERKIN), (value[end:], token)] + + +class TemplatedKeywordCall(Tokenizer): + _tokens = (ARGUMENT,) + + +class ForLoop(Tokenizer): + + def __init__(self): + Tokenizer.__init__(self) + self._in_arguments = False + + def _tokenize(self, value, index): + token = self._in_arguments and ARGUMENT or SYNTAX + if value.upper() in ('IN', 'IN RANGE'): + self._in_arguments = True + return token + + +class _Table(object): + _tokenizer_class = None + + def __init__(self, prev_tokenizer=None): + self._tokenizer = self._tokenizer_class() + self._prev_tokenizer = prev_tokenizer + self._prev_values_on_row = [] + + def tokenize(self, value, index): + if self._continues(value, index): + self._tokenizer = self._prev_tokenizer + yield value, SYNTAX + else: + for value_and_token in self._tokenize(value, index): + yield value_and_token + self._prev_values_on_row.append(value) + + def _continues(self, value, index): + return value == '...' and all(self._is_empty(t) + for t in self._prev_values_on_row) + + def _is_empty(self, value): + return value in ('', '\\') + + def _tokenize(self, value, index): + return self._tokenizer.tokenize(value) + + def end_row(self): + self.__init__(prev_tokenizer=self._tokenizer) + + +class UnknownTable(_Table): + _tokenizer_class = Comment + + def _continues(self, value, index): + return False + + +class VariableTable(_Table): + _tokenizer_class = Variable + + +class SettingTable(_Table): + _tokenizer_class = Setting + + def __init__(self, template_setter, prev_tokenizer=None): + _Table.__init__(self, prev_tokenizer) + self._template_setter = template_setter + + def _tokenize(self, value, index): + if index == 0 and normalize(value) == 'testtemplate': + self._tokenizer = Setting(self._template_setter) + return _Table._tokenize(self, value, index) + + def end_row(self): + self.__init__(self._template_setter, prev_tokenizer=self._tokenizer) + + +class TestCaseTable(_Table): + _setting_class = TestCaseSetting + _test_template = None + _default_template = None + + @property + def _tokenizer_class(self): + if self._test_template or (self._default_template and + self._test_template is not False): + return TemplatedKeywordCall + return KeywordCall + + def _continues(self, value, index): + return index > 0 and _Table._continues(self, value, index) + + def _tokenize(self, value, index): + if index == 0: + if value: + self._test_template = None + return GherkinTokenizer().tokenize(value, TC_KW_NAME) + if index == 1 and self._is_setting(value): + if self._is_template(value): + self._test_template = False + self._tokenizer = self._setting_class(self.set_test_template) + else: + self._tokenizer = self._setting_class() + if index == 1 and self._is_for_loop(value): + self._tokenizer = ForLoop() + if index == 1 and self._is_empty(value): + return [(value, SYNTAX)] + return _Table._tokenize(self, value, index) + + def _is_setting(self, value): + return value.startswith('[') and value.endswith(']') + + def _is_template(self, value): + return normalize(value) == '[template]' + + def _is_for_loop(self, value): + return value.startswith(':') and normalize(value, remove=':') == 'for' + + def set_test_template(self, template): + self._test_template = self._is_template_set(template) + + def set_default_template(self, template): + self._default_template = self._is_template_set(template) + + def _is_template_set(self, template): + return normalize(template) not in ('', '\\', 'none', '${empty}') + + +class KeywordTable(TestCaseTable): + _tokenizer_class = KeywordCall + _setting_class = KeywordSetting + + def _is_template(self, value): + return False + + +# Following code copied directly from Robot Framework 2.7.5. + +class VariableSplitter: + + def __init__(self, string, identifiers): + self.identifier = None + self.base = None + self.index = None + self.start = -1 + self.end = -1 + self._identifiers = identifiers + self._may_have_internal_variables = False + try: + self._split(string) + except ValueError: + pass + else: + self._finalize() + + def get_replaced_base(self, variables): + if self._may_have_internal_variables: + return variables.replace_string(self.base) + return self.base + + def _finalize(self): + self.identifier = self._variable_chars[0] + self.base = ''.join(self._variable_chars[2:-1]) + self.end = self.start + len(self._variable_chars) + if self._has_list_variable_index(): + self.index = ''.join(self._list_variable_index_chars[1:-1]) + self.end += len(self._list_variable_index_chars) + + def _has_list_variable_index(self): + return self._list_variable_index_chars\ + and self._list_variable_index_chars[-1] == ']' + + def _split(self, string): + start_index, max_index = self._find_variable(string) + self.start = start_index + self._open_curly = 1 + self._state = self._variable_state + self._variable_chars = [string[start_index], '{'] + self._list_variable_index_chars = [] + self._string = string + start_index += 2 + for index, char in enumerate(string[start_index:]): + index += start_index # Giving start to enumerate only in Py 2.6+ + try: + self._state(char, index) + except StopIteration: + return + if index == max_index and not self._scanning_list_variable_index(): + return + + def _scanning_list_variable_index(self): + return self._state in [self._waiting_list_variable_index_state, + self._list_variable_index_state] + + def _find_variable(self, string): + max_end_index = string.rfind('}') + if max_end_index == -1: + return ValueError('No variable end found') + if self._is_escaped(string, max_end_index): + return self._find_variable(string[:max_end_index]) + start_index = self._find_start_index(string, 1, max_end_index) + if start_index == -1: + return ValueError('No variable start found') + return start_index, max_end_index + + def _find_start_index(self, string, start, end): + index = string.find('{', start, end) - 1 + if index < 0: + return -1 + if self._start_index_is_ok(string, index): + return index + return self._find_start_index(string, index+2, end) + + def _start_index_is_ok(self, string, index): + return string[index] in self._identifiers\ + and not self._is_escaped(string, index) + + def _is_escaped(self, string, index): + escaped = False + while index > 0 and string[index-1] == '\\': + index -= 1 + escaped = not escaped + return escaped + + def _variable_state(self, char, index): + self._variable_chars.append(char) + if char == '}' and not self._is_escaped(self._string, index): + self._open_curly -= 1 + if self._open_curly == 0: + if not self._is_list_variable(): + raise StopIteration + self._state = self._waiting_list_variable_index_state + elif char in self._identifiers: + self._state = self._internal_variable_start_state + + def _is_list_variable(self): + return self._variable_chars[0] == '@' + + def _internal_variable_start_state(self, char, index): + self._state = self._variable_state + if char == '{': + self._variable_chars.append(char) + self._open_curly += 1 + self._may_have_internal_variables = True + else: + self._variable_state(char, index) + + def _waiting_list_variable_index_state(self, char, index): + if char != '[': + raise StopIteration + self._list_variable_index_chars.append(char) + self._state = self._list_variable_index_state + + def _list_variable_index_state(self, char, index): + self._list_variable_index_chars.append(char) + if char == ']': + raise StopIteration diff --git a/pygments/lexers/_scilab_builtins.py b/pygments/lexers/_scilab_builtins.py index 7e5530d2..ed0dc819 100644 --- a/pygments/lexers/_scilab_builtins.py +++ b/pygments/lexers/_scilab_builtins.py @@ -1,3 +1,14 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers._scilab_builtins + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Builtin list for the ScilabLexer. + + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + # These lists are generated automatically. # Run the following in a Scilab script: # diff --git a/pygments/lexers/_sourcemodbuiltins.py b/pygments/lexers/_sourcemodbuiltins.py index 19c783a0..0f6b4770 100644 --- a/pygments/lexers/_sourcemodbuiltins.py +++ b/pygments/lexers/_sourcemodbuiltins.py @@ -1,14 +1,14 @@ # -*- coding: utf-8 -*- """ pygments.lexers._sourcemodbuiltins - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This file contains the names of SourceMod functions. It is able to re-generate itself. Do not edit the FUNCTIONS list by hand. - :copyright: Copyright 2006-2011 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/_stan_builtins.py b/pygments/lexers/_stan_builtins.py index 7ce82e53..69d8ce75 100644 --- a/pygments/lexers/_stan_builtins.py +++ b/pygments/lexers/_stan_builtins.py @@ -1,13 +1,13 @@ # -*- coding: utf-8 -*- """ -pygments.lexers._stan_builtins -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + pygments.lexers._stan_builtins + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This file contains the names of functions for Stan used by -``pygments.lexers.math.StanLexer. + This file contains the names of functions for Stan used by + ``pygments.lexers.math.StanLexer. -:copyright: Copyright 2012 by the Pygments team, see AUTHORS. -:license: BSD, see LICENSE for details. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. """ CONSTANTS=[ 'e', diff --git a/pygments/lexers/agile.py b/pygments/lexers/agile.py index 223873de..f65046c6 100644 --- a/pygments/lexers/agile.py +++ b/pygments/lexers/agile.py @@ -5,7 +5,7 @@ Lexers for agile languages. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -1224,7 +1224,8 @@ class CrocLexer(RegexLexer): r'|this|throw|try|vararg|while|with|yield)\b', Keyword), (r'(false|true|null)\b', Keyword.Constant), # FloatLiteral - (r'([0-9][0-9_]*)(?=[.eE])(\.[0-9][0-9_]*)?([eE][+\-]?[0-9_]+)?', Number.Float), + (r'([0-9][0-9_]*)(?=[.eE])(\.[0-9][0-9_]*)?([eE][+\-]?[0-9_]+)?', + Number.Float), # IntegerLiteral # -- Binary (r'0[bB][01][01_]*', Number), diff --git a/pygments/lexers/asm.py b/pygments/lexers/asm.py index c1d46bcb..01ce71ab 100644 --- a/pygments/lexers/asm.py +++ b/pygments/lexers/asm.py @@ -5,7 +5,7 @@ Lexers for assembly languages. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -240,8 +240,8 @@ class LlvmLexer(RegexLexer): r'|linkonce_odr|weak|weak_odr|appending|dllimport|dllexport' r'|common|default|hidden|protected|extern_weak|external' r'|thread_local|zeroinitializer|undef|null|to|tail|target|triple' - r'|deplibs|datalayout|volatile|nuw|nsw|exact|inbounds|align' - r'|addrspace|section|alias|module|asm|sideeffect|gc|dbg' + r'|datalayout|volatile|nuw|nsw|nnan|ninf|nsz|arcp|fast|exact|inbounds' + r'|align|addrspace|section|alias|module|asm|sideeffect|gc|dbg' r'|ccc|fastcc|coldcc|x86_stdcallcc|x86_fastcallcc|arm_apcscc' r'|arm_aapcscc|arm_aapcs_vfpcc' diff --git a/pygments/lexers/compiled.py b/pygments/lexers/compiled.py index 66b782e5..a2536951 100644 --- a/pygments/lexers/compiled.py +++ b/pygments/lexers/compiled.py @@ -5,7 +5,7 @@ Lexers for compiled languages. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -13,7 +13,7 @@ import re from string import Template from pygments.lexer import Lexer, RegexLexer, include, bygroups, using, \ - this, combined + this, combined, inherit from pygments.util import get_bool_opt, get_list_opt from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ Number, Punctuation, Error, Literal @@ -23,21 +23,19 @@ from pygments.scanner import Scanner from pygments.lexers.functional import OcamlLexer from pygments.lexers.jvm import JavaLexer, ScalaLexer -__all__ = ['CLexer', 'CppLexer', 'DLexer', 'DelphiLexer', 'ECLexer', - 'DylanLexer', 'ObjectiveCLexer', 'FortranLexer', 'GLShaderLexer', +__all__ = ['CLexer', 'CppLexer', 'DLexer', 'DelphiLexer', 'ECLexer', 'DylanLexer', + 'ObjectiveCLexer', 'ObjectiveCppLexer', 'FortranLexer', 'GLShaderLexer', 'PrologLexer', 'CythonLexer', 'ValaLexer', 'OocLexer', 'GoLexer', 'FelixLexer', 'AdaLexer', 'Modula2Lexer', 'BlitzMaxLexer', - 'NimrodLexer', 'FantomLexer', 'RustLexer', 'CudaLexer', 'MonkeyLexer'] + 'NimrodLexer', 'FantomLexer', 'RustLexer', 'CudaLexer', 'MonkeyLexer', + 'DylanLidLexer'] -class CLexer(RegexLexer): +class CFamilyLexer(RegexLexer): """ - For C source code with preprocessor directives. + For C family source code. This is used as a base class to avoid repetitious + definitions. """ - name = 'C' - aliases = ['c'] - filenames = ['*.c', '*.h', '*.idc'] - mimetypes = ['text/x-chdr', 'text/x-csrc'] #: optional Comment or Whitespace _ws = r'(?:\s|//.*?\n|/[*].*?[*]/)+' @@ -76,12 +74,17 @@ class CLexer(RegexLexer): (r'\b(case)(.+?)(:)', bygroups(Keyword, using(this), Text)), (r'(auto|break|case|const|continue|default|do|else|enum|extern|' r'for|goto|if|register|restricted|return|sizeof|static|struct|' - r'switch|typedef|union|volatile|virtual|while)\b', Keyword), - (r'(int|long|float|short|double|char|unsigned|signed|void)\b', + r'switch|typedef|union|volatile|while)\b', Keyword), + (r'(bool|int|long|float|short|double|char|unsigned|signed|void|' + r'[a-z_][a-z0-9_]*_t)\b', Keyword.Type), (r'(_{0,2}inline|naked|restrict|thread|typename)\b', Keyword.Reserved), + # Vector intrinsics + (r'(__(m128i|m128d|m128|m64))\b', Keyword.Reserved), + # Microsoft-isms (r'__(asm|int8|based|except|int16|stdcall|cdecl|fastcall|int32|' - r'declspec|finally|int64|try|leave)\b', Keyword.Reserved), + r'declspec|finally|int64|try|leave|wchar_t|w64|unaligned|' + r'raise|noop|identifier|forceinline|assume)\b', Keyword.Reserved), (r'(true|false|NULL)\b', Name.Builtin), ('[a-zA-Z_][a-zA-Z0-9_]*', Name), ], @@ -119,7 +122,8 @@ class CLexer(RegexLexer): ], 'string': [ (r'"', String, '#pop'), - (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape), + (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|' + r'u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|[0-7]{1,3})', String.Escape), (r'[^\\"\n]+', String), # all other characters (r'\\\n', String), # line continuation (r'\\', String), # stray backslash @@ -141,16 +145,17 @@ class CLexer(RegexLexer): } stdlib_types = ['size_t', 'ssize_t', 'off_t', 'wchar_t', 'ptrdiff_t', - 'sig_atomic_t', 'fpos_t', 'clock_t', 'time_t', 'va_list', - 'jmp_buf', 'FILE', 'DIR', 'div_t', 'ldiv_t', 'mbstate_t', - 'wctrans_t', 'wint_t', 'wctype_t'] + 'sig_atomic_t', 'fpos_t', 'clock_t', 'time_t', 'va_list', + 'jmp_buf', 'FILE', 'DIR', 'div_t', 'ldiv_t', 'mbstate_t', + 'wctrans_t', 'wint_t', 'wctype_t'] c99_types = ['_Bool', '_Complex', 'int8_t', 'int16_t', 'int32_t', 'int64_t', - 'uint8_t', 'uint16_t', 'uint32_t', 'uint64_t', 'int_least8_t', - 'int_least16_t', 'int_least32_t', 'int_least64_t', - 'uint_least8_t', 'uint_least16_t', 'uint_least32_t', - 'uint_least64_t', 'int_fast8_t', 'int_fast16_t', 'int_fast32_t', - 'int_fast64_t', 'uint_fast8_t', 'uint_fast16_t', 'uint_fast32_t', - 'uint_fast64_t', 'intptr_t', 'uintptr_t', 'intmax_t', 'uintmax_t'] + 'uint8_t', 'uint16_t', 'uint32_t', 'uint64_t', 'int_least8_t', + 'int_least16_t', 'int_least32_t', 'int_least64_t', + 'uint_least8_t', 'uint_least16_t', 'uint_least32_t', + 'uint_least64_t', 'int_fast8_t', 'int_fast16_t', 'int_fast32_t', + 'int_fast64_t', 'uint_fast8_t', 'uint_fast16_t', 'uint_fast32_t', + 'uint_fast64_t', 'intptr_t', 'uintptr_t', 'intmax_t', + 'uintmax_t'] def __init__(self, **options): self.stdlibhighlighting = get_bool_opt(options, @@ -169,7 +174,22 @@ class CLexer(RegexLexer): token = Keyword.Type yield index, token, value -class CppLexer(RegexLexer): + +class CLexer(CFamilyLexer): + """ + For C source code with preprocessor directives. + """ + name = 'C' + aliases = ['c'] + filenames = ['*.c', '*.h', '*.idc'] + mimetypes = ['text/x-chdr', 'text/x-csrc'] + priority = 0.1 + + def analyse_text(text): + return 0.1 + + +class CppLexer(CFamilyLexer): """ For C++ source code with preprocessor directives. """ @@ -179,90 +199,38 @@ class CppLexer(RegexLexer): '*.cc', '*.hh', '*.cxx', '*.hxx', '*.C', '*.H', '*.cp', '*.CPP'] mimetypes = ['text/x-c++hdr', 'text/x-c++src'] - - #: optional Comment or Whitespace - _ws = r'(?:\s|//.*?\n|/[*].*?[*]/)+' - #: only one /* */ style comment - _ws1 = r':\s*/[*].*?[*]/\s*' + priority = 0.1 tokens = { - 'root': [ - # preprocessor directives: without whitespace - ('^#if\s+0', Comment.Preproc, 'if0'), - ('^#', Comment.Preproc, 'macro'), - # or with whitespace - ('^(' + _ws1 + r')(#if\s+0)', - bygroups(using(this), Comment.Preproc), 'if0'), - ('^(' + _ws1 + ')(#)', - bygroups(using(this), Comment.Preproc), 'macro'), - (r'\n', Text), - (r'\s+', Text), - (r'\\\n', Text), # line continuation - (r'/(\\\n)?/(\n|(.|\n)*?[^\\]\n)', Comment.Single), - (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline), - (r'[{}]', Punctuation), - (r'L?"', String, 'string'), - (r"L?'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char), - (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*', Number.Float), - (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float), - (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex), - (r'0[0-7]+[LlUu]*', Number.Oct), - (r'\d+[LlUu]*', Number.Integer), - (r'\*/', Error), - (r'[~!%^&*+=|?:<>/-]', Operator), - (r'[()\[\],.;]', Punctuation), - (r'(asm|auto|break|case|catch|const|const_cast|continue|' - r'default|delete|do|dynamic_cast|else|enum|explicit|export|' - r'extern|for|friend|goto|if|mutable|namespace|new|operator|' - r'private|protected|public|register|reinterpret_cast|return|' - r'restrict|sizeof|static|static_cast|struct|switch|template|' - r'this|throw|throws|try|typedef|typeid|typename|union|using|' - r'volatile|virtual|while)\b', Keyword), + 'statements': [ + (r'(asm|catch|const_cast|delete|dynamic_cast|explicit|' + r'export|friend|mutable|namespace|new|operator|' + r'private|protected|public|reinterpret_cast|' + r'restrict|static_cast|template|this|throw|throws|' + r'typeid|typename|using|virtual)\b', Keyword), (r'(class)(\s+)', bygroups(Keyword, Text), 'classname'), - (r'(bool|int|long|float|short|double|char|unsigned|signed|' - r'void|wchar_t)\b', Keyword.Type), - (r'(_{0,2}inline|naked|thread)\b', Keyword.Reserved), - (r'__(asm|int8|based|except|int16|stdcall|cdecl|fastcall|int32|' - r'declspec|finally|int64|try|leave|wchar_t|w64|virtual_inheritance|' - r'uuidof|unaligned|super|single_inheritance|raise|noop|' - r'multiple_inheritance|m128i|m128d|m128|m64|interface|' - r'identifier|forceinline|event|assume)\b', Keyword.Reserved), + inherit, + ], + 'root': [ + inherit, + # C++ Microsoft-isms + (r'__(virtual_inheritance|uuidof|super|single_inheritance|' + r'multiple_inheritance|interface|event)\b', Keyword.Reserved), # Offload C++ extensions, http://offload.codeplay.com/ (r'(__offload|__blockingoffload|__outer)\b', Keyword.Pseudo), - (r'(true|false)\b', Keyword.Constant), - (r'NULL\b', Name.Builtin), - ('[a-zA-Z_][a-zA-Z0-9_]*:(?!:)', Name.Label), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), ], 'classname': [ (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop'), # template specification (r'\s*(?=>)', Text, '#pop'), ], - 'string': [ - (r'"', String, '#pop'), - (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape), - (r'[^\\"\n]+', String), # all other characters - (r'\\\n', String), # line continuation - (r'\\', String), # stray backslash - ], - 'macro': [ - (r'[^/\n]+', Comment.Preproc), - (r'/[*](.|\n)*?[*]/', Comment.Multiline), - (r'//.*?\n', Comment.Single, '#pop'), - (r'/', Comment.Preproc), - (r'(?<=\\)\n', Comment.Preproc), - (r'\n', Comment.Preproc, '#pop'), - ], - 'if0': [ - (r'^\s*#if.*?(?<!\\)\n', Comment.Preproc, '#push'), - (r'^\s*#endif.*?(?<!\\)\n', Comment.Preproc, '#pop'), - (r'.*?\n', Comment), - ] } + def analyse_text(text): + return 0.1 + -class ECLexer(RegexLexer): +class ECLexer(CLexer): """ For eC source code with preprocessor directives. @@ -273,146 +241,29 @@ class ECLexer(RegexLexer): filenames = ['*.ec', '*.eh'] mimetypes = ['text/x-echdr', 'text/x-ecsrc'] - #: optional Comment or Whitespace - _ws = r'(?:\s|//.*?\n|/[*].*?[*]/)+' - #: only one /* */ style comment - _ws1 = r':\s*/[*].*?[*]/\s*' - tokens = { - 'whitespace': [ - # preprocessor directives: without whitespace - ('^#if\s+0', Comment.Preproc, 'if0'), - ('^#', Comment.Preproc, 'macro'), - # or with whitespace - ('^' + _ws1 + r'#if\s+0', Comment.Preproc, 'if0'), - ('^' + _ws1 + '#', Comment.Preproc, 'macro'), - (r'^(\s*)([a-zA-Z_][a-zA-Z0-9_]*:(?!:))', bygroups(Text, Name.Label)), - (r'\n', Text), - (r'\s+', Text), - (r'\\\n', Text), # line continuation - (r'//(\n|(.|\n)*?[^\\]\n)', Comment.Single), - (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline), - ], 'statements': [ - (r'L?"', String, 'string'), - (r"L?'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char), - (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*', Number.Float), - (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float), - (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex), - (r'0[0-7]+[LlUu]*', Number.Oct), - (r'\d+[LlUu]*', Number.Integer), - (r'\*/', Error), - (r'[~!%^&*+=|?:<>/-]', Operator), - (r'[()\[\],.]', Punctuation), - (r'\b(case)(.+?)(:)', bygroups(Keyword, using(this), Text)), - (r'(auto|break|case|const|continue|default|do|else|enum|extern|' - r'for|goto|if|register|restricted|return|sizeof|static|struct|' - r'switch|typedef|union|volatile|virtual|while|class|private|public|' - r'property|import|delete|new|new0|renew|renew0|define|get|set|remote|dllexport|dllimport|stdcall|' - r'subclass|__on_register_module|namespace|using|typed_object|any_object|incref|register|watch|' - r'stopwatching|firewatchers|watchable|class_designer|class_fixed|class_no_expansion|isset|' - r'class_default_property|property_category|class_data|class_property|virtual|thisclass|' + (r'(virtual|class|private|public|property|import|delete|new|new0|' + r'renew|renew0|define|get|set|remote|dllexport|dllimport|stdcall|' + r'subclass|__on_register_module|namespace|using|typed_object|' + r'any_object|incref|register|watch|stopwatching|firewatchers|' + r'watchable|class_designer|class_fixed|class_no_expansion|isset|' + r'class_default_property|property_category|class_data|' + r'class_property|virtual|thisclass|' r'dbtable|dbindex|database_open|dbfield)\b', Keyword), - (r'(int|long|float|short|double|char|unsigned|signed|void)\b', - Keyword.Type), (r'(uint|uint16|uint32|uint64|bool|byte|unichar|int64)\b', Keyword.Type), (r'(class)(\s+)', bygroups(Keyword, Text), 'classname'), - (r'(_{0,2}inline|naked|restrict|thread|typename)\b', Keyword.Reserved), - (r'__(asm|int8|based|except|int16|stdcall|cdecl|fastcall|int32|' - r'declspec|finally|int64|try|leave)\b', Keyword.Reserved), - (r'(true|false|null|value|this|NULL)\b', Name.Builtin), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), - ], - 'root': [ - include('whitespace'), - # functions - (r'((?:[a-zA-Z0-9_*\s])+?(?:\s|[*]))' # return arguments - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name - r'(\s*\([^;]*?\))' # signature - r'(' + _ws + r')?({)', - bygroups(using(this), Name.Function, using(this), using(this), - Punctuation), - 'function'), - # function declarations - (r'((?:[a-zA-Z0-9_*\s])+?(?:\s|[*]))' # return arguments - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name - r'(\s*\([^;]*?\))' # signature - r'(' + _ws + r')?(;)', - bygroups(using(this), Name.Function, using(this), using(this), - Punctuation)), - ('', Text, 'statement'), + (r'(null|value|this)\b', Name.Builtin), + inherit, ], 'classname': [ (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop'), # template specification (r'\s*(?=>)', Text, '#pop'), ], - 'statement' : [ - include('whitespace'), - include('statements'), - ('[{}]', Punctuation), - (';', Punctuation, '#pop'), - ], - 'function': [ - include('whitespace'), - include('statements'), - (';', Punctuation), - ('{', Punctuation, '#push'), - ('}', Punctuation, '#pop'), - ], - 'string': [ - (r'"', String, '#pop'), - (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape), - (r'[^\\"\n]+', String), # all other characters - (r'\\\n', String), # line continuation - (r'\\', String), # stray backslash - ], - 'macro': [ - (r'[^/\n]+', Comment.Preproc), - (r'/[*](.|\n)*?[*]/', Comment.Multiline), - (r'//.*?\n', Comment.Single, '#pop'), - (r'/', Comment.Preproc), - (r'(?<=\\)\n', Comment.Preproc), - (r'\n', Comment.Preproc, '#pop'), - ], - 'if0': [ - (r'^\s*#if.*?(?<!\\)\n', Comment.Preproc, '#push'), - (r'^\s*#el(?:se|if).*\n', Comment.Preproc, '#pop'), - (r'^\s*#endif.*?(?<!\\)\n', Comment.Preproc, '#pop'), - (r'.*?\n', Comment), - ] } - stdlib_types = ['size_t', 'ssize_t', 'off_t', 'wchar_t', 'ptrdiff_t', - 'sig_atomic_t', 'fpos_t', 'clock_t', 'time_t', 'va_list', - 'jmp_buf', 'FILE', 'DIR', 'div_t', 'ldiv_t', 'mbstate_t', - 'wctrans_t', 'wint_t', 'wctype_t'] - c99_types = ['_Bool', '_Complex', 'int8_t', 'int16_t', 'int32_t', 'int64_t', - 'uint8_t', 'uint16_t', 'uint32_t', 'uint64_t', 'int_least8_t', - 'int_least16_t', 'int_least32_t', 'int_least64_t', - 'uint_least8_t', 'uint_least16_t', 'uint_least32_t', - 'uint_least64_t', 'int_fast8_t', 'int_fast16_t', 'int_fast32_t', - 'int_fast64_t', 'uint_fast8_t', 'uint_fast16_t', 'uint_fast32_t', - 'uint_fast64_t', 'intptr_t', 'uintptr_t', 'intmax_t', 'uintmax_t'] - - def __init__(self, **options): - self.stdlibhighlighting = get_bool_opt(options, - 'stdlibhighlighting', True) - self.c99highlighting = get_bool_opt(options, - 'c99highlighting', True) - RegexLexer.__init__(self, **options) - - def get_tokens_unprocessed(self, text): - for index, token, value in \ - RegexLexer.get_tokens_unprocessed(self, text): - if token is Name: - if self.stdlibhighlighting and value in self.stdlib_types: - token = Keyword.Type - elif self.c99highlighting and value in self.c99_types: - token = Keyword.Type - yield index, token, value - class DLexer(RegexLexer): """ @@ -1057,40 +908,176 @@ class DylanLexer(RegexLexer): name = 'Dylan' aliases = ['dylan'] - filenames = ['*.dylan', '*.dyl'] + filenames = ['*.dylan', '*.dyl', '*.intr'] mimetypes = ['text/x-dylan'] - flags = re.DOTALL + flags = re.IGNORECASE + + builtins = set([ + 'subclass', 'abstract', 'block', 'concrete', 'constant', 'class', + 'compiler-open', 'compiler-sideways', 'domain', 'dynamic', + 'each-subclass', 'exception', 'exclude', 'function', 'generic', + 'handler', 'inherited', 'inline', 'inline-only', 'instance', + 'interface', 'import', 'keyword', 'library', 'macro', 'method', + 'module', 'open', 'primary', 'required', 'sealed', 'sideways', + 'singleton', 'slot', 'thread', 'variable', 'virtual']) + + keywords = set([ + 'above', 'afterwards', 'begin', 'below', 'by', 'case', 'cleanup', + 'create', 'define', 'else', 'elseif', 'end', 'export', 'finally', + 'for', 'from', 'if', 'in', 'let', 'local', 'otherwise', 'rename', + 'select', 'signal', 'then', 'to', 'unless', 'until', 'use', 'when', + 'while']) + + operators = set([ + '~', '+', '-', '*', '|', '^', '=', '==', '~=', '~==', '<', '<=', + '>', '>=', '&', '|']) + + functions = set([ + 'abort', 'abs', 'add', 'add!', 'add-method', 'add-new', 'add-new!', + 'all-superclasses', 'always', 'any?', 'applicable-method?', 'apply', + 'aref', 'aref-setter', 'as', 'as-lowercase', 'as-lowercase!', + 'as-uppercase', 'as-uppercase!', 'ash', 'backward-iteration-protocol', + 'break', 'ceiling', 'ceiling/', 'cerror', 'check-type', 'choose', + 'choose-by', 'complement', 'compose', 'concatenate', 'concatenate-as', + 'condition-format-arguments', 'condition-format-string', 'conjoin', + 'copy-sequence', 'curry', 'default-handler', 'dimension', 'dimensions', + 'direct-subclasses', 'direct-superclasses', 'disjoin', 'do', + 'do-handlers', 'element', 'element-setter', 'empty?', 'error', 'even?', + 'every?', 'false-or', 'fill!', 'find-key', 'find-method', 'first', + 'first-setter', 'floor', 'floor/', 'forward-iteration-protocol', + 'function-arguments', 'function-return-values', + 'function-specializers', 'gcd', 'generic-function-mandatory-keywords', + 'generic-function-methods', 'head', 'head-setter', 'identity', + 'initialize', 'instance?', 'integral?', 'intersection', + 'key-sequence', 'key-test', 'last', 'last-setter', 'lcm', 'limited', + 'list', 'logand', 'logbit?', 'logior', 'lognot', 'logxor', 'make', + 'map', 'map-as', 'map-into', 'max', 'member?', 'merge-hash-codes', + 'min', 'modulo', 'negative', 'negative?', 'next-method', + 'object-class', 'object-hash', 'odd?', 'one-of', 'pair', 'pop', + 'pop-last', 'positive?', 'push', 'push-last', 'range', 'rank', + 'rcurry', 'reduce', 'reduce1', 'remainder', 'remove', 'remove!', + 'remove-duplicates', 'remove-duplicates!', 'remove-key!', + 'remove-method', 'replace-elements!', 'replace-subsequence!', + 'restart-query', 'return-allowed?', 'return-description', + 'return-query', 'reverse', 'reverse!', 'round', 'round/', + 'row-major-index', 'second', 'second-setter', 'shallow-copy', + 'signal', 'singleton', 'size', 'size-setter', 'slot-initialized?', + 'sort', 'sort!', 'sorted-applicable-methods', 'subsequence-position', + 'subtype?', 'table-protocol', 'tail', 'tail-setter', 'third', + 'third-setter', 'truncate', 'truncate/', 'type-error-expected-type', + 'type-error-value', 'type-for-copy', 'type-union', 'union', 'values', + 'vector', 'zero?']) + + valid_name = '\\\\?[a-zA-Z0-9' + re.escape('!&*<>|^$%@_-+~?/=') + ']+' + + def get_tokens_unprocessed(self, text): + for index, token, value in RegexLexer.get_tokens_unprocessed(self, text): + if token is Name: + if value in self.builtins: + yield index, Name.Builtin, value + continue + if value in self.keywords: + yield index, Keyword, value + continue + if value in self.functions: + yield index, Name.Builtin, value + continue + if value in self.operators: + yield index, Operator, value + continue + yield index, token, value tokens = { 'root': [ - (r'\b(subclass|abstract|block|c(on(crete|stant)|lass)|domain' - r'|ex(c(eption|lude)|port)|f(unction(al)?)|generic|handler' - r'|i(n(herited|line|stance|terface)|mport)|library|m(acro|ethod)' - r'|open|primary|sealed|si(deways|ngleton)|slot' - r'|v(ariable|irtual))\b', Name.Builtin), - (r'<\w+>', Keyword.Type), + # Whitespace + (r'\s+', Text), + + # single line comment + (r'//.*?\n', Comment.Single), + + # lid header + (r'([A-Za-z0-9-]+)(:)([ \t]*)(.*(?:\n[ \t].+)*)', + bygroups(Name.Attribute, Operator, Text, String)), + + ('', Text, 'code') # no header match, switch to code + ], + 'code': [ + # Whitespace + (r'\s+', Text), + + # single line comment (r'//.*?\n', Comment.Single), - (r'/\*[\w\W]*?\*/', Comment.Multiline), + + # multi-line comment + (r'/\*', Comment.Multiline, 'comment'), + + # strings and characters (r'"', String, 'string'), (r"'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char), - (r'=>|\b(a(bove|fterwards)|b(e(gin|low)|y)|c(ase|leanup|reate)' - r'|define|else(if)?|end|f(inally|or|rom)|i[fn]|l(et|ocal)|otherwise' - r'|rename|s(elect|ignal)|t(hen|o)|u(n(less|til)|se)|wh(en|ile))\b', - Keyword), - (r'([ \t])([!\$%&\*\/:<=>\?~_^a-zA-Z0-9.+\-]*:)', - bygroups(Text, Name.Variable)), - (r'([ \t]*)(\S+[^:])([ \t]*)(\()([ \t]*)', - bygroups(Text, Name.Function, Text, Punctuation, Text)), - (r'-?[0-9.]+', Number), - (r'[(),;]', Punctuation), - (r'\$[a-zA-Z0-9-]+', Name.Constant), - (r'[!$%&*/:<>=?~^.+\[\]{}-]+', Operator), - (r'\s+', Text), - (r'#"[a-zA-Z0-9-]+"', Keyword), + + # binary integer + (r'#[bB][01]+', Number), + + # octal integer + (r'#[oO][0-7]+', Number.Oct), + + # floating point + (r'[-+]?(\d*\.\d+(e[-+]?\d+)?|\d+(\.\d*)?e[-+]?\d+)', Number.Float), + + # decimal integer + (r'[-+]?\d+', Number.Integer), + + # hex integer + (r'#[xX][0-9a-fA-F]+', Number.Hex), + + # Macro parameters + (r'(\?' + valid_name + ')(:)(token|name|variable|expression|body|case-body|\*)', + bygroups(Name.Tag, Operator, Name.Builtin)), + (r'(\?)(:)(token|name|variable|expression|body|case-body|\*)', + bygroups(Name.Tag, Operator, Name.Builtin)), + (r'\?' + valid_name, Name.Tag), + + # Punctuation + (r'(=>|::|#\(|#\[|##|\?|\?\?|\?=|[(){}\[\],\.;])', Punctuation), + + # Most operators are picked up as names and then re-flagged. + # This one isn't valid in a name though, so we pick it up now. + (r':=', Operator), + + # Pick up #t / #f before we match other stuff with #. + (r'#[tf]', Literal), + + # #"foo" style keywords + (r'#"', String.Symbol, 'keyword'), + + # #rest, #key, #all-keys, etc. (r'#[a-zA-Z0-9-]+', Keyword), - (r'#(\(|\[)', Punctuation), - (r'[a-zA-Z0-9-_]+', Name.Variable), + + # required-init-keyword: style keywords. + (valid_name + ':', Keyword), + + # class names + (r'<' + valid_name + '>', Name.Class), + + # define variable forms. + (r'\*' + valid_name + '\*', Name.Variable.Global), + + # define constant forms. + (r'\$' + valid_name, Name.Constant), + + # everything else. We re-flag some of these in the method above. + (valid_name, Name), + ], + 'comment': [ + (r'[^*/]', Comment.Multiline), + (r'/\*', Comment.Multiline, '#push'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[*/]', Comment.Multiline) + ], + 'keyword': [ + (r'"', String.Symbol, '#pop'), + (r'[^\\"]+', String.Symbol), # all other characters ], 'string': [ (r'"', String, '#pop'), @@ -1098,177 +1085,160 @@ class DylanLexer(RegexLexer): (r'[^\\"\n]+', String), # all other characters (r'\\\n', String), # line continuation (r'\\', String), # stray backslash - ], + ] } -class ObjectiveCLexer(RegexLexer): +class DylanLidLexer(RegexLexer): """ - For Objective-C source code with preprocessor directives. + For Dylan LID (Library Interchange Definition) files. + + *New in Pygments 1.6.* """ - name = 'Objective-C' - aliases = ['objective-c', 'objectivec', 'obj-c', 'objc'] - # XXX: objc has .h files too :-/ - filenames = ['*.m'] - mimetypes = ['text/x-objective-c'] + name = 'DylanLID' + aliases = ['dylan-lid', 'lid'] + filenames = ['*.lid', '*.hdp'] + mimetypes = ['text/x-dylan-lid'] - #: optional Comment or Whitespace - _ws = r'(?:\s|//.*?\n|/[*].*?[*]/)+' - #: only one /* */ style comment - _ws1 = r':\s*/[*].*?[*]/\s*' + flags = re.IGNORECASE tokens = { - 'whitespace': [ - # preprocessor directives: without whitespace - ('^#if\s+0', Comment.Preproc, 'if0'), - ('^#', Comment.Preproc, 'macro'), - # or with whitespace - ('^(' + _ws1 + r')(#if\s+0)', - bygroups(using(this), Comment.Preproc), 'if0'), - ('^(' + _ws1 + ')(#)', - bygroups(using(this), Comment.Preproc), 'macro'), - (r'\n', Text), - (r'\s+', Text), - (r'\\\n', Text), # line continuation - (r'//(\n|(.|\n)*?[^\\]\n)', Comment.Single), - (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline), - ], - 'statements': [ - (r'(L|@)?"', String, 'string'), - (r"(L|@)?'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", - String.Char), - (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[lL]?', Number.Float), - (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float), - (r'0x[0-9a-fA-F]+[Ll]?', Number.Hex), - (r'0[0-7]+[Ll]?', Number.Oct), - (r'\d+[Ll]?', Number.Integer), - (r'[~!%^&*+=|?:<>/-]', Operator), - (r'[()\[\],.]', Punctuation), - (r'(auto|break|case|const|continue|default|do|else|enum|extern|' - r'for|goto|if|register|restricted|return|sizeof|static|struct|' - r'switch|typedef|union|volatile|virtual|while|in|@selector|' - r'@private|@protected|@public|@encode|' - r'@synchronized|@try|@throw|@catch|@finally|@end|@property|' - r'@synthesize|@dynamic|@optional)\b', Keyword), - (r'(int|long|float|short|double|char|unsigned|signed|void|' - r'id|BOOL|IBOutlet|IBAction|SEL)\b', Keyword.Type), - (r'(_{0,2}inline|naked|restrict|thread|typename)\b', - Keyword.Reserved), - (r'__(asm|int8|based|except|int16|stdcall|cdecl|fastcall|int32|' - r'declspec|finally|int64|try|leave)\b', Keyword.Reserved), - (r'(TRUE|FALSE|nil|NULL)\b', Name.Builtin), - ('[a-zA-Z$_][a-zA-Z0-9$_]*:(?!:)', Name.Label), - ('[a-zA-Z$_][a-zA-Z0-9$_]*', Name), - (r'(@interface|@implementation)(\s+)', bygroups(Keyword, Text), - ('#pop', 'classname')), - (r'(@class|@protocol)(\s+)', bygroups(Keyword, Text), - ('#pop', 'forward_classname')), - ], 'root': [ - include('whitespace'), - # functions - (r'((?:[a-zA-Z0-9_*\s])+?(?:\s|[*]))' # return arguments - r'([a-zA-Z$_][a-zA-Z0-9$_]*)' # method name - r'(\s*\([^;]*?\))' # signature - r'(' + _ws + r')?({)', - bygroups(using(this), Name.Function, - using(this), Text, Punctuation), - 'function'), - # methods - (r'^([-+])(\s*)' # method marker - r'(\(.*?\))?(\s*)' # return type - r'([a-zA-Z$_][a-zA-Z0-9$_]*:?)', # begin of method name - bygroups(Keyword, Text, using(this), - Text, Name.Function), - 'method'), - # function declarations - (r'((?:[a-zA-Z0-9_*\s])+?(?:\s|[*]))' # return arguments - r'([a-zA-Z$_][a-zA-Z0-9$_]*)' # method name - r'(\s*\([^;]*?\))' # signature - r'(' + _ws + r')?(;)', - bygroups(using(this), Name.Function, - using(this), Text, Punctuation)), - (r'(@interface|@implementation)(\s+)', bygroups(Keyword, Text), - 'classname'), - (r'(@class|@protocol)(\s+)', bygroups(Keyword, Text), - 'forward_classname'), - (r'(\s*)(@end)(\s*)', bygroups(Text, Keyword, Text)), - ('', Text, 'statement'), - ], - 'classname' : [ - # interface definition that inherits - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*:\s*)([a-zA-Z$_][a-zA-Z0-9$_]*)?', - bygroups(Name.Class, Text, Name.Class), '#pop'), - # interface definition for a category - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*)(\([a-zA-Z$_][a-zA-Z0-9$_]*\))', - bygroups(Name.Class, Text, Name.Label), '#pop'), - # simple interface / implementation - ('([a-zA-Z$_][a-zA-Z0-9$_]*)', Name.Class, '#pop') - ], - 'forward_classname' : [ - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*,\s*)', - bygroups(Name.Class, Text), 'forward_classname'), - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*;?)', - bygroups(Name.Class, Text), '#pop') - ], - 'statement' : [ - include('whitespace'), - include('statements'), - ('[{}]', Punctuation), - (';', Punctuation, '#pop'), - ], - 'function': [ - include('whitespace'), - include('statements'), - (';', Punctuation), - ('{', Punctuation, '#push'), - ('}', Punctuation, '#pop'), - ], - 'method': [ - include('whitespace'), - # TODO unsure if ellipses are allowed elsewhere, see discussion in - # Issue 789 - (r',', Punctuation), - (r'\.\.\.', Punctuation), - (r'(\(.*?\))([a-zA-Z$_][a-zA-Z0-9$_]*)', bygroups(using(this), - Name.Variable)), - (r'[a-zA-Z$_][a-zA-Z0-9$_]*:', Name.Function), - (';', Punctuation, '#pop'), - ('{', Punctuation, 'function'), - ('', Text, '#pop'), - ], - 'string': [ - (r'"', String, '#pop'), - (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape), - (r'[^\\"\n]+', String), # all other characters - (r'\\\n', String), # line continuation - (r'\\', String), # stray backslash - ], - 'macro': [ - (r'[^/\n]+', Comment.Preproc), - (r'/[*](.|\n)*?[*]/', Comment.Multiline), - (r'//.*?\n', Comment.Single, '#pop'), - (r'/', Comment.Preproc), - (r'(?<=\\)\n', Comment.Preproc), - (r'\n', Comment.Preproc, '#pop'), - ], - 'if0': [ - (r'^\s*#if.*?(?<!\\)\n', Comment.Preproc, '#push'), - (r'^\s*#endif.*?(?<!\\)\n', Comment.Preproc, '#pop'), - (r'.*?\n', Comment), + # Whitespace + (r'\s+', Text), + + # single line comment + (r'//.*?\n', Comment.Single), + + # lid header + (r'(.*?)(:)([ \t]*)(.*(?:\n[ \t].+)*)', + bygroups(Name.Attribute, Operator, Text, String)), ] } - def analyse_text(text): - if '@import' in text or '@interface' in text or \ - '@implementation' in text: - return True - elif '@"' in text: # strings - return True - elif re.match(r'\[[a-zA-Z0-9.]:', text): # message - return True - return False +def objective(baselexer): + """ + Generate a subclass of baselexer that accepts the Objective-C syntax + extensions. + """ + + # Have to be careful not to accidentally match JavaDoc/Doxygen syntax here, + # since that's quite common in ordinary C/C++ files. It's OK to match + # JavaDoc/Doxygen keywords that only apply to Objective-C, mind. + # + # The upshot of this is that we CANNOT match @class or @interface + _oc_keywords = re.compile(r'@(?:end|implementation|protocol)') + + # Matches [ <ws>? identifier <ws> ( identifier <ws>? ] | identifier? : ) + # (note the identifier is *optional* when there is a ':'!) + _oc_message = re.compile(r'\[\s*[a-zA-Z_][a-zA-Z0-9_]*\s+' + r'(?:[a-zA-Z_][a-zA-Z0-9_]*\s*\]|' + r'(?:[a-zA-Z_][a-zA-Z0-9_]*)?:)') + + class GeneratedObjectiveCVariant(baselexer): + """ + Implements Objective-C syntax on top of an existing C family lexer. + """ + + tokens = { + 'statements': [ + (r'@"', String, 'string'), + (r"@'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", + String.Char), + (r'@(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[lL]?', Number.Float), + (r'@(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float), + (r'@0x[0-9a-fA-F]+[Ll]?', Number.Hex), + (r'@0[0-7]+[Ll]?', Number.Oct), + (r'@\d+[Ll]?', Number.Integer), + (r'(in|@selector|@private|@protected|@public|@encode|' + r'@synchronized|@try|@throw|@catch|@finally|@end|@property|' + r'@synthesize|@dynamic|@optional)\b', Keyword), + (r'(id|Class|IMP|SEL|BOOL|IBOutlet|IBAction|unichar)\b', + Keyword.Type), + (r'@(true|false|YES|NO)\n', Name.Builtin), + (r'(YES|NO|nil)\b', Name.Builtin), + (r'(@interface|@implementation)(\s+)', bygroups(Keyword, Text), + ('#pop', 'oc_classname')), + (r'(@class|@protocol)(\s+)', bygroups(Keyword, Text), + ('#pop', 'oc_forward_classname')), + inherit, + ], + 'oc_classname' : [ + # interface definition that inherits + ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*:\s*)([a-zA-Z$_][a-zA-Z0-9$_]*)?', + bygroups(Name.Class, Text, Name.Class), '#pop'), + # interface definition for a category + ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*)(\([a-zA-Z$_][a-zA-Z0-9$_]*\))', + bygroups(Name.Class, Text, Name.Label), '#pop'), + # simple interface / implementation + ('([a-zA-Z$_][a-zA-Z0-9$_]*)', Name.Class, '#pop') + ], + 'oc_forward_classname' : [ + ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*,\s*)', + bygroups(Name.Class, Text), 'oc_forward_classname'), + ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*;?)', + bygroups(Name.Class, Text), '#pop') + ], + 'root': [ + # methods + (r'^([-+])(\s*)' # method marker + r'(\(.*?\))?(\s*)' # return type + r'([a-zA-Z$_][a-zA-Z0-9$_]*:?)', # begin of method name + bygroups(Keyword, Text, using(this), + Text, Name.Function), + 'method'), + inherit, + ], + 'method': [ + include('whitespace'), + # TODO unsure if ellipses are allowed elsewhere, see + # discussion in Issue 789 + (r',', Punctuation), + (r'\.\.\.', Punctuation), + (r'(\(.*?\))([a-zA-Z$_][a-zA-Z0-9$_]*)', bygroups(using(this), + Name.Variable)), + (r'[a-zA-Z$_][a-zA-Z0-9$_]*:', Name.Function), + (';', Punctuation, '#pop'), + ('{', Punctuation, 'function'), + ('', Text, '#pop'), + ], + } + + def analyse_text(text): + if _oc_keywords.search(text): + return 1.0 + elif '@"' in text: # strings + return 0.8 + elif _oc_message.search(text): + return 0.8 + return 0 + + return GeneratedObjectiveCVariant + + +class ObjectiveCLexer(objective(CLexer)): + """ + For Objective-C source code with preprocessor directives. + """ + + name = 'Objective-C' + aliases = ['objective-c', 'objectivec', 'obj-c', 'objc'] + filenames = ['*.m', '*.h'] + mimetypes = ['text/x-objective-c'] + priority = 0.05 # Lower than C + + +class ObjectiveCppLexer(objective(CppLexer)): + """ + For Objective-C++ source code with preprocessor directives. + """ + + name = 'Objective-C++' + aliases = ['objective-c++', 'objectivec++', 'obj-c++', 'objc++'] + filenames = ['*.mm', '*.hh'] + mimetypes = ['text/x-objective-c++'] + priority = 0.05 # Lower than C++ class FortranLexer(RegexLexer): @@ -2943,13 +2913,13 @@ class RustLexer(RegexLexer): (r'/[*](.|\n)*?[*]/', Comment.Multiline), # Keywords - (r'(alt|as|assert|be|break|check|claim|class|const' - r'|cont|copy|crust|do|else|enum|export|fail' - r'|false|fn|for|if|iface|impl|import|let|log' - r'|loop|mod|mut|native|pure|resource|ret|true' - r'|type|unsafe|use|white|note|bind|prove|unchecked' - r'|with|syntax|u8|u16|u32|u64|i8|i16|i32|i64|uint' - r'|int|f32|f64)\b', Keyword), + (r'(as|assert|break|const' + r'|copy|do|else|enum|extern|fail' + r'|false|fn|for|if|impl|let|log' + r'|loop|match|mod|move|mut|once|priv|pub|pure' + r'|ref|return|static|struct|trait|true|type|unsafe|use|while' + r'|u8|u16|u32|u64|i8|i16|i32|i64|uint' + r'|int|float|f32|f64|str)\b', Keyword), # Character Literal (r"""'(\\['"\\nrt]|\\x[0-9a-fA-F]{2}|\\[0-7]{1,3}""" @@ -2978,8 +2948,8 @@ class RustLexer(RegexLexer): (r'#\[', Comment.Preproc, 'attribute['), (r'#\(', Comment.Preproc, 'attribute('), # Macros - (r'#[A-Za-z_][A-Za-z0-9_]*\[', Comment.Preproc, 'attribute['), - (r'#[A-Za-z_][A-Za-z0-9_]*\(', Comment.Preproc, 'attribute('), + (r'[A-Za-z_][A-Za-z0-9_]*!\[', Comment.Preproc, 'attribute['), + (r'[A-Za-z_][A-Za-z0-9_]*!\(', Comment.Preproc, 'attribute('), ], 'number_lit': [ (r'(([ui](8|16|32|64)?)|(f(32|64)?))?', Keyword, '#pop'), @@ -3070,7 +3040,7 @@ class MonkeyLexer(RegexLexer): name = 'Monkey' aliases = ['monkey'] filenames = ['*.monkey'] - mimetypes = ['text/x-monkey'] + mimetypes = ['text/x-monkey'] name_variable = r'[a-z_][a-zA-Z0-9_]*' name_function = r'[A-Z][a-zA-Z0-9_]*' @@ -3144,10 +3114,10 @@ class MonkeyLexer(RegexLexer): (r':', Punctuation, 'classname'), (r'\s+', Text), (r'\(', Punctuation, 'variables'), - (r'\)', Punctuation, '#pop') + (r'\)', Punctuation, '#pop') ], 'classname': [ - (r'%s\.' % name_module, Name.Namespace), + (r'%s\.' % name_module, Name.Namespace), (r'%s\b' % keyword_type, Keyword.Type), (r'%s\b' % name_class, Name.Class), # array (of given size) diff --git a/pygments/lexers/dalvik.py b/pygments/lexers/dalvik.py index 861d769c..de9b11fa 100644 --- a/pygments/lexers/dalvik.py +++ b/pygments/lexers/dalvik.py @@ -5,11 +5,11 @@ Pygments lexers for Dalvik VM-related languages. - :copyright: Copyright 2011-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ -from pygments.lexer import RegexLexer, include, bygroups, using +from pygments.lexer import RegexLexer, include, bygroups from pygments.token import Keyword, Text, Comment, Name, String, Number, \ Punctuation diff --git a/pygments/lexers/dotnet.py b/pygments/lexers/dotnet.py index 17edddc0..bdd9edc1 100644 --- a/pygments/lexers/dotnet.py +++ b/pygments/lexers/dotnet.py @@ -5,7 +5,7 @@ Lexers for .net languages. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re diff --git a/pygments/lexers/functional.py b/pygments/lexers/functional.py index 8bb532d0..4947bf7d 100644 --- a/pygments/lexers/functional.py +++ b/pygments/lexers/functional.py @@ -5,7 +5,7 @@ Lexers for functional languages. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -2437,7 +2437,7 @@ class KokaLexer(RegexLexer): # builtin names and special names builtin = [ 'for', 'while', 'repeat', - 'foreach', 'foreach-indexed', + 'foreach', 'foreach-indexed', 'error', 'catch', 'finally', 'cs', 'js', 'file', 'ref', 'assigned', ] diff --git a/pygments/lexers/hdl.py b/pygments/lexers/hdl.py index 0db35e74..647f8396 100644 --- a/pygments/lexers/hdl.py +++ b/pygments/lexers/hdl.py @@ -5,7 +5,7 @@ Lexers for hardware descriptor languages. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/jvm.py b/pygments/lexers/jvm.py index 83eb02f5..16066cc7 100644 --- a/pygments/lexers/jvm.py +++ b/pygments/lexers/jvm.py @@ -5,7 +5,7 @@ Pygments lexers for JVM languages. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -675,7 +675,7 @@ class ClojureLexer(RegexLexer): (r'::?' + valid_name, String.Symbol), # special operators - (r'~@|[`\'#^~&]', Operator), + (r'~@|[`\'#^~&@]', Operator), # highlight the special forms (_multi_escape(special_forms), Keyword), diff --git a/pygments/lexers/math.py b/pygments/lexers/math.py index 0e78bb49..a4e241d6 100644 --- a/pygments/lexers/math.py +++ b/pygments/lexers/math.py @@ -5,12 +5,13 @@ Lexers for math languages. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re +from pygments.util import shebang_matches from pygments.lexer import Lexer, RegexLexer, bygroups, include, \ combined, do_insertions from pygments.token import Comment, String, Punctuation, Keyword, Name, \ @@ -83,7 +84,8 @@ class JuliaLexer(RegexLexer): (r'`(?s).*?`', String.Backtick), # chars - (r"'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,3}|\\u[a-fA-F0-9]{1,4}|\\U[a-fA-F0-9]{1,6}|[^\\\'\n])'", String.Char), + (r"'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,3}|\\u[a-fA-F0-9]{1,4}|" + r"\\U[a-fA-F0-9]{1,6}|[^\\\'\n])'", String.Char), # try to match trailing transpose (r'(?<=[.\w\)\]])\'+', Operator), @@ -97,8 +99,8 @@ class JuliaLexer(RegexLexer): (r'[a-zA-Z_][a-zA-Z0-9_]*', Name), # numbers - (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), - (r'\d+[eE][+-]?[0-9]+', Number.Float), + (r'(\d+\.\d*|\d*\.\d+)([eEf][+-]?[0-9]+)?', Number.Float), + (r'\d+[eEf][+-]?[0-9]+', Number.Float), (r'0b[01]+', Number.Binary), (r'0o[0-7]+', Number.Oct), (r'0x[a-fA-F0-9]+', Number.Hex), @@ -342,6 +344,10 @@ class MatlabLexer(RegexLexer): # (not great, but handles common cases...) (r'(?<=[\w\)\]])\'', Operator), + (r'(\d+\.\d*|\d*\.\d+)([eEf][+-]?[0-9]+)?', Number.Float), + (r'\d+[eEf][+-]?[0-9]+', Number.Float), + (r'\d+', Number.Integer), + (r'(?<![\w\)\]])\'', String, 'string'), ('[a-zA-Z_][a-zA-Z0-9_]*', Name), (r'.', Text), @@ -788,6 +794,10 @@ class OctaveLexer(RegexLexer): (r'"[^"]*"', String), + (r'(\d+\.\d*|\d*\.\d+)([eEf][+-]?[0-9]+)?', Number.Float), + (r'\d+[eEf][+-]?[0-9]+', Number.Float), + (r'\d+', Number.Integer), + # quote can be transpose, instead of string: # (not great, but handles common cases...) (r'(?<=[\w\)\]])\'', Operator), @@ -859,6 +869,10 @@ class ScilabLexer(RegexLexer): (r'(?<=[\w\)\]])\'', Operator), (r'(?<![\w\)\]])\'', String, 'string'), + (r'(\d+\.\d*|\d*\.\d+)([eEf][+-]?[0-9]+)?', Number.Float), + (r'\d+[eEf][+-]?[0-9]+', Number.Float), + (r'\d+', Number.Integer), + ('[a-zA-Z_][a-zA-Z0-9_]*', Name), (r'.', Text), ], @@ -1292,7 +1306,7 @@ class StanLexer(RegexLexer): filenames = ['*.stan'] _RESERVED = ('for', 'in', 'while', 'repeat', 'until', 'if', - 'then', 'else', 'true', 'false', 'T', + 'then', 'else', 'true', 'false', 'T', 'lower', 'upper', 'print') _TYPES = ('int', 'real', 'vector', 'simplex', 'ordered', 'row_vector', @@ -1343,7 +1357,7 @@ class StanLexer(RegexLexer): # Integer Literals (r'-?[0-9]+', Number.Integer), # Assignment operators - # SLexer makes these tokens Operators. + # SLexer makes these tokens Operators. (r'<-|~', Operator), # Infix and prefix operators (and = ) (r"\+|-|\.?\*|\.?/|\\|'|=", Operator), diff --git a/pygments/lexers/other.py b/pygments/lexers/other.py index 8dfb3145..6943985c 100644 --- a/pygments/lexers/other.py +++ b/pygments/lexers/other.py @@ -5,7 +5,7 @@ Lexers for other languages. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -19,6 +19,7 @@ from pygments.util import get_bool_opt from pygments.lexers.web import HtmlLexer from pygments.lexers._openedgebuiltins import OPENEDGEKEYWORDS +from pygments.lexers._robotframeworklexer import RobotFrameworkLexer # backwards compatibility from pygments.lexers.sql import SqlLexer, MySqlLexer, SqliteConsoleLexer @@ -32,7 +33,8 @@ __all__ = ['BrainfuckLexer', 'BefungeLexer', 'RedcodeLexer', 'MOOCodeLexer', 'AutohotkeyLexer', 'GoodDataCLLexer', 'MaqlLexer', 'ProtoBufLexer', 'HybrisLexer', 'AwkLexer', 'Cfengine3Lexer', 'SnobolLexer', 'ECLLexer', 'UrbiscriptLexer', 'OpenEdgeLexer', 'BroLexer', - 'MscgenLexer', 'KconfigLexer', 'VGLLexer', 'SourcePawnLexer'] + 'MscgenLexer', 'KconfigLexer', 'VGLLexer', 'SourcePawnLexer', + 'RobotFrameworkLexer', 'PuppetLexer', 'NSISLexer', 'RPMSpecLexer'] class ECLLexer(RegexLexer): @@ -1255,7 +1257,8 @@ class ModelicaLexer(RegexLexer): ], 'classes': [ (r'(block|class|connector|function|model|package|' - r'record|type)\b', Name.Class), + r'record|type)(\s+)([A-Za-z_]+)', + bygroups(Keyword, Text, Name.Class)) ], 'string': [ (r'"', String, '#pop'), @@ -2771,7 +2774,7 @@ class OpenEdgeLexer(RegexLexer): keywords = (r'(?i)(^|(?<=[^0-9a-z_\-]))(' + r'|'.join(OPENEDGEKEYWORDS) + - r')\s*($|(?=[^0-9a-z_\-]))') + r')\s*($|(?=[^0-9a-z_\-]))') tokens = { 'root': [ (r'/\*', Comment.Multiline, 'comment'), @@ -3156,5 +3159,272 @@ class SourcePawnLexer(RegexLexer): if value in self.SM_TYPES: token = Keyword.Type elif value in self._functions: - tokens = Name.Builtin + token = Name.Builtin yield index, token, value + + +class PuppetLexer(RegexLexer): + """ + For `Puppet <http://puppetlabs.com/>`__ configuration DSL. + + *New in Pygments 1.6.* + """ + name = 'Puppet' + aliases = ['puppet'] + filenames = ['*.pp'] + + tokens = { + 'root': [ + include('comments'), + include('keywords'), + include('names'), + include('numbers'), + include('operators'), + include('strings'), + + (r'[]{}:(),;[]', Punctuation), + (r'[^\S\n]+', Text), + ], + + 'comments': [ + (r'\s*#.*$', Comment), + (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline), + ], + + 'operators': [ + (r'(=>|\?|<|>|=|\+|-|/|\*|~|!|\|)', Operator), + (r'(in|and|or|not)\b', Operator.Word), + ], + + 'names': [ + ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Attribute), + (r'(\$\S+)(\[)(\S+)(\])', bygroups(Name.Variable, Punctuation, + String, Punctuation)), + (r'\$\S+', Name.Variable), + ], + + 'numbers': [ + # Copypasta from the Python lexer + (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float), + (r'\d+[eE][+-]?[0-9]+j?', Number.Float), + (r'0[0-7]+j?', Number.Oct), + (r'0[xX][a-fA-F0-9]+', Number.Hex), + (r'\d+L', Number.Integer.Long), + (r'\d+j?', Number.Integer) + ], + + 'keywords': [ + # Left out 'group' and 'require' + # Since they're often used as attributes + (r'(?i)(absent|alert|alias|audit|augeas|before|case|check|class|' + r'computer|configured|contained|create_resources|crit|cron|debug|' + r'default|define|defined|directory|else|elsif|emerg|err|exec|' + r'extlookup|fail|false|file|filebucket|fqdn_rand|generate|host|if|' + r'import|include|info|inherits|inline_template|installed|' + r'interface|k5login|latest|link|loglevel|macauthorization|' + r'mailalias|maillist|mcx|md5|mount|mounted|nagios_command|' + r'nagios_contact|nagios_contactgroup|nagios_host|' + r'nagios_hostdependency|nagios_hostescalation|nagios_hostextinfo|' + r'nagios_hostgroup|nagios_service|nagios_servicedependency|' + r'nagios_serviceescalation|nagios_serviceextinfo|' + r'nagios_servicegroup|nagios_timeperiod|node|noop|notice|notify|' + r'package|present|purged|realize|regsubst|resources|role|router|' + r'running|schedule|scheduled_task|search|selboolean|selmodule|' + r'service|sha1|shellquote|split|sprintf|ssh_authorized_key|sshkey|' + r'stage|stopped|subscribe|tag|tagged|template|tidy|true|undef|' + r'unmounted|user|versioncmp|vlan|warning|yumrepo|zfs|zone|' + r'zpool)\b', Keyword), + ], + + 'strings': [ + (r'"([^"])*"', String), + (r'\'([^\'])*\'', String), + ], + + } + + +class NSISLexer(RegexLexer): + """ + For `NSIS <http://nsis.sourceforge.net/>`_ scripts. + + *New in Pygments 1.6.* + """ + name = 'NSIS' + aliases = ['nsis', 'nsi', 'nsh'] + filenames = ['*.nsi', '*.nsh'] + mimetypes = ['text/x-nsis'] + + flags = re.IGNORECASE + + tokens = { + 'root': [ + (r'[;\#].*\n', Comment), + (r"'.*'", String.Single), + (r'"', String.Double, 'str_double'), + (r'`', String.Backtick, 'str_backtick'), + include('macro'), + include('interpol'), + include('basic'), + (r'\$\{[a-zA-Z_\|][a-zA-Z0-9_\|]*\}', Keyword.Pseudo), + (r'\/[a-zA-Z_][a-zA-Z0-9_]*', Name.Attribute), + ('.', Text), + ], + 'basic': [ + (r'(\n)(Function)(\s+)([\.\_a-zA-Z][\.\_a-zA-Z0-9]*)\b', + bygroups(Text, Keyword, Text, Name.Function)), + (r'\b([_a-zA-Z][_a-zA-Z0-9]*)(::)([a-zA-Z][a-zA-Z0-9]*)\b', + bygroups(Keyword.Namespace, Punctuation, Name.Function)), + (r'\b([_a-zA-Z][_a-zA-Z0-9]*)(:)', bygroups(Name.Label, Punctuation)), + (r'(\b[ULS]|\B)([\!\<\>=]?=|\<\>?|\>)\B', Operator), + (r'[\+\-\|]', Operator), + (r'[\\]', Punctuation), + (r'\b(Abort|Add(?:BrandingImage|Size)|Allow(?:RootDirInstall|SkipFiles)|' + r'AutoCloseWindow|BG(?:Font|Gradient)|BrandingText|BringToFront|' + r'Call(?:InstDLL)?|(?:Sub)?Caption|ChangeUI|CheckBitmap|ClearErrors|CompletedText|' + r'ComponentText|CopyFiles|CRCCheck|Create(?:Directory|Font|Shortcut)|' + r'Delete(?:INI(?:Sec|Str)|Reg(?:Key|Value))?|DetailPrint|DetailsButtonText|' + r'Dir(?:Show|Text|Var|Verify)|(?:Disabled|Enabled)Bitmap|EnableWindow|' + r'EnumReg(?:Key|Value)|Exch|Exec(?:Shell|Wait)?|ExpandEnvStrings|' + r'File(?:BufSize|Close|ErrorText|Open|Read(?:Byte)?|Seek|Write(?:Byte)?)?|' + r'Find(?:Close|First|Next|Window)|FlushINI|Function(?:End)?|' + r'Get(?:CurInstType|CurrentAddress|DlgItem|DLLVersion(?:Local)?|ErrorLevel|' + r'FileTime(?:Local)?|FullPathName|FunctionAddress|InstDirError|LabelAddress|TempFileName)|' + r'Goto|HideWindow|Icon|If(?:Abort|Errors|FileExists|RebootFlag|Silent)|' + r'InitPluginsDir|Install(?:ButtonText|Colors|Dir(?:RegKey)?)|' + r'Inst(?:ProgressFlags|Type(?:[GS]etText)?)|Int(?:CmpU?|Fmt|Op)|IsWindow|' + r'LangString(?:UP)?|License(?:BkColor|Data|ForceSelection|LangString|Text)|' + r'LoadLanguageFile|LockWindow|Log(?:Set|Text)|MessageBox|MiscButtonText|' + r'Name|Nop|OutFile|(?:Uninst)?Page(?:Ex(?:End)?)?|PluginDir|Pop|Push|Quit|' + r'Read(?:(?:Env|INI|Reg)Str|RegDWORD)|Reboot|(?:Un)?RegDLL|Rename|RequestExecutionLevel|' + r'ReserveFile|Return|RMDir|SearchPath|' + r'Section(?:Divider|End|(?:(?:Get|Set)(?:Flags|InstTypes|Size|Text))|Group(?:End)?|In)?|' + r'SendMessage|' + r'Set(?:AutoClose|BrandingImage|Compress(?:ionLevel|or(?:DictSize)?)?|CtlColors|' + r'CurInstType|DatablockOptimize|DateSave|Details(?:Print|View)|Error(?:s|Level)|' + r'FileAttributes|Font|OutPath|Overwrite|PluginUnload|RebootFlag|ShellVarContext|' + r'Silent|StaticBkColor)|' + r'Show(?:(?:I|Uni)nstDetails|Window)|Silent(?:Un)?Install|Sleep|SpaceTexts|' + r'Str(?:CmpS?|Cpy|Len)|SubSection(?:End)?|' + r'Uninstall(?:ButtonText|(?:Sub)?Caption|EXEName|Icon|Text)|UninstPage|' + r'Var|VI(?:AddVersionKey|ProductVersion)|WindowIcon|' + r'Write(?:INIStr|Reg(:?Bin|DWORD|(?:Expand)?Str)|Uninstaller)|XPStyle)\b', + Keyword), + (r'\b(CUR|END|(?:FILE_ATTRIBUTE_)?(?:ARCHIVE|HIDDEN|NORMAL|OFFLINE|READONLY|SYSTEM|TEMPORARY)|' + r'HK(CC|CR|CU|DD|LM|PD|U)|' + r'HKEY_(?:CLASSES_ROOT|CURRENT_(?:CONFIG|USER)|DYN_DATA|LOCAL_MACHINE|PERFORMANCE_DATA|USERS)|' + r'ID(?:ABORT|CANCEL|IGNORE|NO|OK|RETRY|YES)|' + r'MB_(?:ABORTRETRYIGNORE|DEFBUTTON[1-4]|ICON(?:EXCLAMATION|INFORMATION|QUESTION|STOP)|' + r'OK(?:CANCEL)?|RETRYCANCEL|RIGHT|SETFOREGROUND|TOPMOST|USERICON|YESNO(?:CANCEL)?)|' + r'SET|SHCTX|SW_(?:HIDE|SHOW(?:MAXIMIZED|MINIMIZED|NORMAL))|' + r'admin|all|auto|both|bottom|bzip2|checkbox|colored|current|false|force|' + r'hide|highest|if(?:diff|newer)|lastused|leave|left|listonly|lzma|nevershow|' + r'none|normal|off|on|pop|push|radiobuttons|right|show|silent|silentlog|' + r'smooth|textonly|top|true|try|user|zlib)\b', + Name.Constant), + ], + 'macro': [ + (r'\!(addincludedir(?:dir)?|addplugindir|appendfile|cd|define|' + r'delfilefile|echo(?:message)?|else|endif|error|execute|' + r'if(?:macro)?n?(?:def)?|include|insertmacro|macro(?:end)?|packhdr|' + r'search(?:parse|replace)|system|tempfilesymbol|undef|verbose|warning)\b', + Comment.Preproc), + ], + 'interpol': [ + (r'\$(R?[0-9])', Name.Builtin.Pseudo), # registers + (r'\$(ADMINTOOLS|APPDATA|CDBURN_AREA|COOKIES|COMMONFILES(?:32|64)|' + r'DESKTOP|DOCUMENTS|EXE(?:DIR|FILE|PATH)|FAVORITES|FONTS|HISTORY|' + r'HWNDPARENT|INTERNET_CACHE|LOCALAPPDATA|MUSIC|NETHOOD|PICTURES|' + r'PLUGINSDIR|PRINTHOOD|PROFILE|PROGRAMFILES(?:32|64)|QUICKLAUNCH|' + r'RECENT|RESOURCES(?:_LOCALIZED)?|SENDTO|SM(?:PROGRAMS|STARTUP)|' + r'STARTMENU|SYSDIR|TEMP(?:LATES)?|VIDEOS|WINDIR|\{NSISDIR\})', + Name.Builtin), + (r'\$(CMDLINE|INSTDIR|OUTDIR|LANGUAGE)', Name.Variable.Global), + (r'\$[a-zA-Z_][a-zA-Z0-9_]*', Name.Variable), + ], + 'str_double': [ + (r'"', String, '#pop'), + (r'\$(\\[nrt"]|\$)', String.Escape), + include('interpol'), + (r'.', String.Double), + ], + 'str_backtick': [ + (r'`', String, '#pop'), + (r'\$(\\[nrt"]|\$)', String.Escape), + include('interpol'), + (r'.', String.Double), + ], + } + + +class RPMSpecLexer(RegexLexer): + """ + For RPM *.spec files + + *New in Pygments 1.6.* + """ + + name = 'RPMSpec' + aliases = ['spec'] + filenames = ['*.spec'] + mimetypes = ['text/x-rpm-spec'] + + _directives = ('(?:package|prep|build|install|clean|check|pre[a-z]*|' + 'post[a-z]*|trigger[a-z]*|files)') + + tokens = { + 'root': [ + (r'#.*\n', Comment), + include('basic'), + ], + 'description': [ + (r'^(%' + _directives + ')(.*)$', + bygroups(Name.Decorator, Text), '#pop'), + (r'\n', Text), + (r'.', Text), + ], + 'changelog': [ + (r'\*.*\n', Generic.Subheading), + (r'^(%' + _directives + ')(.*)$', + bygroups(Name.Decorator, Text), '#pop'), + (r'\n', Text), + (r'.', Text), + ], + 'string': [ + (r'"', String.Double, '#pop'), + (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape), + include('interpol'), + (r'.', String.Double), + ], + 'basic': [ + include('macro'), + (r'(?i)^(Name|Version|Release|Epoch|Summary|Group|License|Packager|' + r'Vendor|Icon|URL|Distribution|Prefix|Patch[0-9]*|Source[0-9]*|' + r'Requires\(?[a-z]*\)?|[A-Za-z]+Req|Obsoletes|Provides|Conflicts|' + r'Build[A-Za-z]+|[A-Za-z]+Arch|Auto[A-Za-z]+)(:)(.*)$', + bygroups(Generic.Heading, Punctuation, using(this))), + (r'^%description', Name.Decorator, 'description'), + (r'^%changelog', Name.Decorator, 'changelog'), + (r'^(%' + _directives + ')(.*)$', bygroups(Name.Decorator, Text)), + (r'%(attr|defattr|dir|doc(?:dir)?|setup|config(?:ure)?|' + r'make(?:install)|ghost|patch[0-9]+|find_lang|exclude|verify)', + Keyword), + include('interpol'), + (r"'.*'", String.Single), + (r'"', String.Double, 'string'), + (r'.', Text), + ], + 'macro': [ + (r'%define.*\n', Comment.Preproc), + (r'%\{\!\?.*%define.*\}', Comment.Preproc), + (r'(%(?:if(?:n?arch)?|else(?:if)?|endif))(.*)$', + bygroups(Comment.Preproc, Text)), + ], + 'interpol': [ + (r'%\{?__[a-z_]+\}?', Name.Function), + (r'%\{?_([a-z_]+dir|[a-z_]+path|prefix)\}?', Keyword.Pseudo), + (r'%\{\?[A-Za-z0-9_]+\}', Name.Variable), + (r'\$\{?RPM_[A-Z0-9_]+\}?', Name.Variable.Global), + (r'%\{[a-zA-Z][a-zA-Z0-9_]+\}', Keyword.Constant), + ] + } diff --git a/pygments/lexers/parsers.py b/pygments/lexers/parsers.py index cd11b46c..c1ad710f 100644 --- a/pygments/lexers/parsers.py +++ b/pygments/lexers/parsers.py @@ -5,7 +5,7 @@ Lexers for parser generators. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -738,7 +738,8 @@ class TreetopBaseLexer(RegexLexer): (r'[()]', Punctuation), (r'[?+*/&!~]', Operator), (r'\[(?:\\.|\[:\^?[a-z]+:\]|[^\\\]])+\]', String.Regex), - (r'([0-9]*)(\.\.)([0-9]*)', bygroups(Number.Integer, Operator, Number.Integer)), + (r'([0-9]*)(\.\.)([0-9]*)', + bygroups(Number.Integer, Operator, Number.Integer)), (r'(<)([^>]+)(>)', bygroups(Punctuation, Name.Class, Punctuation)), (r'{', Punctuation, 'inline_module'), (r'\.', String.Regex), diff --git a/pygments/lexers/shell.py b/pygments/lexers/shell.py index 52df5f00..803005f3 100644 --- a/pygments/lexers/shell.py +++ b/pygments/lexers/shell.py @@ -5,7 +5,7 @@ Lexers for various shells. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/special.py b/pygments/lexers/special.py index 785ab73c..9b3cd508 100644 --- a/pygments/lexers/special.py +++ b/pygments/lexers/special.py @@ -5,7 +5,7 @@ Special lexers. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/sql.py b/pygments/lexers/sql.py index e7e02fce..dcfd8fa8 100644 --- a/pygments/lexers/sql.py +++ b/pygments/lexers/sql.py @@ -34,7 +34,7 @@ The ``tests/examplefiles`` contains a few test files with data to be parsed by these lexers. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/lexers/templates.py b/pygments/lexers/templates.py index 0e6f46d7..65627b0d 100644 --- a/pygments/lexers/templates.py +++ b/pygments/lexers/templates.py @@ -5,7 +5,7 @@ Lexers for various template engines' markup. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -1651,7 +1651,6 @@ class LassoHtmlLexer(DelegatingLexer): 'application/x-httpd-lasso[89]'] def __init__(self, **options): - options['requiredelimiters'] = True super(LassoHtmlLexer, self).__init__(HtmlLexer, LassoLexer, **options) def analyse_text(text): @@ -1678,7 +1677,6 @@ class LassoXmlLexer(DelegatingLexer): mimetypes = ['application/xml+lasso'] def __init__(self, **options): - options['requiredelimiters'] = True super(LassoXmlLexer, self).__init__(XmlLexer, LassoLexer, **options) def analyse_text(text): diff --git a/pygments/lexers/text.py b/pygments/lexers/text.py index ca50665e..79797874 100644 --- a/pygments/lexers/text.py +++ b/pygments/lexers/text.py @@ -5,7 +5,7 @@ Lexers for non-source code file types. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -25,7 +25,7 @@ __all__ = ['IniLexer', 'PropertiesLexer', 'SourcesListLexer', 'BaseMakefileLexer 'RstLexer', 'VimLexer', 'GettextLexer', 'SquidConfLexer', 'DebianControlLexer', 'DarcsPatchLexer', 'YamlLexer', 'LighttpdConfLexer', 'NginxConfLexer', 'CMakeLexer', 'HttpLexer', - 'PyPyLogLexer', 'RegeditLexer'] + 'PyPyLogLexer', 'RegeditLexer', 'HxmlLexer'] class IniLexer(RegexLexer): @@ -1749,8 +1749,8 @@ class PyPyLogLexer(RegexLexer): ], "jit-log": [ (r"\[\w+\] jit-log-.*?}$", Keyword, "#pop"), - (r"^\+\d+: ", Comment), + (r"--end of the loop--", Comment), (r"[ifp]\d+", Name), (r"ptr\d+", Name), (r"(\()(\w+(?:\.\w+)?)(\))", @@ -1760,7 +1760,7 @@ class PyPyLogLexer(RegexLexer): (r"-?\d+", Number.Integer), (r"'.*'", String), (r"(None|descr|ConstClass|ConstPtr|TargetToken)", Name), - (r"<.*?>", Name.Builtin), + (r"<.*?>+", Name.Builtin), (r"(label|debug_merge_point|jump|finish)", Name.Class), (r"(int_add_ovf|int_add|int_sub_ovf|int_sub|int_mul_ovf|int_mul|" r"int_floordiv|int_mod|int_lshift|int_rshift|int_and|int_or|" @@ -1800,3 +1800,44 @@ class PyPyLogLexer(RegexLexer): (r"#.*?$", Comment), ], } + + +class HxmlLexer(RegexLexer): + """ + Lexer for `haXe build <http://haxe.org/doc/compiler>`_ files. + + *New in Pygments 1.6.* + """ + name = 'Hxml' + aliases = ['haxeml', 'hxml'] + filenames = ['*.hxml'] + + tokens = { + 'root': [ + # Seperator + (r'(--)(next)', bygroups(Punctuation, Generic.Heading)), + # Compiler switches with one dash + (r'(-)(prompt|debug|v)', bygroups(Punctuation, Keyword.Keyword)), + # Compilerswitches with two dashes + (r'(--)(neko-source|flash-strict|flash-use-stage|no-opt|no-traces|' + r'no-inline|times|no-output)', bygroups(Punctuation, Keyword)), + # Targets and other options that take an argument + (r'(-)(cpp|js|neko|x|as3|swf9?|swf-lib|php|xml|main|lib|D|resource|' + r'cp|cmd)( +)(.+)', + bygroups(Punctuation, Keyword, Whitespace, String)), + # Options that take only numerical arguments + (r'(-)(swf-version)( +)(\d+)', + bygroups(Punctuation, Keyword, Number.Integer)), + # An Option that defines the size, the fps and the background + # color of an flash movie + (r'(-)(swf-header)( +)(\d+)(:)(\d+)(:)(\d+)(:)([A-Fa-f0-9]{6})', + bygroups(Punctuation, Keyword, Whitespace, Number.Integer, + Punctuation, Number.Integer, Punctuation, Number.Integer, + Punctuation, Number.Hex)), + # options with two dashes that takes arguments + (r'(--)(js-namespace|php-front|php-lib|remap|gen-hx-classes)( +)' + r'(.+)', bygroups(Punctuation, Keyword, Whitespace, String)), + # Single line comment, multiline ones are not allowed. + (r'#.*', Comment.Single) + ] + } diff --git a/pygments/lexers/web.py b/pygments/lexers/web.py index 79245d34..6df65131 100644 --- a/pygments/lexers/web.py +++ b/pygments/lexers/web.py @@ -5,7 +5,7 @@ Lexers for web-related languages and markup. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -27,7 +27,7 @@ __all__ = ['HtmlLexer', 'XmlLexer', 'JavascriptLexer', 'JsonLexer', 'CssLexer', 'MxmlLexer', 'HaxeLexer', 'HamlLexer', 'SassLexer', 'ScssLexer', 'ObjectiveJLexer', 'CoffeeScriptLexer', 'LiveScriptLexer', 'DuelLexer', 'ScamlLexer', 'JadeLexer', 'XQueryLexer', - 'DtdLexer', 'DartLexer', 'LassoLexer'] + 'DtdLexer', 'DartLexer', 'LassoLexer', 'QmlLexer'] class JavascriptLexer(RegexLexer): @@ -969,7 +969,8 @@ class DtdLexer(RegexLexer): 'attlist': [ include('common'), - (r'CDATA|IDREFS|IDREF|ID|NMTOKENS|NMTOKEN|ENTITIES|ENTITY|NOTATION', Keyword.Constant), + (r'CDATA|IDREFS|IDREF|ID|NMTOKENS|NMTOKEN|ENTITIES|ENTITY|NOTATION', + Keyword.Constant), (r'#REQUIRED|#IMPLIED|#FIXED', Keyword.Constant), (r'xml:space|xml:lang', Keyword.Reserved), (r'[^>\s\|()?+*,]+', Name.Attribute), @@ -1762,7 +1763,7 @@ class ScssLexer(RegexLexer): (r'(@include)( [\w-]+)', bygroups(Keyword, Name.Decorator), 'value'), (r'@extend', Keyword, 'selector'), (r'@[a-z0-9_-]+', Keyword, 'selector'), - (r'(\$[\w-]\w*)([ \t]*:)', bygroups(Name.Variable, Operator), 'value'), + (r'(\$[\w-]*\w)([ \t]*:)', bygroups(Name.Variable, Operator), 'value'), (r'(?=[^;{}][;}])', Name.Attribute, 'attr'), (r'(?=[^;{}:]+:[^a-z])', Name.Attribute, 'attr'), (r'', Text, 'selector'), @@ -3015,19 +3016,14 @@ class LassoLexer(RegexLexer): tokens = { 'root': [ (r'^#!.+lasso9\b', Comment.Preproc, 'lasso'), - (r'\s+', Other), (r'\[no_square_brackets\]', Comment.Preproc, 'nosquarebrackets'), (r'\[noprocess\]', Comment.Preproc, ('delimiters', 'noprocess')), (r'\[', Comment.Preproc, ('delimiters', 'squarebrackets')), (r'<\?(LassoScript|lasso|=)', Comment.Preproc, ('delimiters', 'anglebrackets')), (r'<', Other, 'delimiters'), - include('lasso'), - ], - 'nosquarebrackets': [ - (r'<\?(LassoScript|lasso|=)', Comment.Preproc, 'anglebrackets'), - (r'<', Other), - (r'[^<]+', Other), + (r'\s+', Other), + (r'', Other, ('delimiters', 'lassofile')), ], 'delimiters': [ (r'\[no_square_brackets\]', Comment.Preproc, 'nosquarebrackets'), @@ -3037,6 +3033,11 @@ class LassoLexer(RegexLexer): (r'<', Other), (r'[^[<]+', Other), ], + 'nosquarebrackets': [ + (r'<\?(LassoScript|lasso|=)', Comment.Preproc, 'anglebrackets'), + (r'<', Other), + (r'[^<]+', Other), + ], 'noprocess': [ (r'\[/noprocess\]', Comment.Preproc, '#pop'), (r'\[', Other), @@ -3050,12 +3051,20 @@ class LassoLexer(RegexLexer): (r'\?>', Comment.Preproc, '#pop'), include('lasso'), ], - 'lasso': [ - # whitespace/comments + 'lassofile': [ + (r'\]', Comment.Preproc, '#pop'), + (r'\?>', Comment.Preproc, '#pop'), + include('lasso'), + ], + 'whitespacecomments': [ (r'\s+', Text), (r'//.*?\n', Comment.Single), (r'/\*\*!.*?\*/', String.Doc), (r'/\*.*?\*/', Comment.Multiline), + ], + 'lasso': [ + # whitespace/comments + include('whitespacecomments'), # literals (r'\d*\.\d+(e[+-]?\d+)?', Number.Float), @@ -3098,10 +3107,9 @@ class LassoLexer(RegexLexer): (r'(define)(\s+)([a-z_][\w.]*)', bygroups(Keyword.Declaration, Text, Name.Function), 'signature'), - (r'(public|protected|private|provide)(\s+)([a-z_][\w.]*=?|' - r'[-+*/%<>]|==)(\s*)(\()', - bygroups(Keyword, Text, Name.Function, Text, Punctuation), - ('signature', 'parameter')), + (r'(public|protected|private|provide)(\s+)(([a-z_][\w.]*=?|' + r'[-+*/%<>]|==)(?=\s*\())', bygroups(Keyword, Text, Name.Function), + 'signature'), (r'(public|protected|private)(\s+)([a-z_][\w.]*)', bygroups(Keyword, Text, Name.Function)), @@ -3112,6 +3120,9 @@ class LassoLexer(RegexLexer): r'null)\b', Keyword.Type), (r'([a-z_][\w.]*)(\s+)(in)\b', bygroups(Name, Text, Keyword)), (r'(let|into)(\s+)([a-z_][\w.]*)', bygroups(Keyword, Text, Name)), + (r'require\b', Keyword, 'requiresection'), + (r'(/?)(Namespace_Using)\b', + bygroups(Punctuation, Keyword.Namespace)), (r'(/?)(Cache|Database_Names|Database_SchemaNames|' r'Database_TableNames|Define_Tag|Define_Type|Email_Batch|' r'Encode_Set|HTML_Comment|Handle|Handle_Error|Header|If|Inline|' @@ -3119,19 +3130,21 @@ class LassoLexer(RegexLexer): r'Link_CurrentRecord|Link_Detail|Link_FirstGroup|' r'Link_FirstRecord|Link_LastGroup|Link_LastRecord|Link_NextGroup|' r'Link_NextRecord|Link_PrevGroup|Link_PrevRecord|Log|Loop|' - r'Namespace_Using|NoProcess|Output_None|Portal|Private|Protect|' - r'Records|Referer|Referrer|Repeating|ResultSet|Rows|Search_Args|' - r'Search_Arguments|Select|Sort_Args|Sort_Arguments|Thread_Atomic|' - r'Value_List|While|Abort|Case|Else|If_Empty|If_False|If_Null|' - r'If_True|Loop_Abort|Loop_Continue|Loop_Count|Params|Params_Up|' - r'Return|Return_Value|Run_Children|SOAP_DefineTag|' - r'SOAP_LastRequest|SOAP_LastResponse|Tag_Name|ascending|average|' - r'by|define|descending|do|equals|frozen|group|handle_failure|' - r'import|in|into|join|let|match|max|min|on|order|parent|protected|' - r'provide|public|require|skip|split_thread|sum|take|thread|to|' - r'trait|type|where|with|yield)\b', bygroups(Punctuation, Keyword)), + r'NoProcess|Output_None|Portal|Private|Protect|Records|Referer|' + r'Referrer|Repeating|ResultSet|Rows|Search_Args|Search_Arguments|' + r'Select|Sort_Args|Sort_Arguments|Thread_Atomic|Value_List|While|' + r'Abort|Case|Else|If_Empty|If_False|If_Null|If_True|Loop_Abort|' + r'Loop_Continue|Loop_Count|Params|Params_Up|Return|Return_Value|' + r'Run_Children|SOAP_DefineTag|SOAP_LastRequest|SOAP_LastResponse|' + r'Tag_Name|ascending|average|by|define|descending|do|equals|' + r'frozen|group|handle_failure|import|in|into|join|let|match|max|' + r'min|on|order|parent|protected|provide|public|require|skip|' + r'split_thread|sum|take|thread|to|trait|type|where|with|yield)\b', + bygroups(Punctuation, Keyword)), # other + (r'(([a-z_][\w.]*=?|[-+*/%<>]|==)(?=\s*\([^)]*\)\s*=>))', + Name.Function, 'signature'), (r'(and|or|not)\b', Operator.Word), (r'([a-z_][\w.]*)(\s*)(::\s*)([a-z_][\w.]*)(\s*)(=)', bygroups(Name, Text, Punctuation, Name.Label, Text, Operator)), @@ -3160,16 +3173,34 @@ class LassoLexer(RegexLexer): r'[abefnrtv?\"\'\\]|$)', String.Escape), ], 'signature': [ - (r'[(,]', Punctuation, 'parameter'), (r'=>', Operator, '#pop'), + (r'\)', Punctuation, '#pop'), + (r'[(,]', Punctuation, 'parameter'), include('lasso'), ], 'parameter': [ - (r'\.\.\.', Name.Builtin.Pseudo), - (r'-?[a-z_][\w.]*', Name.Attribute, '#pop'), (r'\)', Punctuation, '#pop'), + (r'-?[a-z_][\w.]*', Name.Attribute, '#pop'), + (r'\.\.\.', Name.Builtin.Pseudo), include('lasso'), ], + 'requiresection': [ + (r'(([a-z_][\w.]*=?|[-+*/%<>]|==)(?=\s*\())', Name, 'requiresignature'), + (r'(([a-z_][\w.]*=?|[-+*/%<>]|==)(?=(\s*::\s*[\w.]+)?\s*,))', Name), + (r'[a-z_][\w.]*=?|[-+*/%<>]|==', Name, '#pop'), + (r'(::\s*)([a-z_][\w.]*)', bygroups(Punctuation, Name.Label)), + (r',', Punctuation), + include('whitespacecomments'), + ], + 'requiresignature': [ + (r'(\)(?=(\s*::\s*[\w.]+)?\s*,))', Punctuation, '#pop'), + (r'\)', Punctuation, '#pop:2'), + (r'-?[a-z_][\w.]*', Name.Attribute), + (r'(::\s*)([a-z_][\w.]*)', bygroups(Punctuation, Name.Label)), + (r'\.\.\.', Name.Builtin.Pseudo), + (r'[(,]', Punctuation), + include('whitespacecomments'), + ], } def __init__(self, **options): @@ -3208,3 +3239,75 @@ class LassoLexer(RegexLexer): if re.search(r'\[\n|\?>', text): rv += 0.4 return rv + + +class QmlLexer(RegexLexer): + """ + For QML files. See http://doc.qt.digia.com/4.7/qdeclarativeintroduction.html. + + *New in Pygments 1.6.* + """ + + # QML is based on javascript, so much of this is taken from the + # JavascriptLexer above. + + name = 'QML' + aliases = ['qml', 'Qt Meta Language', 'Qt modeling Language'] + filenames = ['*.qml',] + mimetypes = [ 'application/x-qml',] + + + # pasted from JavascriptLexer, with some additions + flags = re.DOTALL + tokens = { + 'commentsandwhitespace': [ + (r'\s+', Text), + (r'<!--', Comment), + (r'//.*?\n', Comment.Single), + (r'/\*.*?\*/', Comment.Multiline) + ], + 'slashstartsregex': [ + include('commentsandwhitespace'), + (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/' + r'([gim]+\b|\B)', String.Regex, '#pop'), + (r'(?=/)', Text, ('#pop', 'badregex')), + (r'', Text, '#pop') + ], + 'badregex': [ + (r'\n', Text, '#pop') + ], + 'root' : [ + (r'^(?=\s|/|<!--)', Text, 'slashstartsregex'), + include('commentsandwhitespace'), + (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|' + r'(<<|>>>?|==?|!=?|[-<>+*%&\|\^/])=?', Operator, 'slashstartsregex'), + (r'[{(\[;,]', Punctuation, 'slashstartsregex'), + (r'[})\].]', Punctuation), + + #QML insertions + (r'\bid\s*:\s*[A-Za-z][_A-Za-z.0-9]*',Keyword.Declaration, 'slashstartsregex'), + (r'\b[A-Za-z][_A-Za-z.0-9]*\s*:',Keyword, 'slashstartsregex'), + + #the rest from JavascriptLexer + (r'(for|in|while|do|break|return|continue|switch|case|default|if|else|' + r'throw|try|catch|finally|new|delete|typeof|instanceof|void|' + r'this)\b', Keyword, 'slashstartsregex'), + (r'(var|let|with|function)\b', Keyword.Declaration, 'slashstartsregex'), + (r'(abstract|boolean|byte|char|class|const|debugger|double|enum|export|' + r'extends|final|float|goto|implements|import|int|interface|long|native|' + r'package|private|protected|public|short|static|super|synchronized|throws|' + r'transient|volatile)\b', Keyword.Reserved), + (r'(true|false|null|NaN|Infinity|undefined)\b', Keyword.Constant), + (r'(Array|Boolean|Date|Error|Function|Math|netscape|' + r'Number|Object|Packages|RegExp|String|sun|decodeURI|' + r'decodeURIComponent|encodeURI|encodeURIComponent|' + r'Error|eval|isFinite|isNaN|parseFloat|parseInt|document|this|' + r'window)\b', Name.Builtin), + (r'[$a-zA-Z_][a-zA-Z0-9_]*', Name.Other), + (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), + (r'0x[0-9a-fA-F]+', Number.Hex), + (r'[0-9]+', Number.Integer), + (r'"(\\\\|\\"|[^"])*"', String.Double), + (r"'(\\\\|\\'|[^'])*'", String.Single), + ] + } diff --git a/pygments/plugin.py b/pygments/plugin.py index ea606b9c..58662e96 100644 --- a/pygments/plugin.py +++ b/pygments/plugin.py @@ -32,7 +32,7 @@ yourfilter = yourfilter:YourFilter - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ try: diff --git a/pygments/scanner.py b/pygments/scanner.py index 2acd6e6b..f469e694 100644 --- a/pygments/scanner.py +++ b/pygments/scanner.py @@ -12,7 +12,7 @@ Have a look at the `DelphiLexer` to get an idea of how to use this scanner. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re diff --git a/pygments/style.py b/pygments/style.py index 0d9e6eb6..0fc01b40 100644 --- a/pygments/style.py +++ b/pygments/style.py @@ -5,7 +5,7 @@ Basic style object. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/__init__.py b/pygments/styles/__init__.py index 2be93da4..3d6ef73c 100644 --- a/pygments/styles/__init__.py +++ b/pygments/styles/__init__.py @@ -5,7 +5,7 @@ Contains built-in styles. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/autumn.py b/pygments/styles/autumn.py index a311487c..3960536b 100644 --- a/pygments/styles/autumn.py +++ b/pygments/styles/autumn.py @@ -5,7 +5,7 @@ A colorful style, inspired by the terminal highlighting style. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/borland.py b/pygments/styles/borland.py index ce6120b5..9858034e 100644 --- a/pygments/styles/borland.py +++ b/pygments/styles/borland.py @@ -5,7 +5,7 @@ Style similar to the style used in the Borland IDEs. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/bw.py b/pygments/styles/bw.py index fdec8a2f..170442ad 100644 --- a/pygments/styles/bw.py +++ b/pygments/styles/bw.py @@ -5,7 +5,7 @@ Simple black/white only style. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/colorful.py b/pygments/styles/colorful.py index 819e81ba..eb595467 100644 --- a/pygments/styles/colorful.py +++ b/pygments/styles/colorful.py @@ -5,7 +5,7 @@ A colorful style, inspired by CodeRay. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/default.py b/pygments/styles/default.py index d90f08d8..77bdac0d 100644 --- a/pygments/styles/default.py +++ b/pygments/styles/default.py @@ -5,7 +5,7 @@ The default highlighting style. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/emacs.py b/pygments/styles/emacs.py index 96640f87..9f8b4074 100644 --- a/pygments/styles/emacs.py +++ b/pygments/styles/emacs.py @@ -5,7 +5,7 @@ A highlighting style for Pygments, inspired by Emacs. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/friendly.py b/pygments/styles/friendly.py index 25dda6ca..732a1252 100644 --- a/pygments/styles/friendly.py +++ b/pygments/styles/friendly.py @@ -5,7 +5,7 @@ A modern style based on the VIM pyte theme. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/fruity.py b/pygments/styles/fruity.py index 6f6f2864..45334159 100644 --- a/pygments/styles/fruity.py +++ b/pygments/styles/fruity.py @@ -5,7 +5,7 @@ pygments version of my "fruity" vim theme. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/manni.py b/pygments/styles/manni.py index 2f689419..036a2120 100644 --- a/pygments/styles/manni.py +++ b/pygments/styles/manni.py @@ -8,7 +8,7 @@ This is a port of the style used in the `php port`_ of pygments by Manni. The style is called 'default' there. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/monokai.py b/pygments/styles/monokai.py index b5338122..31dc83b2 100644 --- a/pygments/styles/monokai.py +++ b/pygments/styles/monokai.py @@ -7,7 +7,7 @@ http://www.monokai.nl/blog/2006/07/15/textmate-color-theme/ - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/murphy.py b/pygments/styles/murphy.py index 03494fb6..dbf4eba9 100644 --- a/pygments/styles/murphy.py +++ b/pygments/styles/murphy.py @@ -5,7 +5,7 @@ Murphy's style from CodeRay. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/native.py b/pygments/styles/native.py index 53749e64..0de84386 100644 --- a/pygments/styles/native.py +++ b/pygments/styles/native.py @@ -5,7 +5,7 @@ pygments version of my "native" vim theme. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/pastie.py b/pygments/styles/pastie.py index ca8f6206..2a2f386f 100644 --- a/pygments/styles/pastie.py +++ b/pygments/styles/pastie.py @@ -7,7 +7,7 @@ .. _pastie: http://pastie.caboo.se/ - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/perldoc.py b/pygments/styles/perldoc.py index 73f9bd3c..b8b67b29 100644 --- a/pygments/styles/perldoc.py +++ b/pygments/styles/perldoc.py @@ -7,7 +7,7 @@ .. _perldoc: http://perldoc.perl.org/ - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/rrt.py b/pygments/styles/rrt.py index 1c363858..1a2fc6a4 100644 --- a/pygments/styles/rrt.py +++ b/pygments/styles/rrt.py @@ -5,7 +5,7 @@ pygments "rrt" theme, based on Zap and Emacs defaults. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/tango.py b/pygments/styles/tango.py index fea7ae5d..7b1c4f3c 100644 --- a/pygments/styles/tango.py +++ b/pygments/styles/tango.py @@ -33,7 +33,7 @@ have been chosen to have the same style. Similarly, keywords (Keyword.*), and Operator.Word (and, or, in) have been assigned the same style. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/trac.py b/pygments/styles/trac.py index 468a5822..714e36cc 100644 --- a/pygments/styles/trac.py +++ b/pygments/styles/trac.py @@ -5,7 +5,7 @@ Port of the default trac highlighter design. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/vim.py b/pygments/styles/vim.py index 0aab8bcd..a5462db3 100644 --- a/pygments/styles/vim.py +++ b/pygments/styles/vim.py @@ -5,7 +5,7 @@ A highlighting style for Pygments, inspired by vim. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/styles/vs.py b/pygments/styles/vs.py index 5cdda4ae..14a56faa 100644 --- a/pygments/styles/vs.py +++ b/pygments/styles/vs.py @@ -5,7 +5,7 @@ Simple style with MS Visual Studio colors. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/token.py b/pygments/token.py index cf78cca2..19a83f2e 100644 --- a/pygments/token.py +++ b/pygments/token.py @@ -5,7 +5,7 @@ Basic token types and the standard tokens. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/pygments/unistring.py b/pygments/unistring.py index b6f53e89..4b07028e 100644 --- a/pygments/unistring.py +++ b/pygments/unistring.py @@ -8,7 +8,7 @@ Inspired by chartypes_create.py from the MoinMoin project. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ from pygments.util import u_prefix diff --git a/pygments/util.py b/pygments/util.py index 127f6e87..caac1144 100644 --- a/pygments/util.py +++ b/pygments/util.py @@ -5,7 +5,7 @@ Utility functions. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/scripts/check_sources.py b/scripts/check_sources.py index e1c7544d..d9e5c2ae 100755 --- a/scripts/check_sources.py +++ b/scripts/check_sources.py @@ -7,7 +7,7 @@ Make sure each Python file has a correct file header including copyright and license information. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -30,7 +30,7 @@ def checker(*suffixes, **kwds): name_mail_re = r'[\w ]+(<.*?>)?' -copyright_re = re.compile(r'^ :copyright: Copyright 2006-2012 by ' +copyright_re = re.compile(r'^ :copyright: Copyright 2006-2013 by ' r'the Pygments team, see AUTHORS\.$', re.UNICODE) copyright_2_re = re.compile(r'^ %s(, %s)*[,.]$' % (name_mail_re, name_mail_re), re.UNICODE) diff --git a/scripts/find_codetags.py b/scripts/find_codetags.py index 35db143f..2fb18333 100755 --- a/scripts/find_codetags.py +++ b/scripts/find_codetags.py @@ -7,7 +7,7 @@ Find code tags in specified files and/or directories and create a report in HTML format. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/scripts/find_error.py b/scripts/find_error.py index 453f16ed..8e4c79db 100755 --- a/scripts/find_error.py +++ b/scripts/find_error.py @@ -8,7 +8,7 @@ the text where Error tokens are being generated, along with some context. - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -23,7 +23,7 @@ .. _Pygments tip: http://bitbucket.org/birkenfeld/pygments-main/get/tip.zip#egg=Pygments-dev - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/examplefiles/BOM.js b/tests/examplefiles/BOM.js new file mode 100644 index 00000000..930599c1 --- /dev/null +++ b/tests/examplefiles/BOM.js @@ -0,0 +1 @@ +/* There is a BOM at the beginning of this file. */
\ No newline at end of file diff --git a/tests/examplefiles/bigtest.nsi b/tests/examplefiles/bigtest.nsi new file mode 100644 index 00000000..62f5211c --- /dev/null +++ b/tests/examplefiles/bigtest.nsi @@ -0,0 +1,308 @@ +; bigtest.nsi +; +; This script attempts to test most of the functionality of the NSIS exehead. + +;-------------------------------- + +!ifdef HAVE_UPX +!packhdr tmp.dat "upx\upx -9 tmp.dat" +!endif + +!ifdef NOCOMPRESS +SetCompress off +!endif + +;-------------------------------- + +Name "BigNSISTest" +Caption "NSIS Big Test" +Icon "${NSISDIR}\Contrib\Graphics\Icons\nsis1-install.ico" +OutFile "bigtest.exe" + +SetDateSave on +SetDatablockOptimize on +CRCCheck on +SilentInstall normal +BGGradient 000000 800000 FFFFFF +InstallColors FF8080 000030 +XPStyle on + +InstallDir "$PROGRAMFILES\NSISTest\BigNSISTest" +InstallDirRegKey HKLM "Software\NSISTest\BigNSISTest" "Install_Dir" + +CheckBitmap "${NSISDIR}\Contrib\Graphics\Checks\classic-cross.bmp" + +LicenseText "A test text, make sure it's all there" +LicenseData "bigtest.nsi" + +RequestExecutionLevel admin + +;-------------------------------- + +Page license +Page components +Page directory +Page instfiles + +UninstPage uninstConfirm +UninstPage instfiles + +;-------------------------------- + +!ifndef NOINSTTYPES ; only if not defined + InstType "Most" + InstType "Full" + InstType "More" + InstType "Base" + ;InstType /NOCUSTOM + ;InstType /COMPONENTSONLYONCUSTOM +!endif + +AutoCloseWindow false +ShowInstDetails show + +;-------------------------------- + +Section "" ; empty string makes it hidden, so would starting with - + + ; write reg info + StrCpy $1 "POOOOOOOOOOOP" + DetailPrint "I like to be able to see what is going on (debug) $1" + WriteRegStr HKLM SOFTWARE\NSISTest\BigNSISTest "Install_Dir" "$INSTDIR" + + ; write uninstall strings + WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\BigNSISTest" "DisplayName" "BigNSISTest (remove only)" + WriteRegStr HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\BigNSISTest" "UninstallString" '"$INSTDIR\bt-uninst.exe"' + + SetOutPath $INSTDIR + File /a "silent.nsi" + CreateDirectory "$INSTDIR\MyProjectFamily\MyProject" ; 2 recursively create a directory for fun. + WriteUninstaller "bt-uninst.exe" + + Nop ; for fun + +SectionEnd + +Section "TempTest" + +SectionIn 1 2 3 + Start: MessageBox MB_OK "Start:" + + MessageBox MB_YESNO "Goto MyLabel" IDYES MyLabel + + MessageBox MB_OK "Right before MyLabel:" + + MyLabel: MessageBox MB_OK "MyLabel:" + + MessageBox MB_OK "Right after MyLabel:" + + MessageBox MB_YESNO "Goto Start:?" IDYES Start + +SectionEnd + +SectionGroup /e SectionGroup1 + +Section "Test Registry/INI functions" + +SectionIn 1 4 3 + + WriteRegStr HKLM SOFTWARE\NSISTest\BigNSISTest "StrTest_INSTDIR" "$INSTDIR" + WriteRegDword HKLM SOFTWARE\NSISTest\BigNSISTest "DwordTest_0xDEADBEEF" 0xdeadbeef + WriteRegDword HKLM SOFTWARE\NSISTest\BigNSISTest "DwordTest_123456" 123456 + WriteRegDword HKLM SOFTWARE\NSISTest\BigNSISTest "DwordTest_0123" 0123 + WriteRegBin HKLM SOFTWARE\NSISTest\BigNSISTest "BinTest_deadbeef01f00dbeef" "DEADBEEF01F00DBEEF" + StrCpy $8 "$SYSDIR\IniTest" + WriteINIStr "$INSTDIR\test.ini" "MySection" "Value1" $8 + WriteINIStr "$INSTDIR\test.ini" "MySectionIni" "Value1" $8 + WriteINIStr "$INSTDIR\test.ini" "MySectionIni" "Value2" $8 + WriteINIStr "$INSTDIR\test.ini" "IniOn" "Value1" $8 + + Call MyFunctionTest + + DeleteINIStr "$INSTDIR\test.ini" "IniOn" "Value1" + DeleteINISec "$INSTDIR\test.ini" "MySectionIni" + + ReadINIStr $1 "$INSTDIR\test.ini" "MySectionIni" "Value1" + StrCmp $1 "" INIDelSuccess + MessageBox MB_OK "DeleteINISec failed" + INIDelSuccess: + + ClearErrors + ReadRegStr $1 HKCR "software\microsoft" xyz_cc_does_not_exist + IfErrors 0 NoError + MessageBox MB_OK "could not read from HKCR\software\microsoft\xyz_cc_does_not_exist" + Goto ErrorYay + NoError: + MessageBox MB_OK "read '$1' from HKCR\software\microsoft\xyz_cc_does_not_exist" + ErrorYay: + +SectionEnd + +Section "Test CreateShortCut" + + SectionIn 1 2 3 + + Call CSCTest + +SectionEnd + +SectionGroup Group2 + +Section "Test Branching" + + BeginTestSection: + SectionIn 1 2 3 + + SetOutPath $INSTDIR + + IfFileExists "$INSTDIR\LogicLib.nsi" 0 BranchTest69 + + MessageBox MB_YESNO|MB_ICONQUESTION "Would you like to overwrite $INSTDIR\LogicLib.nsi?" IDNO NoOverwrite ; skipped if file doesn't exist + + BranchTest69: + + SetOverwrite ifnewer ; NOT AN INSTRUCTION, NOT COUNTED IN SKIPPINGS + + NoOverwrite: + + File "LogicLib.nsi" ; skipped if answered no + SetOverwrite try ; NOT AN INSTRUCTION, NOT COUNTED IN SKIPPINGS + + MessageBox MB_YESNO|MB_ICONQUESTION "Would you like to skip the rest of this section?" IDYES EndTestBranch + MessageBox MB_YESNO|MB_ICONQUESTION "Would you like to go back to the beginning of this section?" IDYES BeginTestSection + MessageBox MB_YESNO|MB_ICONQUESTION "Would you like to hide the installer and wait five seconds?" IDNO NoHide + + HideWindow + Sleep 5000 + BringToFront + + NoHide: + + MessageBox MB_YESNO|MB_ICONQUESTION "Would you like to call the function 5 times?" IDNO NoRecurse + + StrCpy $1 "x" + + LoopTest: + + Call myfunc + StrCpy $1 "x$1" + StrCmp $1 "xxxxxx" 0 LoopTest + + NoRecurse: + + EndTestBranch: + +SectionEnd + +SectionGroupEnd + +Section "Test CopyFiles" + + SectionIn 1 2 3 + + SetOutPath $INSTDIR\cpdest + CopyFiles "$WINDIR\*.ini" "$INSTDIR\cpdest" 0 + +SectionEnd + +SectionGroupEnd + +Section "Test Exec functions" TESTIDX + + SectionIn 1 2 3 + + SearchPath $1 notepad.exe + + MessageBox MB_OK "notepad.exe=$1" + Exec '"$1"' + ExecShell "open" '"$INSTDIR"' + Sleep 500 + BringToFront + +SectionEnd + +Section "Test ActiveX control registration" + + SectionIn 2 + + UnRegDLL "$SYSDIR\spin32.ocx" + Sleep 1000 + RegDLL "$SYSDIR\spin32.ocx" + Sleep 1000 + +SectionEnd + +;-------------------------------- + +Function "CSCTest" + + CreateDirectory "$SMPROGRAMS\Big NSIS Test" + SetOutPath $INSTDIR ; for working directory + CreateShortCut "$SMPROGRAMS\Big NSIS Test\Uninstall BIG NSIS Test.lnk" "$INSTDIR\bt-uninst.exe" ; use defaults for parameters, icon, etc. + ; this one will use notepad's icon, start it minimized, and give it a hotkey (of Ctrl+Shift+Q) + CreateShortCut "$SMPROGRAMS\Big NSIS Test\silent.nsi.lnk" "$INSTDIR\silent.nsi" "" "$WINDIR\notepad.exe" 0 SW_SHOWMINIMIZED CONTROL|SHIFT|Q + CreateShortCut "$SMPROGRAMS\Big NSIS Test\TheDir.lnk" "$INSTDIR\" "" "" 0 SW_SHOWMAXIMIZED CONTROL|SHIFT|Z + +FunctionEnd + +Function myfunc + + StrCpy $2 "MyTestVar=$1" + MessageBox MB_OK "myfunc: $2" + +FunctionEnd + +Function MyFunctionTest + + ReadINIStr $1 "$INSTDIR\test.ini" "MySectionIni" "Value1" + StrCmp $1 $8 NoFailedMsg + MessageBox MB_OK "WriteINIStr failed" + + NoFailedMsg: + +FunctionEnd + +Function .onSelChange + + SectionGetText ${TESTIDX} $0 + StrCmp $0 "" e + SectionSetText ${TESTIDX} "" + Goto e2 +e: + SectionSetText ${TESTIDX} "TextInSection" +e2: + +FunctionEnd + +;-------------------------------- + +; Uninstaller + +UninstallText "This will uninstall example2. Hit next to continue." +UninstallIcon "${NSISDIR}\Contrib\Graphics\Icons\nsis1-uninstall.ico" + +Section "Uninstall" + + DeleteRegKey HKLM "Software\Microsoft\Windows\CurrentVersion\Uninstall\BigNSISTest" + DeleteRegKey HKLM "SOFTWARE\NSISTest\BigNSISTest" + Delete "$INSTDIR\silent.nsi" + Delete "$INSTDIR\LogicLib.nsi" + Delete "$INSTDIR\bt-uninst.exe" + Delete "$INSTDIR\test.ini" + Delete "$SMPROGRAMS\Big NSIS Test\*.*" + RMDir "$SMPROGRAMS\BiG NSIS Test" + + MessageBox MB_YESNO|MB_ICONQUESTION "Would you like to remove the directory $INSTDIR\cpdest?" IDNO NoDelete + Delete "$INSTDIR\cpdest\*.*" + RMDir "$INSTDIR\cpdest" ; skipped if no + NoDelete: + + RMDir "$INSTDIR\MyProjectFamily\MyProject" + RMDir "$INSTDIR\MyProjectFamily" + RMDir "$INSTDIR" + + IfFileExists "$INSTDIR" 0 NoErrorMsg + MessageBox MB_OK "Note: $INSTDIR could not be removed!" IDOK 0 ; skipped if file doesn't exist + NoErrorMsg: + +SectionEnd diff --git a/tests/examplefiles/classes.dylan b/tests/examplefiles/classes.dylan index 6dd55ff2..7bb88faa 100644 --- a/tests/examplefiles/classes.dylan +++ b/tests/examplefiles/classes.dylan @@ -1,12 +1,26 @@ +module: sample +comment: for make sure that does not highlight per word. + and it continues on to the next line. + define class <car> (<object>) slot serial-number :: <integer> = unique-serial-number(); - slot model-name :: <string>, + constant slot model-name :: <string>, required-init-keyword: model:; - slot has-sunroof? :: <boolean>, + each-subclass slot has-sunroof? :: <boolean>, init-keyword: sunroof?:, init-value: #f; + keyword foo:; + required keyword bar:; end class <car>; +define class <flying-car> (<car>) +end class <flying-car>; + +let flying-car = make(<flying-car>); +let car? :: <car?> = #f; +let prefixed-car :: <vehicles/car> = #f; +let model :: <car-911> = #f; + define constant $empty-string = ""; define constant $escaped-backslash = '\\'; define constant $escaped-single-quote = '\''; @@ -31,10 +45,79 @@ define method foo() => _ :: <boolean>; #t end method; -define method \+() -end; +define method \+ + (offset1 :: <time-offset>, offset2 :: <time-offset>) + => (sum :: <time-offset>) + let sum = offset1.total-seconds + offset2.total-seconds; + make(<time-offset>, total-seconds: sum); +end method \+; + +define method bar () + 1 | 2 & 3 +end + +if (bar) + 1 +elseif (foo) + 2 +else + 3 +end if; + +select (foo by instance?) + <integer> => 1 + otherwise => 3 +end select; + +/* multi + line + comment +*/ + +/* multi line comments + /* can be */ + nested */ define constant $symbol = #"hello"; define variable *vector* = #[3.5, 5] define constant $list = #(1, 2); define constant $pair = #(1 . "foo") + +let octal-number = #o238; +let hex-number = #x3890ADEF; +let binary-number = #b1010; +let float-exponent = 3.5e10; + +block (return) + with-lock (lock) + return(); + end; +exception (e :: <error>) + format-out("Oh no"); +cleanup + return(); +afterwards + format-out("Hello"); +end; + +define macro repeat + { repeat ?:body end } + => { block (?=stop!) + local method again() ?body; again() end; + again(); + end } +end macro repeat; + +define macro with-decoded-seconds + { + with-decoded-seconds + (?max:variable, ?min:variable, ?sec:variable = ?time:expression) + ?:body + end + } + => { + let (?max, ?min, ?sec) = decode-total-seconds(?time); + ?body + } +end macro; + diff --git a/tests/examplefiles/minehunt.qml b/tests/examplefiles/minehunt.qml new file mode 100644 index 00000000..548e7e89 --- /dev/null +++ b/tests/examplefiles/minehunt.qml @@ -0,0 +1,112 @@ + /**************************************************************************** + ** + ** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies). + ** All rights reserved. + ** Contact: Nokia Corporation (qt-info@nokia.com) + ** + ** This file is part of the QtDeclarative module of the Qt Toolkit. + ** + ** $QT_BEGIN_LICENSE:LGPL$ + ** GNU Lesser General Public License Usage + ** This file may be used under the terms of the GNU Lesser General Public + ** License version 2.1 as published by the Free Software Foundation and + ** appearing in the file LICENSE.LGPL included in the packaging of this + ** file. Please review the following information to ensure the GNU Lesser + ** General Public License version 2.1 requirements will be met: + ** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. + ** + ** In addition, as a special exception, Nokia gives you certain additional + ** rights. These rights are described in the Nokia Qt LGPL Exception + ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. + ** + ** GNU General Public License Usage + ** Alternatively, this file may be used under the terms of the GNU General + ** Public License version 3.0 as published by the Free Software Foundation + ** and appearing in the file LICENSE.GPL included in the packaging of this + ** file. Please review the following information to ensure the GNU General + ** Public License version 3.0 requirements will be met: + ** http://www.gnu.org/copyleft/gpl.html. + ** + ** Other Usage + ** Alternatively, this file may be used in accordance with the terms and + ** conditions contained in a signed written agreement between you and Nokia. + ** + ** + ** + ** + ** + ** $QT_END_LICENSE$ + ** + ****************************************************************************/ + + import QtQuick 1.0 + import "MinehuntCore" 1.0 + + Item { + id: field + property int clickx: 0 + property int clicky: 0 + + width: 450; height: 450 + + Image { source: "MinehuntCore/pics/background.png"; anchors.fill: parent; fillMode: Image.Tile } + + Grid { + anchors.horizontalCenter: parent.horizontalCenter + columns: 9; spacing: 1 + + Repeater { + id: repeater + model: tiles + delegate: Tile {} + } + } + + Row { + id: gamedata + x: 20; spacing: 20 + anchors.bottom: field.bottom; anchors.bottomMargin: 15 + + Image { + source: "MinehuntCore/pics/quit.png" + scale: quitMouse.pressed ? 0.8 : 1.0 + smooth: quitMouse.pressed + y: 10 + MouseArea { + id: quitMouse + anchors.fill: parent + anchors.margins: -20 + onClicked: Qt.quit() + } + } + Column { + spacing: 2 + Image { source: "MinehuntCore/pics/bomb-color.png" } + Text { anchors.horizontalCenter: parent.horizontalCenter; color: "white"; text: numMines } + } + + Column { + spacing: 2 + Image { source: "MinehuntCore/pics/flag-color.png" } + Text { anchors.horizontalCenter: parent.horizontalCenter; color: "white"; text: numFlags } + } + } + + Image { + anchors.bottom: field.bottom; anchors.bottomMargin: 15 + anchors.right: field.right; anchors.rightMargin: 20 + source: isPlaying ? 'MinehuntCore/pics/face-smile.png' : + hasWon ? 'MinehuntCore/pics/face-smile-big.png': 'MinehuntCore/pics/face-sad.png' + + MouseArea { anchors.fill: parent; onPressed: reset() } + } + Text { + anchors.centerIn: parent; width: parent.width - 20 + horizontalAlignment: Text.AlignHCenter + wrapMode: Text.WordWrap + text: "Minehunt demo has to be compiled to run.\n\nPlease see README." + color: "white"; font.bold: true; font.pixelSize: 14 + visible: tiles == undefined + } + + } diff --git a/tests/examplefiles/nanomsg.intr b/tests/examplefiles/nanomsg.intr new file mode 100644 index 00000000..d21f62cc --- /dev/null +++ b/tests/examplefiles/nanomsg.intr @@ -0,0 +1,95 @@ +module: nanomsg +synopsis: generated bindings for the nanomsg library +author: Bruce Mitchener, Jr. +copyright: See LICENSE file in this distribution. + +define simple-C-mapped-subtype <C-buffer-offset> (<C-char*>) + export-map <machine-word>, export-function: identity; +end; + +define interface + #include { + "sp/sp.h", + "sp/fanin.h", + "sp/inproc.h", + "sp/pair.h", + "sp/reqrep.h", + "sp/survey.h", + "sp/fanout.h", + "sp/ipc.h", + "sp/pubsub.h", + "sp/tcp.h" + }, + + exclude: { + "SP_HAUSNUMERO", + "SP_PAIR_ID", + "SP_PUBSUB_ID", + "SP_REQREP_ID", + "SP_FANIN_ID", + "SP_FANOUT_ID", + "SP_SURVEY_ID" + }, + + equate: {"char *" => <c-string>}, + + rename: { + "sp_recv" => %sp-recv, + "sp_send" => %sp-send, + "sp_setsockopt" => %sp-setsockopt + }; + + function "sp_version", + output-argument: 1, + output-argument: 2, + output-argument: 3; + + function "sp_send", + map-argument: { 2 => <C-buffer-offset> }; + + function "sp_recv", + map-argument: { 2 => <C-buffer-offset> }; + +end interface; + +// Function for adding the base address of the repeated slots of a <buffer> +// to an offset and returning the result as a <machine-word>. This is +// necessary for passing <buffer> contents across the FFI. + +define function buffer-offset + (the-buffer :: <buffer>, data-offset :: <integer>) + => (result-offset :: <machine-word>) + u%+(data-offset, + primitive-wrap-machine-word + (primitive-repeated-slot-as-raw + (the-buffer, primitive-repeated-slot-offset(the-buffer)))) +end function; + +define inline function sp-send (socket :: <integer>, data :: <buffer>, flags :: <integer>) => (res :: <integer>) + %sp-send(socket, buffer-offset(data, 0), data.size, flags) +end; + +define inline function sp-recv (socket :: <integer>, data :: <buffer>, flags :: <integer>) => (res :: <integer>) + %sp-recv(socket, buffer-offset(data, 0), data.size, flags); +end; + +define inline method sp-setsockopt (socket :: <integer>, level :: <integer>, option :: <integer>, value :: <integer>) + with-stack-structure (int :: <C-int*>) + pointer-value(int) := value; + let setsockopt-result = + %sp-setsockopt(socket, level, option, int, size-of(<C-int*>)); + if (setsockopt-result < 0) + // Check error! + end; + setsockopt-result + end; +end; + +define inline method sp-setsockopt (socket :: <integer>, level :: <integer>, option :: <integer>, data :: <byte-string>) + let setsockopt-result = + %sp-setsockopt(socket, level, option, as(<c-string>, data), data.size); + if (setsockopt-result < 0) + // Check error! + end; + setsockopt-result +end; diff --git a/tests/examplefiles/phpMyAdmin.spec b/tests/examplefiles/phpMyAdmin.spec new file mode 100644 index 00000000..120fbc92 --- /dev/null +++ b/tests/examplefiles/phpMyAdmin.spec @@ -0,0 +1,163 @@ +%define _myadminpath /var/www/myadmin +%define pkgrelease rc1 +%define microrelease 1 + +Name: phpMyAdmin +Version: 3.1.1 +Release: %{pkgrelease}.%{microrelease} +License: GPL +Group: Applications/Databases/Interfaces +Source0: http://prdownloads.sourceforge.net/phpmyadmin/%{name}-%{version}-%{pkgrelease}.tar.bz2 +Source1: phpMyAdmin-http.conf +URL: http://sourceforge.net/projects/phpmyadmin/ +Requires: mysql +Requires: php-mysql +Buildarch: noarch +#BuildRoot: %{_tmppath}/%{name}-root + +Summary: phpMyAdmin - web-based MySQL administration + +%description +phpMyAdmin can manage a whole MySQL-server (needs a super-user) but +also a single database. To accomplish the latter you'll need a +properly set up MySQL-user which can read/write only the desired +database. It's up to you to look up the appropiate part in the MySQL +manual. Currently phpMyAdmin can: + - create and drop databases + - create, copy, drop and alter tables + - delete, edit and add fields + - execute any SQL-statement, even batch-queries + - manage keys on fields + - load text files into tables + - create (*) and read dumps of tables + - export (*) and import data to CSV values + - administer multiple servers and single databases + - check referencial integrity + - create complex queries automatically connecting required tables + - create PDF graphics of your database layout + - communicate in more than 38 different languages + + +%prep +%setup -q -n %{name}-%{version}-%{pkgrelease} + + +%build + + +%install +[ "${RPM_BUILD_ROOT}" != "/" ] && [ -d "${RPM_BUILD_ROOT}" ] && \ + rm -rf "${RPM_BUILD_ROOT}" + +# Create directories. + +install -d "${RPM_BUILD_ROOT}%{_myadminpath}"/{css,js,lang,libraries,themes} +install -d "${RPM_BUILD_ROOT}%{_myadminpath}"/libraries/{auth,dbg,dbi,engines} +install -d "${RPM_BUILD_ROOT}%{_myadminpath}"/libraries/{export,tcpdf,import} +install -d "${RPM_BUILD_ROOT}%{_myadminpath}"/libraries/transformations +install -d "${RPM_BUILD_ROOT}%{_myadminpath}"/libraries/tcpdf/font +install -d "${RPM_BUILD_ROOT}%{_myadminpath}"/themes/{darkblue_orange,original} +install -d "${RPM_BUILD_ROOT}%{_myadminpath}"/themes/darkblue_orange/{css,img} +install -d "${RPM_BUILD_ROOT}%{_myadminpath}"/themes/original/{css,img} + +# Install files. + +install libraries/config.default.php \ + "${RPM_BUILD_ROOT}%{_myadminpath}"/config.inc.php +install *.{php,ico} "${RPM_BUILD_ROOT}%{_myadminpath}"/ +install ChangeLog LICENSE README "${RPM_BUILD_ROOT}%{_myadminpath}"/ +install Documentation.html docs.css "${RPM_BUILD_ROOT}%{_myadminpath}"/ +install css/* "${RPM_BUILD_ROOT}%{_myadminpath}/css"/ +install js/* "${RPM_BUILD_ROOT}%{_myadminpath}/js"/ +install lang/*.php "${RPM_BUILD_ROOT}%{_myadminpath}/lang"/ +install libraries/*.php "${RPM_BUILD_ROOT}%{_myadminpath}/libraries"/ +install libraries/auth/*.php "${RPM_BUILD_ROOT}%{_myadminpath}/libraries/auth"/ +install libraries/dbg/*.php "${RPM_BUILD_ROOT}%{_myadminpath}/libraries/dbg"/ +install libraries/dbi/*.php "${RPM_BUILD_ROOT}%{_myadminpath}/libraries/dbi"/ +install libraries/engines/*.php \ + "${RPM_BUILD_ROOT}%{_myadminpath}/libraries/engines"/ +install libraries/export/*.php \ + "${RPM_BUILD_ROOT}%{_myadminpath}/libraries/export"/ +install libraries/tcpdf/*.php "${RPM_BUILD_ROOT}%{_myadminpath}/libraries/tcpdf"/ +install libraries/tcpdf/font/*.{php,z} \ + "${RPM_BUILD_ROOT}%{_myadminpath}/libraries/tcpdf/font"/ +install libraries/import/*.php \ + "${RPM_BUILD_ROOT}%{_myadminpath}/libraries/import"/ +install libraries/transformations/*.php \ + "${RPM_BUILD_ROOT}%{_myadminpath}/libraries/transformations"/ +install themes/darkblue_orange/*.{php,png} \ + "${RPM_BUILD_ROOT}%{_myadminpath}/themes/darkblue_orange"/ +install themes/darkblue_orange/css/*.php \ + "${RPM_BUILD_ROOT}%{_myadminpath}/themes/darkblue_orange/css"/ +install themes/darkblue_orange/img/*.{png,ico} \ + "${RPM_BUILD_ROOT}%{_myadminpath}/themes/darkblue_orange/img"/ +install themes/original/*.{php,png} \ + "${RPM_BUILD_ROOT}%{_myadminpath}/themes/original"/ +install themes/original/css/*.php \ + "${RPM_BUILD_ROOT}%{_myadminpath}/themes/original/css"/ +install themes/original/img/*.{png,ico} \ + "${RPM_BUILD_ROOT}%{_myadminpath}/themes/original/img"/ + +# Create documentation directories. + +DOCROOT="${RPM_BUILD_ROOT}%{_docdir}/%{name}-%{version}" +install -d "${DOCROOT}" +install -d "${DOCROOT}"/{lang,scripts,transformations} + +# Install documentation files. + +install RELEASE-DATE-* "${DOCROOT}"/ +install CREDITS ChangeLog INSTALL LICENSE "${DOCROOT}"/ +install README TODO "${DOCROOT}"/ +install Documentation.* docs.css "${DOCROOT}"/ +install translators.html "${DOCROOT}"/ +install lang/*.sh "${DOCROOT}"/lang/ +install scripts/* "${DOCROOT}"/scripts/ +install libraries/tcpdf/README "${DOCROOT}"/README.tcpdf +install libraries/import/README "${DOCROOT}"/README.import +install libraries/transformations/README "${DOCROOT}"/transformations/ +install libraries/transformations/TEMPLATE* "${DOCROOT}"/transformations/ +install libraries/transformations/*.sh "${DOCROOT}"/transformations/ + +# Install configuration file for Apache. + +install -d "${RPM_BUILD_ROOT}%{_sysconfdir}/httpd/conf.d" +install "%{SOURCE1}" \ + "${RPM_BUILD_ROOT}%{_sysconfdir}/httpd/conf.d/phpMyAdmin.conf" + +# Generate non-configuration file list. + +(cd "${RPM_BUILD_ROOT}"; ls -d ."%{_myadminpath}"/*) | + sed -e '/\/config\.inc\.php$/d' -e 's/^.//' > files.list + + + +%clean +[ "${RPM_BUILD_ROOT}" != "/" ] && [ -d "${RPM_BUILD_ROOT}" ] && \ + rm -rf "${RPM_BUILD_ROOT}" + + +%files -f files.list +%defattr(644, root, root, 755) +%doc %{_docdir}/%{name}-%{version} +%dir %{_myadminpath} +%attr(640,root,apache) %config(noreplace) %verify(not size mtime md5) %{_myadminpath}/config.inc.php +%config(noreplace) %verify(not size mtime md5) %{_sysconfdir}/httpd/conf.d/* + + +%changelog +* Thu Feb 23 2006 Patrick Monnerat <pm@datasphere.ch> +- Version 2.8.0-rc1.1. + +* Thu Dec 22 2005 Patrick Monnerat <patrick.monnerat@econophone.ch> +- Path "nullpw" to allow trying connection with null password after failure. +- Version 2.7.0-pl1.1. + +* Mon Aug 22 2005 Patrick Monnerat <patrick.monnerat@econophone.ch> +- Version 2.6.3-pl1. + +* Wed Jul 21 2004 Patrick Monnerat <patrick.monnerat@econophone.ch> +- Version 2.5.7-pl1. + +* Fri Nov 22 2002 Patrick Monnerat <patrick.monnerat@econophone.ch> +- Version 2.3.0-rc1. diff --git a/tests/examplefiles/robotframework.txt b/tests/examplefiles/robotframework.txt new file mode 100644 index 00000000..63ba63e6 --- /dev/null +++ b/tests/examplefiles/robotframework.txt @@ -0,0 +1,39 @@ +*** Settings *** +Documentation Simple example demonstrating syntax highlighting. +Library ExampleLibrary +Test Setup Keyword argument argument with ${VARIABLE} + +*** Variables *** +${VARIABLE} Variable value +@{LIST} List variable here + +*** Test Cases *** +Keyword-driven example + Initialize System + Do Something + Result Should Be 42 + [Teardown] Cleanup System + +Data-driven example + [Template] Keyword + argument1 argument2 + argument ${VARIABLE} + @{LIST} + +Gherkin + Given system is initialized + When something is done + Then result should be "42" + +| Pipes | +| | [Documentation] | Also pipe separated format is supported. | +| | Log | As this example demonstrates. | + +*** Keywords *** +Result Should Be + [Arguments] ${expected} + ${actual} = Get Value + Should be Equal ${actual} ${expected} + +Then result should be "${expected}" + Result Should Be ${expected} diff --git a/tests/examplefiles/rust_example.rs b/tests/examplefiles/rust_example.rs index af791fbc..1c0a70c3 100644 --- a/tests/examplefiles/rust_example.rs +++ b/tests/examplefiles/rust_example.rs @@ -1,743 +1,233 @@ - -#[doc = "json serialization"]; - -import result::{result, ok, err}; -import io; -import io::{reader_util, writer_util}; -import map; -import map::hashmap; - -export json; -export error; -export to_writer; -export to_str; -export from_reader; -export from_str; -export eq; - -export num; -export string; -export boolean; -export list; -export dict; -export null; - -#[doc = "Represents a json value"] -enum json { - num(float), - string(str), - boolean(bool), - list([json]), - dict(map::hashmap<str,json>), - null, +// Copyright 2012 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// based on: +// http://shootout.alioth.debian.org/u32/benchmark.php?test=nbody&lang=java + +extern mod std; + +use core::os; + +// Using sqrt from the standard library is way slower than using libc +// directly even though std just calls libc, I guess it must be +// because the the indirection through another dynamic linker +// stub. Kind of shocking. Might be able to make it faster still with +// an llvm intrinsic. +#[nolink] +extern mod libc { + #[legacy_exports]; + fn sqrt(n: float) -> float; } -type error = { - line: uint, - col: uint, - msg: str, -}; - -#[doc = "Serializes a json value into a io::writer"] -fn to_writer(wr: io::writer, j: json) { - alt j { - num(n) { wr.write_str(float::to_str(n, 6u)); } - string(s) { - wr.write_char('"'); - let mut escaped = ""; - str::chars_iter(s) { |c| - alt c { - '"' { escaped += "\\\""; } - '\\' { escaped += "\\\\"; } - '\x08' { escaped += "\\b"; } - '\x0c' { escaped += "\\f"; } - '\n' { escaped += "\\n"; } - '\r' { escaped += "\\r"; } - '\t' { escaped += "\\t"; } - _ { escaped += str::from_char(c); } - } - }; - wr.write_str(escaped); - wr.write_char('"'); - } - boolean(b) { - wr.write_str(if b { "true" } else { "false" }); - } - list(v) { - wr.write_char('['); - let mut first = true; - vec::iter(v) { |item| - if !first { - wr.write_str(", "); - } - first = false; - to_writer(wr, item); - }; - wr.write_char(']'); - } - dict(d) { - if d.size() == 0u { - wr.write_str("{}"); - ret; - } - - wr.write_str("{ "); - let mut first = true; - d.items { |key, value| - if !first { - wr.write_str(", "); - } - first = false; - to_writer(wr, string(key)); - wr.write_str(": "); - to_writer(wr, value); - }; - wr.write_str(" }"); - } - null { - wr.write_str("null"); - } - } +fn main() { + let args = os::args(); + let args = if os::getenv(~"RUST_BENCH").is_some() { + ~[~"", ~"4000000"] + } else if args.len() <= 1u { + ~[~"", ~"100000"] + } else { + args + }; + let n = int::from_str(args[1]).get(); + let mut bodies: ~[Body::props] = NBodySystem::make(); + io::println(fmt!("%f", NBodySystem::energy(bodies))); + let mut i = 0; + while i < n { + NBodySystem::advance(bodies, 0.01); + i += 1; + } + io::println(fmt!("%f", NBodySystem::energy(bodies))); } -#[doc = "Serializes a json value into a string"] -fn to_str(j: json) -> str { - io::with_str_writer { |wr| to_writer(wr, j) } -} +mod NBodySystem { + use Body; -type parser = { - rdr: io::reader, - mut ch: char, - mut line: uint, - mut col: uint, -}; + pub fn make() -> ~[Body::props] { + let mut bodies: ~[Body::props] = + ~[Body::sun(), + Body::jupiter(), + Body::saturn(), + Body::uranus(), + Body::neptune()]; -impl parser for parser { - fn eof() -> bool { self.ch == -1 as char } + let mut px = 0.0; + let mut py = 0.0; + let mut pz = 0.0; - fn bump() { - self.ch = self.rdr.read_char(); + let mut i = 0; + while i < 5 { + px += bodies[i].vx * bodies[i].mass; + py += bodies[i].vy * bodies[i].mass; + pz += bodies[i].vz * bodies[i].mass; - if self.ch == '\n' { - self.line += 1u; - self.col = 1u; - } else { - self.col += 1u; + i += 1; } - } - fn next_char() -> char { - self.bump(); - self.ch - } + // side-effecting + Body::offset_momentum(&mut bodies[0], px, py, pz); - fn error<T>(msg: str) -> result<T, error> { - err({ line: self.line, col: self.col, msg: msg }) + return bodies; } - fn parse() -> result<json, error> { - alt self.parse_value() { - ok(value) { - // Skip trailing whitespaces. - self.parse_whitespace(); - // Make sure there is no trailing characters. - if self.eof() { - ok(value) - } else { - self.error("trailing characters") + pub fn advance(bodies: &mut [Body::props], dt: float) { + let mut i = 0; + while i < 5 { + let mut j = i + 1; + while j < 5 { + advance_one(&mut bodies[i], + &mut bodies[j], dt); + j += 1; } - } - e { e } - } - } - fn parse_value() -> result<json, error> { - self.parse_whitespace(); - - if self.eof() { ret self.error("EOF while parsing value"); } - - alt self.ch { - 'n' { self.parse_ident("ull", null) } - 't' { self.parse_ident("rue", boolean(true)) } - 'f' { self.parse_ident("alse", boolean(false)) } - '0' to '9' | '-' { self.parse_number() } - '"' { - alt self.parse_str() { - ok(s) { ok(string(s)) } - err(e) { err(e) } - } - } - '[' { self.parse_list() } - '{' { self.parse_object() } - _ { self.error("invalid syntax") } + i += 1; } - } - fn parse_whitespace() { - while char::is_whitespace(self.ch) { self.bump(); } - } - - fn parse_ident(ident: str, value: json) -> result<json, error> { - if str::all(ident, { |c| c == self.next_char() }) { - self.bump(); - ok(value) - } else { - self.error("invalid syntax") + i = 0; + while i < 5 { + move_(&mut bodies[i], dt); + i += 1; } } - fn parse_number() -> result<json, error> { - let mut neg = 1f; + pub fn advance_one(bi: &mut Body::props, + bj: &mut Body::props, + dt: float) unsafe { + let dx = bi.x - bj.x; + let dy = bi.y - bj.y; + let dz = bi.z - bj.z; - if self.ch == '-' { - self.bump(); - neg = -1f; - } + let dSquared = dx * dx + dy * dy + dz * dz; - let mut res = alt self.parse_integer() { - ok(res) { res } - err(e) { ret err(e); } - }; + let distance = ::libc::sqrt(dSquared); + let mag = dt / (dSquared * distance); - if self.ch == '.' { - alt self.parse_decimal(res) { - ok(r) { res = r; } - err(e) { ret err(e); } - } - } - - if self.ch == 'e' || self.ch == 'E' { - alt self.parse_exponent(res) { - ok(r) { res = r; } - err(e) { ret err(e); } - } - } + bi.vx -= dx * bj.mass * mag; + bi.vy -= dy * bj.mass * mag; + bi.vz -= dz * bj.mass * mag; - ok(num(neg * res)) + bj.vx += dx * bi.mass * mag; + bj.vy += dy * bi.mass * mag; + bj.vz += dz * bi.mass * mag; } - fn parse_integer() -> result<float, error> { - let mut res = 0f; - - alt self.ch { - '0' { - self.bump(); - - // There can be only one leading '0'. - alt self.ch { - '0' to '9' { ret self.error("invalid number"); } - _ {} - } - } - '1' to '9' { - while !self.eof() { - alt self.ch { - '0' to '9' { - res *= 10f; - res += ((self.ch as int) - ('0' as int)) as float; - - self.bump(); - } - _ { break; } - } - } - } - _ { ret self.error("invalid number"); } - } - - ok(res) + pub fn move_(b: &mut Body::props, dt: float) { + b.x += dt * b.vx; + b.y += dt * b.vy; + b.z += dt * b.vz; } - fn parse_decimal(res: float) -> result<float, error> { - self.bump(); + pub fn energy(bodies: &[Body::props]) -> float unsafe { + let mut dx; + let mut dy; + let mut dz; + let mut distance; + let mut e = 0.0; - // Make sure a digit follows the decimal place. - alt self.ch { - '0' to '9' {} - _ { ret self.error("invalid number"); } - } - - let mut res = res; - let mut dec = 1f; - while !self.eof() { - alt self.ch { - '0' to '9' { - dec /= 10f; - res += (((self.ch as int) - ('0' as int)) as float) * dec; - - self.bump(); - } - _ { break; } - } - } - - ok(res) - } - - fn parse_exponent(res: float) -> result<float, error> { - self.bump(); + let mut i = 0; + while i < 5 { + e += + 0.5 * bodies[i].mass * + (bodies[i].vx * bodies[i].vx + bodies[i].vy * bodies[i].vy + + bodies[i].vz * bodies[i].vz); - let mut res = res; - let mut exp = 0u; - let mut neg_exp = false; + let mut j = i + 1; + while j < 5 { + dx = bodies[i].x - bodies[j].x; + dy = bodies[i].y - bodies[j].y; + dz = bodies[i].z - bodies[j].z; - alt self.ch { - '+' { self.bump(); } - '-' { self.bump(); neg_exp = true; } - _ {} - } - - // Make sure a digit follows the exponent place. - alt self.ch { - '0' to '9' {} - _ { ret self.error("invalid number"); } - } - - while !self.eof() { - alt self.ch { - '0' to '9' { - exp *= 10u; - exp += (self.ch as uint) - ('0' as uint); + distance = ::libc::sqrt(dx * dx + dy * dy + dz * dz); + e -= bodies[i].mass * bodies[j].mass / distance; - self.bump(); - } - _ { break; } + j += 1; } - } - let exp = float::pow_with_uint(10u, exp); - if neg_exp { - res /= exp; - } else { - res *= exp; + i += 1; } + return e; - ok(res) - } - - fn parse_str() -> result<str, error> { - let mut escape = false; - let mut res = ""; - - while !self.eof() { - self.bump(); - - if (escape) { - alt self.ch { - '"' { str::push_char(res, '"'); } - '\\' { str::push_char(res, '\\'); } - '/' { str::push_char(res, '/'); } - 'b' { str::push_char(res, '\x08'); } - 'f' { str::push_char(res, '\x0c'); } - 'n' { str::push_char(res, '\n'); } - 'r' { str::push_char(res, '\r'); } - 't' { str::push_char(res, '\t'); } - 'u' { - // Parse \u1234. - let mut i = 0u; - let mut n = 0u; - while i < 4u { - alt self.next_char() { - '0' to '9' { - n = n * 10u + - (self.ch as uint) - ('0' as uint); - } - _ { ret self.error("invalid \\u escape"); } - } - i += 1u; - } - - // Error out if we didn't parse 4 digits. - if i != 4u { - ret self.error("invalid \\u escape"); - } - - str::push_char(res, n as char); - } - _ { ret self.error("invalid escape"); } - } - escape = false; - } else if self.ch == '\\' { - escape = true; - } else { - if self.ch == '"' { - self.bump(); - ret ok(res); - } - str::push_char(res, self.ch); - } - } - - self.error("EOF while parsing string") - } - - fn parse_list() -> result<json, error> { - self.bump(); - self.parse_whitespace(); - - let mut values = []; - - if self.ch == ']' { - self.bump(); - ret ok(list(values)); - } - - loop { - alt self.parse_value() { - ok(v) { vec::push(values, v); } - e { ret e; } - } - - self.parse_whitespace(); - if self.eof() { - ret self.error("EOF while parsing list"); - } - - alt self.ch { - ',' { self.bump(); } - ']' { self.bump(); ret ok(list(values)); } - _ { ret self.error("expecting ',' or ']'"); } - } - }; - } - - fn parse_object() -> result<json, error> { - self.bump(); - self.parse_whitespace(); - - let values = map::str_hash(); - - if self.ch == '}' { - self.bump(); - ret ok(dict(values)); - } - - while !self.eof() { - self.parse_whitespace(); - - if self.ch != '"' { - ret self.error("key must be a string"); - } - - let key = alt self.parse_str() { - ok(key) { key } - err(e) { ret err(e); } - }; - - self.parse_whitespace(); - - if self.ch != ':' { - if self.eof() { break; } - ret self.error("expecting ':'"); - } - self.bump(); - - alt self.parse_value() { - ok(value) { values.insert(key, value); } - e { ret e; } - } - self.parse_whitespace(); - - alt self.ch { - ',' { self.bump(); } - '}' { self.bump(); ret ok(dict(values)); } - _ { - if self.eof() { break; } - ret self.error("expecting ',' or '}'"); - } - } - } - - ret self.error("EOF while parsing object"); - } -} - -#[doc = "Deserializes a json value from an io::reader"] -fn from_reader(rdr: io::reader) -> result<json, error> { - let parser = { - rdr: rdr, - mut ch: rdr.read_char(), - mut line: 1u, - mut col: 1u, - }; - - parser.parse() -} - -#[doc = "Deserializes a json value from a string"] -fn from_str(s: str) -> result<json, error> { - io::with_str_reader(s, from_reader) -} - -#[doc = "Test if two json values are equal"] -fn eq(value0: json, value1: json) -> bool { - alt (value0, value1) { - (num(f0), num(f1)) { f0 == f1 } - (string(s0), string(s1)) { s0 == s1 } - (boolean(b0), boolean(b1)) { b0 == b1 } - (list(l0), list(l1)) { vec::all2(l0, l1, eq) } - (dict(d0), dict(d1)) { - if d0.size() == d1.size() { - let mut equal = true; - d0.items { |k, v0| - alt d1.find(k) { - some(v1) { - if !eq(v0, v1) { equal = false; } } - none { equal = false; } - } - }; - equal - } else { - false - } - } - (null, null) { true } - _ { false } } } -#[cfg(test)] -mod tests { - fn mk_dict(items: [(str, json)]) -> json { - let d = map::str_hash(); - - vec::iter(items) { |item| - let (key, value) = item; - d.insert(key, value); - }; - - dict(d) - } - - #[test] - fn test_write_null() { - assert to_str(null) == "null"; - } - - #[test] - fn test_write_num() { - assert to_str(num(3f)) == "3"; - assert to_str(num(3.1f)) == "3.1"; - assert to_str(num(-1.5f)) == "-1.5"; - assert to_str(num(0.5f)) == "0.5"; - } - - #[test] - fn test_write_str() { - assert to_str(string("")) == "\"\""; - assert to_str(string("foo")) == "\"foo\""; - } - - #[test] - fn test_write_bool() { - assert to_str(boolean(true)) == "true"; - assert to_str(boolean(false)) == "false"; - } - - #[test] - fn test_write_list() { - assert to_str(list([])) == "[]"; - assert to_str(list([boolean(true)])) == "[true]"; - assert to_str(list([ - boolean(false), - null, - list([string("foo\nbar"), num(3.5f)]) - ])) == "[false, null, [\"foo\\nbar\", 3.5]]"; - } - - #[test] - fn test_write_dict() { - assert to_str(mk_dict([])) == "{}"; - assert to_str(mk_dict([("a", boolean(true))])) == "{ \"a\": true }"; - assert to_str(mk_dict([ - ("a", boolean(true)), - ("b", list([ - mk_dict([("c", string("\x0c\r"))]), - mk_dict([("d", string(""))]) - ])) - ])) == - "{ " + - "\"a\": true, " + - "\"b\": [" + - "{ \"c\": \"\\f\\r\" }, " + - "{ \"d\": \"\" }" + - "]" + - " }"; +mod Body { + use Body; + + pub const PI: float = 3.141592653589793; + pub const SOLAR_MASS: float = 39.478417604357432; + // was 4 * PI * PI originally + pub const DAYS_PER_YEAR: float = 365.24; + + pub type props = + {mut x: float, + mut y: float, + mut z: float, + mut vx: float, + mut vy: float, + mut vz: float, + mass: float}; + + pub fn jupiter() -> Body::props { + return {mut x: 4.84143144246472090e+00, + mut y: -1.16032004402742839e+00, + mut z: -1.03622044471123109e-01, + mut vx: 1.66007664274403694e-03 * DAYS_PER_YEAR, + mut vy: 7.69901118419740425e-03 * DAYS_PER_YEAR, + mut vz: -6.90460016972063023e-05 * DAYS_PER_YEAR, + mass: 9.54791938424326609e-04 * SOLAR_MASS}; + } + + pub fn saturn() -> Body::props { + return {mut x: 8.34336671824457987e+00, + mut y: 4.12479856412430479e+00, + mut z: -4.03523417114321381e-01, + mut vx: -2.76742510726862411e-03 * DAYS_PER_YEAR, + mut vy: 4.99852801234917238e-03 * DAYS_PER_YEAR, + mut vz: 2.30417297573763929e-05 * DAYS_PER_YEAR, + mass: 2.85885980666130812e-04 * SOLAR_MASS}; + } + + pub fn uranus() -> Body::props { + return {mut x: 1.28943695621391310e+01, + mut y: -1.51111514016986312e+01, + mut z: -2.23307578892655734e-01, + mut vx: 2.96460137564761618e-03 * DAYS_PER_YEAR, + mut vy: 2.37847173959480950e-03 * DAYS_PER_YEAR, + mut vz: -2.96589568540237556e-05 * DAYS_PER_YEAR, + mass: 4.36624404335156298e-05 * SOLAR_MASS}; + } + + pub fn neptune() -> Body::props { + return {mut x: 1.53796971148509165e+01, + mut y: -2.59193146099879641e+01, + mut z: 1.79258772950371181e-01, + mut vx: 2.68067772490389322e-03 * DAYS_PER_YEAR, + mut vy: 1.62824170038242295e-03 * DAYS_PER_YEAR, + mut vz: -9.51592254519715870e-05 * DAYS_PER_YEAR, + mass: 5.15138902046611451e-05 * SOLAR_MASS}; + } + + pub fn sun() -> Body::props { + return {mut x: 0.0, + mut y: 0.0, + mut z: 0.0, + mut vx: 0.0, + mut vy: 0.0, + mut vz: 0.0, + mass: SOLAR_MASS}; + } + + pub fn offset_momentum(props: &mut Body::props, + px: float, py: float, pz: float) { + props.vx = -px / SOLAR_MASS; + props.vy = -py / SOLAR_MASS; + props.vz = -pz / SOLAR_MASS; } - #[test] - fn test_trailing_characters() { - assert from_str("nulla") == - err({line: 1u, col: 5u, msg: "trailing characters"}); - assert from_str("truea") == - err({line: 1u, col: 5u, msg: "trailing characters"}); - assert from_str("falsea") == - err({line: 1u, col: 6u, msg: "trailing characters"}); - assert from_str("1a") == - err({line: 1u, col: 2u, msg: "trailing characters"}); - assert from_str("[]a") == - err({line: 1u, col: 3u, msg: "trailing characters"}); - assert from_str("{}a") == - err({line: 1u, col: 3u, msg: "trailing characters"}); - } - - #[test] - fn test_read_identifiers() { - assert from_str("n") == - err({line: 1u, col: 2u, msg: "invalid syntax"}); - assert from_str("nul") == - err({line: 1u, col: 4u, msg: "invalid syntax"}); - - assert from_str("t") == - err({line: 1u, col: 2u, msg: "invalid syntax"}); - assert from_str("truz") == - err({line: 1u, col: 4u, msg: "invalid syntax"}); - - assert from_str("f") == - err({line: 1u, col: 2u, msg: "invalid syntax"}); - assert from_str("faz") == - err({line: 1u, col: 3u, msg: "invalid syntax"}); - - assert from_str("null") == ok(null); - assert from_str("true") == ok(boolean(true)); - assert from_str("false") == ok(boolean(false)); - assert from_str(" null ") == ok(null); - assert from_str(" true ") == ok(boolean(true)); - assert from_str(" false ") == ok(boolean(false)); - } - - #[test] - fn test_read_num() { - assert from_str("+") == - err({line: 1u, col: 1u, msg: "invalid syntax"}); - assert from_str(".") == - err({line: 1u, col: 1u, msg: "invalid syntax"}); - - assert from_str("-") == - err({line: 1u, col: 2u, msg: "invalid number"}); - assert from_str("00") == - err({line: 1u, col: 2u, msg: "invalid number"}); - assert from_str("1.") == - err({line: 1u, col: 3u, msg: "invalid number"}); - assert from_str("1e") == - err({line: 1u, col: 3u, msg: "invalid number"}); - assert from_str("1e+") == - err({line: 1u, col: 4u, msg: "invalid number"}); - - assert from_str("3") == ok(num(3f)); - assert from_str("3.1") == ok(num(3.1f)); - assert from_str("-1.2") == ok(num(-1.2f)); - assert from_str("0.4") == ok(num(0.4f)); - assert from_str("0.4e5") == ok(num(0.4e5f)); - assert from_str("0.4e+15") == ok(num(0.4e15f)); - assert from_str("0.4e-01") == ok(num(0.4e-01f)); - assert from_str(" 3 ") == ok(num(3f)); - } - - #[test] - fn test_read_str() { - assert from_str("\"") == - err({line: 1u, col: 2u, msg: "EOF while parsing string"}); - assert from_str("\"lol") == - err({line: 1u, col: 5u, msg: "EOF while parsing string"}); - - assert from_str("\"\"") == ok(string("")); - assert from_str("\"foo\"") == ok(string("foo")); - assert from_str("\"\\\"\"") == ok(string("\"")); - assert from_str("\"\\b\"") == ok(string("\x08")); - assert from_str("\"\\n\"") == ok(string("\n")); - assert from_str("\"\\r\"") == ok(string("\r")); - assert from_str("\"\\t\"") == ok(string("\t")); - assert from_str(" \"foo\" ") == ok(string("foo")); - } - - #[test] - fn test_read_list() { - assert from_str("[") == - err({line: 1u, col: 2u, msg: "EOF while parsing value"}); - assert from_str("[1") == - err({line: 1u, col: 3u, msg: "EOF while parsing list"}); - assert from_str("[1,") == - err({line: 1u, col: 4u, msg: "EOF while parsing value"}); - assert from_str("[1,]") == - err({line: 1u, col: 4u, msg: "invalid syntax"}); - assert from_str("[6 7]") == - err({line: 1u, col: 4u, msg: "expecting ',' or ']'"}); - - assert from_str("[]") == ok(list([])); - assert from_str("[ ]") == ok(list([])); - assert from_str("[true]") == ok(list([boolean(true)])); - assert from_str("[ false ]") == ok(list([boolean(false)])); - assert from_str("[null]") == ok(list([null])); - assert from_str("[3, 1]") == ok(list([num(3f), num(1f)])); - assert from_str("\n[3, 2]\n") == ok(list([num(3f), num(2f)])); - assert from_str("[2, [4, 1]]") == - ok(list([num(2f), list([num(4f), num(1f)])])); - } - - #[test] - fn test_read_dict() { - assert from_str("{") == - err({line: 1u, col: 2u, msg: "EOF while parsing object"}); - assert from_str("{ ") == - err({line: 1u, col: 3u, msg: "EOF while parsing object"}); - assert from_str("{1") == - err({line: 1u, col: 2u, msg: "key must be a string"}); - assert from_str("{ \"a\"") == - err({line: 1u, col: 6u, msg: "EOF while parsing object"}); - assert from_str("{\"a\"") == - err({line: 1u, col: 5u, msg: "EOF while parsing object"}); - assert from_str("{\"a\" ") == - err({line: 1u, col: 6u, msg: "EOF while parsing object"}); - - assert from_str("{\"a\" 1") == - err({line: 1u, col: 6u, msg: "expecting ':'"}); - assert from_str("{\"a\":") == - err({line: 1u, col: 6u, msg: "EOF while parsing value"}); - assert from_str("{\"a\":1") == - err({line: 1u, col: 7u, msg: "EOF while parsing object"}); - assert from_str("{\"a\":1 1") == - err({line: 1u, col: 8u, msg: "expecting ',' or '}'"}); - assert from_str("{\"a\":1,") == - err({line: 1u, col: 8u, msg: "EOF while parsing object"}); - - assert eq(result::get(from_str("{}")), mk_dict([])); - assert eq(result::get(from_str("{\"a\": 3}")), - mk_dict([("a", num(3.0f))])); - - assert eq(result::get(from_str("{ \"a\": null, \"b\" : true }")), - mk_dict([("a", null), ("b", boolean(true))])); - assert eq(result::get(from_str("\n{ \"a\": null, \"b\" : true }\n")), - mk_dict([("a", null), ("b", boolean(true))])); - assert eq(result::get(from_str("{\"a\" : 1.0 ,\"b\": [ true ]}")), - mk_dict([ - ("a", num(1.0)), - ("b", list([boolean(true)])) - ])); - assert eq(result::get(from_str( - "{" + - "\"a\": 1.0, " + - "\"b\": [" + - "true," + - "\"foo\\nbar\", " + - "{ \"c\": {\"d\": null} } " + - "]" + - "}")), - mk_dict([ - ("a", num(1.0f)), - ("b", list([ - boolean(true), - string("foo\nbar"), - mk_dict([ - ("c", mk_dict([("d", null)])) - ]) - ])) - ])); - } - - #[test] - fn test_multiline_errors() { - assert from_str("{\n \"foo\":\n \"bar\"") == - err({line: 3u, col: 8u, msg: "EOF while parsing object"}); - } } diff --git a/tests/examplefiles/test2.pypylog b/tests/examplefiles/test2.pypylog new file mode 100644 index 00000000..543e21dd --- /dev/null +++ b/tests/examplefiles/test2.pypylog @@ -0,0 +1,120 @@ +[2f1dd6c3b8b7] {jit-log-opt-loop +# Loop 0 (<Function object at 0xb720e550> ds1dr4 dsdr3 ds1dr4) : loop with 115 ops +[p0, p1] ++33: label(p0, p1, descr=TargetToken(-1223434224)) +debug_merge_point(0, 0, '<Function object at 0xb710b120> ds1dr4 dsdr3 ds1dr4') ++33: guard_nonnull_class(p1, 138371488, descr=<Guard2>) [p1, p0] ++54: p3 = getfield_gc_pure(p1, descr=<FieldP pyhaskell.interpreter.haskell.Substitution.inst_rhs 8>) ++57: guard_value(p3, ConstPtr(ptr4), descr=<Guard3>) [p1, p0, p3] ++69: p5 = getfield_gc_pure(p1, descr=<FieldP pyhaskell.interpreter.haskell.Substitution.inst_subst 12>) ++72: p7 = getarrayitem_gc(p5, 0, descr=<ArrayP 4>) ++75: guard_class(p7, 138371552, descr=<Guard4>) [p0, p5, p7] ++88: p9 = getfield_gc(p7, descr=<FieldP pyhaskell.interpreter.haskell.Thunk.inst_application 8>) ++91: guard_nonnull_class(p9, 138373024, descr=<Guard5>) [p0, p5, p7, p9] ++109: p12 = getarrayitem_gc(p5, 1, descr=<ArrayP 4>) ++112: guard_class(p12, 138371552, descr=<Guard6>) [p0, p5, p12, p7] ++125: p14 = getfield_gc(p12, descr=<FieldP pyhaskell.interpreter.haskell.Thunk.inst_application 8>) ++128: guard_nonnull_class(p14, 138373024, descr=<Guard7>) [p0, p5, p12, p14, p7] +debug_merge_point(0, 0, 'None') +debug_merge_point(0, 0, 'None') ++146: p16 = getfield_gc_pure(p9, descr=<FieldP pyhaskell.interpreter.haskell.Application.inst_function 8>) ++149: guard_value(p16, ConstPtr(ptr17), descr=<Guard8>) [p16, p9, p0, p12, p7] ++161: p18 = getfield_gc_pure(p9, descr=<FieldP pyhaskell.interpreter.haskell.Application3.inst_arg0 12>) ++164: guard_class(p18, 138371648, descr=<Guard9>) [p18, p9, p0, p12, p7] ++177: p20 = getfield_gc_pure(p9, descr=<FieldP pyhaskell.interpreter.haskell.Application3.inst_arg1 16>) ++180: guard_class(p20, 138371648, descr=<Guard10>) [p20, p9, p18, p0, p12, p7] ++193: p22 = getfield_gc_pure(p9, descr=<FieldP pyhaskell.interpreter.haskell.Application3.inst_arg2 20>) ++196: guard_class(p22, 138371936, descr=<Guard11>) [p22, p9, p20, p18, p0, p12, p7] +debug_merge_point(0, 0, 'None') ++209: p24 = getfield_gc_pure(p22, descr=<FieldP pyhaskell.interpreter.haskell.Application.inst_function 8>) ++215: guard_value(p24, ConstPtr(ptr25), descr=<Guard12>) [p24, p22, p9, None, None, p0, p12, p7] ++227: p27 = getfield_gc_pure(p22, descr=<FieldP pyhaskell.interpreter.haskell.Application1.inst_arg0 12>) ++230: guard_class(p27, 138371648, descr=<Guard13>) [p22, p27, p9, None, None, p0, p12, p7] +debug_merge_point(0, 0, '_') +debug_merge_point(0, 0, 'None') ++243: p30 = getfield_gc(ConstPtr(ptr29), descr=<FieldP pyhaskell.interpreter.module.CoreMod.inst_qvars 24>) ++249: i34 = call(ConstClass(ll_dict_lookup_trampoline__v64___simple_call__function_ll), p30, ConstPtr(ptr32), 360200661, descr=<Calli 4 rri EF=4>) ++281: guard_no_exception(, descr=<Guard14>) [p27, p20, p18, i34, p30, None, None, None, p0, p12, p7] ++294: i36 = int_and(i34, -2147483648) ++302: i37 = int_is_true(i36) +guard_false(i37, descr=<Guard15>) [p27, p20, p18, i34, p30, None, None, None, p0, p12, p7] ++311: p38 = getfield_gc(p30, descr=<FieldP dicttable.entries 12>) ++314: p39 = getinteriorfield_gc(p38, i34, descr=<InteriorFieldDescr <FieldP dictentry.value 4>>) ++318: i40 = instance_ptr_eq(p18, p39) +guard_true(i40, descr=<Guard16>) [p27, p20, None, None, None, p0, p12, p7] +debug_merge_point(0, 0, 'None') ++327: i41 = getfield_gc_pure(p20, descr=<FieldS pyhaskell.interpreter.primtype.Int.inst_value 8>) ++330: i42 = getfield_gc_pure(p27, descr=<FieldS pyhaskell.interpreter.primtype.Int.inst_value 8>) ++333: i43 = int_sub(i41, i42) +debug_merge_point(0, 0, 'None') +debug_merge_point(0, 0, 'None') +debug_merge_point(0, 0, 'None') ++335: i45 = int_eq(0, i43) +guard_false(i45, descr=<Guard17>) [p0, i43, None, None, None, None, p12, p7] +p47 = new_with_vtable(138371648) ++393: setfield_gc(p47, i43, descr=<FieldS pyhaskell.interpreter.primtype.Int.inst_value 8>) +setfield_gc(p7, p47, descr=<FieldP pyhaskell.interpreter.haskell.Thunk.inst_application 8>) ++414: p48 = getfield_gc(p12, descr=<FieldP pyhaskell.interpreter.haskell.Thunk.inst_application 8>) ++420: guard_nonnull_class(p48, 138371648, descr=<Guard18>) [p0, p48, p12, p47, p7] +debug_merge_point(0, 0, '<PrimFunction object at 0x83f3f6c> 1 <Function object at 0xb710b3b0> 1 <Function object at 0xb710b3c0> <PrimFunction object at 0x83f3f3c> 1 dsdr3 <Function object at 0xb710b210> 1') +debug_merge_point(0, 0, 'None') +debug_merge_point(0, 0, '_') +debug_merge_point(0, 0, 'None') +debug_merge_point(0, 0, 'None') +debug_merge_point(0, 0, '<Function object at 0xb710b3d0> dsdr3 dsdr3') +debug_merge_point(0, 0, '<Function object at 0xb710b120> ds1dr4 dsdr3 ds1dr4') ++438: label(p0, p48, p30, p38, descr=TargetToken(-1223434176)) +debug_merge_point(0, 0, '<Function object at 0xb710b120> ds1dr4 dsdr3 ds1dr4') +debug_merge_point(0, 0, 'None') +debug_merge_point(0, 0, 'None') +debug_merge_point(0, 0, 'None') +debug_merge_point(0, 0, '_') +debug_merge_point(0, 0, 'None') ++438: i50 = call(ConstClass(ll_dict_lookup_trampoline__v64___simple_call__function_ll), p30, ConstPtr(ptr32), 360200661, descr=<Calli 4 rri EF=4>) ++464: guard_no_exception(, descr=<Guard19>) [p48, i50, p30, p0] ++477: i51 = int_and(i50, -2147483648) ++485: i52 = int_is_true(i51) +guard_false(i52, descr=<Guard20>) [p48, i50, p30, p0] ++494: p53 = getinteriorfield_gc(p38, i50, descr=<InteriorFieldDescr <FieldP dictentry.value 4>>) ++501: i55 = instance_ptr_eq(ConstPtr(ptr54), p53) +guard_true(i55, descr=<Guard21>) [p48, p0] +debug_merge_point(0, 0, 'None') ++513: i56 = getfield_gc_pure(p48, descr=<FieldS pyhaskell.interpreter.primtype.Int.inst_value 8>) ++516: i58 = int_sub(i56, 1) +debug_merge_point(0, 0, 'None') +debug_merge_point(0, 0, 'None') +debug_merge_point(0, 0, 'None') ++519: i59 = int_eq(0, i58) +guard_false(i59, descr=<Guard22>) [i58, p48, p0] +debug_merge_point(0, 0, '<PrimFunction object at 0x83f3f6c> 1 <Function object at 0xb710b3b0> 1 <Function object at 0xb710b3c0> <PrimFunction object at 0x83f3f3c> 1 dsdr3 <Function object at 0xb710b210> 1') +debug_merge_point(0, 0, 'None') +debug_merge_point(0, 0, '_') +debug_merge_point(0, 0, 'None') +debug_merge_point(0, 0, 'None') +debug_merge_point(0, 0, '<Function object at 0xb710b3d0> dsdr3 dsdr3') +debug_merge_point(0, 0, '<Function object at 0xb710b120> ds1dr4 dsdr3 ds1dr4') +p61 = new_with_vtable(138371700) +p63 = new_with_vtable(138373024) +p65 = new_with_vtable(138371936) ++606: setfield_gc(p63, ConstPtr(ptr66), descr=<FieldP pyhaskell.interpreter.haskell.Application.inst_function 8>) +p68 = new_with_vtable(138373024) ++632: setfield_gc(p65, ConstPtr(ptr69), descr=<FieldP pyhaskell.interpreter.haskell.Application.inst_function 8>) +p71 = new_with_vtable(138371936) ++658: setfield_gc(p68, ConstPtr(ptr17), descr=<FieldP pyhaskell.interpreter.haskell.Application.inst_function 8>) ++665: setfield_gc(p71, ConstPtr(ptr72), descr=<FieldP pyhaskell.interpreter.haskell.Application1.inst_arg0 12>) ++672: setfield_gc(p68, p71, descr=<FieldP pyhaskell.interpreter.haskell.Application3.inst_arg2 20>) ++675: setfield_gc(p68, p48, descr=<FieldP pyhaskell.interpreter.haskell.Application3.inst_arg1 16>) ++678: setfield_gc(p68, ConstPtr(ptr54), descr=<FieldP pyhaskell.interpreter.haskell.Application3.inst_arg0 12>) +p73 = new_with_vtable(138371648) ++701: setfield_gc(p61, p0, descr=<FieldP pyhaskell.interpreter.haskell.StackElement.inst_next 8>) ++716: setfield_gc(p61, 2, descr=<FieldS pyhaskell.interpreter.haskell.CopyStackElement.inst_index 16>) ++723: setfield_gc(p71, ConstPtr(ptr25), descr=<FieldP pyhaskell.interpreter.haskell.Application.inst_function 8>) ++730: setfield_gc(p65, p68, descr=<FieldP pyhaskell.interpreter.haskell.Application1.inst_arg0 12>) ++733: setfield_gc(p63, p65, descr=<FieldP pyhaskell.interpreter.haskell.Application3.inst_arg2 20>) ++736: setfield_gc(p63, ConstPtr(ptr75), descr=<FieldP pyhaskell.interpreter.haskell.Application3.inst_arg1 16>) ++743: setfield_gc(p63, ConstPtr(ptr54), descr=<FieldP pyhaskell.interpreter.haskell.Application3.inst_arg0 12>) ++750: setfield_gc(p61, p63, descr=<FieldP pyhaskell.interpreter.haskell.CopyStackElement.inst_application 12>) ++753: setfield_gc(p73, i58, descr=<FieldS pyhaskell.interpreter.primtype.Int.inst_value 8>) ++762: jump(p61, p73, p30, p38, descr=TargetToken(-1223434176)) ++775: --end of the loop-- +[2f1dd6da3b99] jit-log-opt-loop} diff --git a/tests/examplefiles/unix-io.lid b/tests/examplefiles/unix-io.lid new file mode 100644 index 00000000..617fcaa4 --- /dev/null +++ b/tests/examplefiles/unix-io.lid @@ -0,0 +1,37 @@ +Library: io +Synopsis: A portable IO library +Author: Gail Zacharias +Files: library + streams/defs + streams/stream + streams/sequence-stream + streams/native-buffer + streams/buffer + streams/typed-stream + streams/external-stream + streams/buffered-stream + streams/convenience + streams/wrapper-stream + streams/cleanup-streams + streams/native-speed + streams/async-writes + streams/file-stream + streams/multi-buffered-streams + pprint + print + print-double-integer-kludge + format + buffered-format + format-condition + unix-file-accessor + unix-standard-io + unix-interface + format-out +C-Source-Files: unix-portability.c +Major-Version: 2 +Minor-Version: 1 +Target-Type: dll +Copyright: Original Code is Copyright (c) 1995-2004 Functional Objects, Inc. + All rights reserved. +License: See License.txt in this distribution for details. +Warranty: Distributed WITHOUT WARRANTY OF ANY KIND diff --git a/tests/old_run.py b/tests/old_run.py index 7929d743..4f7cef16 100644 --- a/tests/old_run.py +++ b/tests/old_run.py @@ -8,7 +8,7 @@ python run.py [testfile ...] - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/run.py b/tests/run.py index ef92fe09..b7308a31 100644 --- a/tests/run.py +++ b/tests/run.py @@ -8,7 +8,7 @@ python run.py [testfile ...] - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index bc4eb771..f6643308 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -3,7 +3,7 @@ Pygments basic API tests ~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -93,7 +93,7 @@ def test_lexer_options(): 'PythonConsoleLexer', 'RConsoleLexer', 'RubyConsoleLexer', 'SqliteConsoleLexer', 'MatlabSessionLexer', 'ErlangShellLexer', 'BashSessionLexer', 'LiterateHaskellLexer', 'PostgresConsoleLexer', - 'ElixirConsoleLexer', 'JuliaConsoleLexer'): + 'ElixirConsoleLexer', 'JuliaConsoleLexer', 'RobotFrameworkLexer'): inst = cls(ensurenl=False) ensure(inst.get_tokens('a\nb'), 'a\nb') inst = cls(ensurenl=False, stripall=True) diff --git a/tests/test_clexer.py b/tests/test_clexer.py index 08fb42cf..8b37bf57 100644 --- a/tests/test_clexer.py +++ b/tests/test_clexer.py @@ -3,7 +3,7 @@ Basic CLexer Test ~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_cmdline.py b/tests/test_cmdline.py index 56036183..5ad815c0 100644 --- a/tests/test_cmdline.py +++ b/tests/test_cmdline.py @@ -3,7 +3,7 @@ Command line test ~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_examplefiles.py b/tests/test_examplefiles.py index 41acf4ef..d785cf3b 100644 --- a/tests/test_examplefiles.py +++ b/tests/test_examplefiles.py @@ -3,7 +3,7 @@ Pygments tests with example files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -58,6 +58,8 @@ def check_lexer(lx, absfn, outfn): text = text.strip(b('\n')) + b('\n') try: text = text.decode('utf-8') + if text.startswith(u'\ufeff'): + text = text[len(u'\ufeff'):] except UnicodeError: text = text.decode('latin1') ntext = [] diff --git a/tests/test_html_formatter.py b/tests/test_html_formatter.py index 284a6c75..96273638 100644 --- a/tests/test_html_formatter.py +++ b/tests/test_html_formatter.py @@ -3,7 +3,7 @@ Pygments HTML formatter tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_latex_formatter.py b/tests/test_latex_formatter.py index 8412ec41..06a74c3d 100644 --- a/tests/test_latex_formatter.py +++ b/tests/test_latex_formatter.py @@ -3,7 +3,7 @@ Pygments LaTeX formatter tests ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_perllexer.py b/tests/test_perllexer.py index b9c3cb74..315b20e3 100644 --- a/tests/test_perllexer.py +++ b/tests/test_perllexer.py @@ -3,7 +3,7 @@ Pygments regex lexer tests ~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_regexlexer.py b/tests/test_regexlexer.py index b1392d3a..28d9689b 100644 --- a/tests/test_regexlexer.py +++ b/tests/test_regexlexer.py @@ -3,7 +3,7 @@ Pygments regex lexer tests ~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_token.py b/tests/test_token.py index a9d1edeb..6a5b00b7 100644 --- a/tests/test_token.py +++ b/tests/test_token.py @@ -3,7 +3,7 @@ Test suite for the token module ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_using_api.py b/tests/test_using_api.py index 83d3f18e..bb89d1e2 100644 --- a/tests/test_using_api.py +++ b/tests/test_using_api.py @@ -3,7 +3,7 @@ Pygments tests for using() ~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ diff --git a/tests/test_util.py b/tests/test_util.py index be1662f9..dbbc66ce 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -3,7 +3,7 @@ Test suite for the util module ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ @@ -98,8 +98,10 @@ class UtilTest(unittest.TestCase): self.assertTrue(util.shebang_matches('#!C:\\Python2.4\\Python.exe', r'python(2\.\d)?')) - self.assertFalse(util.shebang_matches('#!/usr/bin/python-ruby', r'python(2\.\d)?')) - self.assertFalse(util.shebang_matches('#!/usr/bin/python/ruby', r'python(2\.\d)?')) + self.assertFalse(util.shebang_matches('#!/usr/bin/python-ruby', + r'python(2\.\d)?')) + self.assertFalse(util.shebang_matches('#!/usr/bin/python/ruby', + r'python(2\.\d)?')) self.assertFalse(util.shebang_matches('#!', r'python')) def test_doctype_matches(self): @@ -131,4 +133,3 @@ class UtilTest(unittest.TestCase): m = r.match(first_non_bmp * 2) self.assertTrue(m) self.assertEquals(m.end(), len(first_non_bmp) * 2) - |