diff options
Diffstat (limited to 'pygments/token.py')
| -rw-r--r-- | pygments/token.py | 213 |
1 files changed, 0 insertions, 213 deletions
diff --git a/pygments/token.py b/pygments/token.py deleted file mode 100644 index 5c30eb46..00000000 --- a/pygments/token.py +++ /dev/null @@ -1,213 +0,0 @@ -# -*- coding: utf-8 -*- -""" - pygments.token - ~~~~~~~~~~~~~~ - - Basic token types and the standard tokens. - - :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. - :license: BSD, see LICENSE for details. -""" - - -class _TokenType(tuple): - parent = None - - def split(self): - buf = [] - node = self - while node is not None: - buf.append(node) - node = node.parent - buf.reverse() - return buf - - def __init__(self, *args): - # no need to call super.__init__ - self.subtypes = set() - - def __contains__(self, val): - return self is val or ( - type(val) is self.__class__ and - val[:len(self)] == self - ) - - def __getattr__(self, val): - if not val or not val[0].isupper(): - return tuple.__getattribute__(self, val) - new = _TokenType(self + (val,)) - setattr(self, val, new) - self.subtypes.add(new) - new.parent = self - return new - - def __repr__(self): - return 'Token' + (self and '.' or '') + '.'.join(self) - - def __copy__(self): - # These instances are supposed to be singletons - return self - - def __deepcopy__(self, memo): - # These instances are supposed to be singletons - return self - - -Token = _TokenType() - -# Special token types -Text = Token.Text -Whitespace = Text.Whitespace -Escape = Token.Escape -Error = Token.Error -# Text that doesn't belong to this lexer (e.g. HTML in PHP) -Other = Token.Other - -# Common token types for source code -Keyword = Token.Keyword -Name = Token.Name -Literal = Token.Literal -String = Literal.String -Number = Literal.Number -Punctuation = Token.Punctuation -Operator = Token.Operator -Comment = Token.Comment - -# Generic types for non-source code -Generic = Token.Generic - -# String and some others are not direct children of Token. -# alias them: -Token.Token = Token -Token.String = String -Token.Number = Number - - -def is_token_subtype(ttype, other): - """ - Return True if ``ttype`` is a subtype of ``other``. - - exists for backwards compatibility. use ``ttype in other`` now. - """ - return ttype in other - - -def string_to_tokentype(s): - """ - Convert a string into a token type:: - - >>> string_to_token('String.Double') - Token.Literal.String.Double - >>> string_to_token('Token.Literal.Number') - Token.Literal.Number - >>> string_to_token('') - Token - - Tokens that are already tokens are returned unchanged: - - >>> string_to_token(String) - Token.Literal.String - """ - if isinstance(s, _TokenType): - return s - if not s: - return Token - node = Token - for item in s.split('.'): - node = getattr(node, item) - return node - - -# Map standard token types to short names, used in CSS class naming. -# If you add a new item, please be sure to run this file to perform -# a consistency check for duplicate values. -STANDARD_TYPES = { - Token: '', - - Text: '', - Whitespace: 'w', - Escape: 'esc', - Error: 'err', - Other: 'x', - - Keyword: 'k', - Keyword.Constant: 'kc', - Keyword.Declaration: 'kd', - Keyword.Namespace: 'kn', - Keyword.Pseudo: 'kp', - Keyword.Reserved: 'kr', - Keyword.Type: 'kt', - - Name: 'n', - Name.Attribute: 'na', - Name.Builtin: 'nb', - Name.Builtin.Pseudo: 'bp', - Name.Class: 'nc', - Name.Constant: 'no', - Name.Decorator: 'nd', - Name.Entity: 'ni', - Name.Exception: 'ne', - Name.Function: 'nf', - Name.Function.Magic: 'fm', - Name.Property: 'py', - Name.Label: 'nl', - Name.Namespace: 'nn', - Name.Other: 'nx', - Name.Tag: 'nt', - Name.Variable: 'nv', - Name.Variable.Class: 'vc', - Name.Variable.Global: 'vg', - Name.Variable.Instance: 'vi', - Name.Variable.Magic: 'vm', - - Literal: 'l', - Literal.Date: 'ld', - - String: 's', - String.Affix: 'sa', - String.Backtick: 'sb', - String.Char: 'sc', - String.Delimiter: 'dl', - String.Doc: 'sd', - String.Double: 's2', - String.Escape: 'se', - String.Heredoc: 'sh', - String.Interpol: 'si', - String.Other: 'sx', - String.Regex: 'sr', - String.Single: 's1', - String.Symbol: 'ss', - - Number: 'm', - Number.Bin: 'mb', - Number.Float: 'mf', - Number.Hex: 'mh', - Number.Integer: 'mi', - Number.Integer.Long: 'il', - Number.Oct: 'mo', - - Operator: 'o', - Operator.Word: 'ow', - - Punctuation: 'p', - - Comment: 'c', - Comment.Hashbang: 'ch', - Comment.Multiline: 'cm', - Comment.Preproc: 'cp', - Comment.PreprocFile: 'cpf', - Comment.Single: 'c1', - Comment.Special: 'cs', - - Generic: 'g', - Generic.Deleted: 'gd', - Generic.Emph: 'ge', - Generic.Error: 'gr', - Generic.Heading: 'gh', - Generic.Inserted: 'gi', - Generic.Output: 'go', - Generic.Prompt: 'gp', - Generic.Strong: 'gs', - Generic.Subheading: 'gu', - Generic.Traceback: 'gt', -} |