diff options
Diffstat (limited to 'pygments/lexers/robotframework.py')
| -rw-r--r-- | pygments/lexers/robotframework.py | 560 |
1 files changed, 0 insertions, 560 deletions
diff --git a/pygments/lexers/robotframework.py b/pygments/lexers/robotframework.py deleted file mode 100644 index 1288da8d..00000000 --- a/pygments/lexers/robotframework.py +++ /dev/null @@ -1,560 +0,0 @@ -# -*- coding: utf-8 -*- -""" - pygments.lexers.robotframework - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Lexer for Robot Framework. - - :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS. - :license: BSD, see LICENSE for details. -""" - -# Copyright 2012 Nokia Siemens Networks Oyj -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from pygments.lexer import Lexer -from pygments.token import Token -from pygments.util import text_type - -__all__ = ['RobotFrameworkLexer'] - - -HEADING = Token.Generic.Heading -SETTING = Token.Keyword.Namespace -IMPORT = Token.Name.Namespace -TC_KW_NAME = Token.Generic.Subheading -KEYWORD = Token.Name.Function -ARGUMENT = Token.String -VARIABLE = Token.Name.Variable -COMMENT = Token.Comment -SEPARATOR = Token.Punctuation -SYNTAX = Token.Punctuation -GHERKIN = Token.Generic.Emph -ERROR = Token.Error - - -def normalize(string, remove=''): - string = string.lower() - for char in remove + ' ': - if char in string: - string = string.replace(char, '') - return string - - -class RobotFrameworkLexer(Lexer): - """ - For `Robot Framework <http://robotframework.org>`_ test data. - - Supports both space and pipe separated plain text formats. - - .. versionadded:: 1.6 - """ - name = 'RobotFramework' - aliases = ['robotframework'] - filenames = ['*.txt', '*.robot'] - mimetypes = ['text/x-robotframework'] - - def __init__(self, **options): - options['tabsize'] = 2 - options['encoding'] = 'UTF-8' - Lexer.__init__(self, **options) - - def get_tokens_unprocessed(self, text): - row_tokenizer = RowTokenizer() - var_tokenizer = VariableTokenizer() - index = 0 - for row in text.splitlines(): - for value, token in row_tokenizer.tokenize(row): - for value, token in var_tokenizer.tokenize(value, token): - if value: - yield index, token, text_type(value) - index += len(value) - - -class VariableTokenizer(object): - - def tokenize(self, string, token): - var = VariableSplitter(string, identifiers='$@%&') - if var.start < 0 or token in (COMMENT, ERROR): - yield string, token - return - for value, token in self._tokenize(var, string, token): - if value: - yield value, token - - def _tokenize(self, var, string, orig_token): - before = string[:var.start] - yield before, orig_token - yield var.identifier + '{', SYNTAX - for value, token in self.tokenize(var.base, VARIABLE): - yield value, token - yield '}', SYNTAX - if var.index: - yield '[', SYNTAX - for value, token in self.tokenize(var.index, VARIABLE): - yield value, token - yield ']', SYNTAX - for value, token in self.tokenize(string[var.end:], orig_token): - yield value, token - - -class RowTokenizer(object): - - def __init__(self): - self._table = UnknownTable() - self._splitter = RowSplitter() - testcases = TestCaseTable() - settings = SettingTable(testcases.set_default_template) - variables = VariableTable() - keywords = KeywordTable() - self._tables = {'settings': settings, 'setting': settings, - 'metadata': settings, - 'variables': variables, 'variable': variables, - 'testcases': testcases, 'testcase': testcases, - 'keywords': keywords, 'keyword': keywords, - 'userkeywords': keywords, 'userkeyword': keywords} - - def tokenize(self, row): - commented = False - heading = False - for index, value in enumerate(self._splitter.split(row)): - # First value, and every second after that, is a separator. - index, separator = divmod(index-1, 2) - if value.startswith('#'): - commented = True - elif index == 0 and value.startswith('*'): - self._table = self._start_table(value) - heading = True - for value, token in self._tokenize(value, index, commented, - separator, heading): - yield value, token - self._table.end_row() - - def _start_table(self, header): - name = normalize(header, remove='*') - return self._tables.get(name, UnknownTable()) - - def _tokenize(self, value, index, commented, separator, heading): - if commented: - yield value, COMMENT - elif separator: - yield value, SEPARATOR - elif heading: - yield value, HEADING - else: - for value, token in self._table.tokenize(value, index): - yield value, token - - -class RowSplitter(object): - _space_splitter = re.compile('( {2,})') - _pipe_splitter = re.compile(r'((?:^| +)\|(?: +|$))') - - def split(self, row): - splitter = (row.startswith('| ') and self._split_from_pipes - or self._split_from_spaces) - for value in splitter(row): - yield value - yield '\n' - - def _split_from_spaces(self, row): - yield '' # Start with (pseudo)separator similarly as with pipes - for value in self._space_splitter.split(row): - yield value - - def _split_from_pipes(self, row): - _, separator, rest = self._pipe_splitter.split(row, 1) - yield separator - while self._pipe_splitter.search(rest): - cell, separator, rest = self._pipe_splitter.split(rest, 1) - yield cell - yield separator - yield rest - - -class Tokenizer(object): - _tokens = None - - def __init__(self): - self._index = 0 - - def tokenize(self, value): - values_and_tokens = self._tokenize(value, self._index) - self._index += 1 - if isinstance(values_and_tokens, type(Token)): - values_and_tokens = [(value, values_and_tokens)] - return values_and_tokens - - def _tokenize(self, value, index): - index = min(index, len(self._tokens) - 1) - return self._tokens[index] - - def _is_assign(self, value): - if value.endswith('='): - value = value[:-1].strip() - var = VariableSplitter(value, identifiers='$@&') - return var.start == 0 and var.end == len(value) - - -class Comment(Tokenizer): - _tokens = (COMMENT,) - - -class Setting(Tokenizer): - _tokens = (SETTING, ARGUMENT) - _keyword_settings = ('suitesetup', 'suiteprecondition', 'suiteteardown', - 'suitepostcondition', 'testsetup', 'testprecondition', - 'testteardown', 'testpostcondition', 'testtemplate') - _import_settings = ('library', 'resource', 'variables') - _other_settings = ('documentation', 'metadata', 'forcetags', 'defaulttags', - 'testtimeout') - _custom_tokenizer = None - - def __init__(self, template_setter=None): - Tokenizer.__init__(self) - self._template_setter = template_setter - - def _tokenize(self, value, index): - if index == 1 and self._template_setter: - self._template_setter(value) - if index == 0: - normalized = normalize(value) - if normalized in self._keyword_settings: - self._custom_tokenizer = KeywordCall(support_assign=False) - elif normalized in self._import_settings: - self._custom_tokenizer = ImportSetting() - elif normalized not in self._other_settings: - return ERROR - elif self._custom_tokenizer: - return self._custom_tokenizer.tokenize(value) - return Tokenizer._tokenize(self, value, index) - - -class ImportSetting(Tokenizer): - _tokens = (IMPORT, ARGUMENT) - - -class TestCaseSetting(Setting): - _keyword_settings = ('setup', 'precondition', 'teardown', 'postcondition', - 'template') - _import_settings = () - _other_settings = ('documentation', 'tags', 'timeout') - - def _tokenize(self, value, index): - if index == 0: - type = Setting._tokenize(self, value[1:-1], index) - return [('[', SYNTAX), (value[1:-1], type), (']', SYNTAX)] - return Setting._tokenize(self, value, index) - - -class KeywordSetting(TestCaseSetting): - _keyword_settings = ('teardown',) - _other_settings = ('documentation', 'arguments', 'return', 'timeout', 'tags') - - -class Variable(Tokenizer): - _tokens = (SYNTAX, ARGUMENT) - - def _tokenize(self, value, index): - if index == 0 and not self._is_assign(value): - return ERROR - return Tokenizer._tokenize(self, value, index) - - -class KeywordCall(Tokenizer): - _tokens = (KEYWORD, ARGUMENT) - - def __init__(self, support_assign=True): - Tokenizer.__init__(self) - self._keyword_found = not support_assign - self._assigns = 0 - - def _tokenize(self, value, index): - if not self._keyword_found and self._is_assign(value): - self._assigns += 1 - return SYNTAX # VariableTokenizer tokenizes this later. - if self._keyword_found: - return Tokenizer._tokenize(self, value, index - self._assigns) - self._keyword_found = True - return GherkinTokenizer().tokenize(value, KEYWORD) - - -class GherkinTokenizer(object): - _gherkin_prefix = re.compile('^(Given|When|Then|And) ', re.IGNORECASE) - - def tokenize(self, value, token): - match = self._gherkin_prefix.match(value) - if not match: - return [(value, token)] - end = match.end() - return [(value[:end], GHERKIN), (value[end:], token)] - - -class TemplatedKeywordCall(Tokenizer): - _tokens = (ARGUMENT,) - - -class ForLoop(Tokenizer): - - def __init__(self): - Tokenizer.__init__(self) - self._in_arguments = False - - def _tokenize(self, value, index): - token = self._in_arguments and ARGUMENT or SYNTAX - if value.upper() in ('IN', 'IN RANGE'): - self._in_arguments = True - return token - - -class _Table(object): - _tokenizer_class = None - - def __init__(self, prev_tokenizer=None): - self._tokenizer = self._tokenizer_class() - self._prev_tokenizer = prev_tokenizer - self._prev_values_on_row = [] - - def tokenize(self, value, index): - if self._continues(value, index): - self._tokenizer = self._prev_tokenizer - yield value, SYNTAX - else: - for value_and_token in self._tokenize(value, index): - yield value_and_token - self._prev_values_on_row.append(value) - - def _continues(self, value, index): - return value == '...' and all(self._is_empty(t) - for t in self._prev_values_on_row) - - def _is_empty(self, value): - return value in ('', '\\') - - def _tokenize(self, value, index): - return self._tokenizer.tokenize(value) - - def end_row(self): - self.__init__(prev_tokenizer=self._tokenizer) - - -class UnknownTable(_Table): - _tokenizer_class = Comment - - def _continues(self, value, index): - return False - - -class VariableTable(_Table): - _tokenizer_class = Variable - - -class SettingTable(_Table): - _tokenizer_class = Setting - - def __init__(self, template_setter, prev_tokenizer=None): - _Table.__init__(self, prev_tokenizer) - self._template_setter = template_setter - - def _tokenize(self, value, index): - if index == 0 and normalize(value) == 'testtemplate': - self._tokenizer = Setting(self._template_setter) - return _Table._tokenize(self, value, index) - - def end_row(self): - self.__init__(self._template_setter, prev_tokenizer=self._tokenizer) - - -class TestCaseTable(_Table): - _setting_class = TestCaseSetting - _test_template = None - _default_template = None - - @property - def _tokenizer_class(self): - if self._test_template or (self._default_template and - self._test_template is not False): - return TemplatedKeywordCall - return KeywordCall - - def _continues(self, value, index): - return index > 0 and _Table._continues(self, value, index) - - def _tokenize(self, value, index): - if index == 0: - if value: - self._test_template = None - return GherkinTokenizer().tokenize(value, TC_KW_NAME) - if index == 1 and self._is_setting(value): - if self._is_template(value): - self._test_template = False - self._tokenizer = self._setting_class(self.set_test_template) - else: - self._tokenizer = self._setting_class() - if index == 1 and self._is_for_loop(value): - self._tokenizer = ForLoop() - if index == 1 and self._is_empty(value): - return [(value, SYNTAX)] - return _Table._tokenize(self, value, index) - - def _is_setting(self, value): - return value.startswith('[') and value.endswith(']') - - def _is_template(self, value): - return normalize(value) == '[template]' - - def _is_for_loop(self, value): - return value.startswith(':') and normalize(value, remove=':') == 'for' - - def set_test_template(self, template): - self._test_template = self._is_template_set(template) - - def set_default_template(self, template): - self._default_template = self._is_template_set(template) - - def _is_template_set(self, template): - return normalize(template) not in ('', '\\', 'none', '${empty}') - - -class KeywordTable(TestCaseTable): - _tokenizer_class = KeywordCall - _setting_class = KeywordSetting - - def _is_template(self, value): - return False - - -# Following code copied directly from Robot Framework 2.7.5. - -class VariableSplitter: - - def __init__(self, string, identifiers): - self.identifier = None - self.base = None - self.index = None - self.start = -1 - self.end = -1 - self._identifiers = identifiers - self._may_have_internal_variables = False - try: - self._split(string) - except ValueError: - pass - else: - self._finalize() - - def get_replaced_base(self, variables): - if self._may_have_internal_variables: - return variables.replace_string(self.base) - return self.base - - def _finalize(self): - self.identifier = self._variable_chars[0] - self.base = ''.join(self._variable_chars[2:-1]) - self.end = self.start + len(self._variable_chars) - if self._has_list_or_dict_variable_index(): - self.index = ''.join(self._list_and_dict_variable_index_chars[1:-1]) - self.end += len(self._list_and_dict_variable_index_chars) - - def _has_list_or_dict_variable_index(self): - return self._list_and_dict_variable_index_chars\ - and self._list_and_dict_variable_index_chars[-1] == ']' - - def _split(self, string): - start_index, max_index = self._find_variable(string) - self.start = start_index - self._open_curly = 1 - self._state = self._variable_state - self._variable_chars = [string[start_index], '{'] - self._list_and_dict_variable_index_chars = [] - self._string = string - start_index += 2 - for index, char in enumerate(string[start_index:]): - index += start_index # Giving start to enumerate only in Py 2.6+ - try: - self._state(char, index) - except StopIteration: - return - if index == max_index and not self._scanning_list_variable_index(): - return - - def _scanning_list_variable_index(self): - return self._state in [self._waiting_list_variable_index_state, - self._list_variable_index_state] - - def _find_variable(self, string): - max_end_index = string.rfind('}') - if max_end_index == -1: - raise ValueError('No variable end found') - if self._is_escaped(string, max_end_index): - return self._find_variable(string[:max_end_index]) - start_index = self._find_start_index(string, 1, max_end_index) - if start_index == -1: - raise ValueError('No variable start found') - return start_index, max_end_index - - def _find_start_index(self, string, start, end): - index = string.find('{', start, end) - 1 - if index < 0: - return -1 - if self._start_index_is_ok(string, index): - return index - return self._find_start_index(string, index+2, end) - - def _start_index_is_ok(self, string, index): - return string[index] in self._identifiers\ - and not self._is_escaped(string, index) - - def _is_escaped(self, string, index): - escaped = False - while index > 0 and string[index-1] == '\\': - index -= 1 - escaped = not escaped - return escaped - - def _variable_state(self, char, index): - self._variable_chars.append(char) - if char == '}' and not self._is_escaped(self._string, index): - self._open_curly -= 1 - if self._open_curly == 0: - if not self._is_list_or_dict_variable(): - raise StopIteration - self._state = self._waiting_list_variable_index_state - elif char in self._identifiers: - self._state = self._internal_variable_start_state - - def _is_list_or_dict_variable(self): - return self._variable_chars[0] in ('@','&') - - def _internal_variable_start_state(self, char, index): - self._state = self._variable_state - if char == '{': - self._variable_chars.append(char) - self._open_curly += 1 - self._may_have_internal_variables = True - else: - self._variable_state(char, index) - - def _waiting_list_variable_index_state(self, char, index): - if char != '[': - raise StopIteration - self._list_and_dict_variable_index_chars.append(char) - self._state = self._list_variable_index_state - - def _list_variable_index_state(self, char, index): - self._list_and_dict_variable_index_chars.append(char) - if char == ']': - raise StopIteration |