Matlabsession line continuation (#1399)

* Add explicit line continuation for Matlab session

Matlab lines can be explicitly continued with the ... syntax at the end
of a line. In the Session lexer, this requires continuing to the next
line to add more text. Otherwise, the next line is marked as output.

* The ellipses in Matlab should be a Keyword

The built-in Matlab syntax highlighter highlights ... with the same
formatting as a keyword. Everything after that on the line should be a
comment.

* Update Matlab functions and keywords from R2018a

* Fix many spaces in assignment formatted as string

In command mode, MATLAB allows mutiple space separated arguments to a
function which are interpreted as char arrays, and are formatted as
Strings. This check was also catching cases where there were multiple
spaces following an assignment or comparison operation and formatting
the rest of the line as a string. Now, if an = or operator is found, the
commandargs state is popped and control returns to the root state.

* Add tests for MATLAB formatting

1 files changed, 204 insertions, 0 deletions

diff --git a/tests/test_matlab.py b/tests/test_matlab.py
new file mode 100644
index 00000000..0ac1df95
--- /dev/null
+++ b/tests/test_matlab.py
@@ -0,0 +1,204 @@
+# -*- coding: utf-8 -*-
+"""
+    MATLAB Tests
+    ~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import pytest
+
+from pygments.token import Token
+from pygments.lexers import MatlabLexer
+
+
+@pytest.fixture(scope='module')
+def lexer():
+    yield MatlabLexer()
+
+
+def test_single_line(lexer):
+    """
+    Test that a single line with strings, a method, and numbers is parsed correctly.
+    """
+    fragment = "set('T',300,'P',101325);\n"
+    tokens = [
+        (Token.Name, 'set'),
+        (Token.Punctuation, '('),
+        (Token.Literal.String, "'"),
+        (Token.Literal.String, "T'"),
+        (Token.Punctuation, ','),
+        (Token.Literal.Number.Integer, '300'),
+        (Token.Punctuation, ','),
+        (Token.Literal.String, "'"),
+        (Token.Literal.String, "P'"),
+        (Token.Punctuation, ','),
+        (Token.Literal.Number.Integer, '101325'),
+        (Token.Punctuation, ')'),
+        (Token.Punctuation, ';'),
+        (Token.Text, '\n'),
+    ]
+    assert list(lexer.get_tokens(fragment)) == tokens
+
+
+def test_line_continuation(lexer):
+    """
+    Test that line continuation by ellipses does not produce generic
+    output on the second line.
+    """
+    fragment = "set('T',300,...\n'P',101325);\n"
+    tokens = [
+        (Token.Name, 'set'),
+        (Token.Punctuation, '('),
+        (Token.Literal.String, "'"),
+        (Token.Literal.String, "T'"),
+        (Token.Punctuation, ','),
+        (Token.Literal.Number.Integer, '300'),
+        (Token.Punctuation, ','),
+        (Token.Keyword, '...'),
+        (Token.Text, '\n'),
+        (Token.Literal.String, "'"),
+        (Token.Literal.String, "P'"),
+        (Token.Punctuation, ','),
+        (Token.Literal.Number.Integer, '101325'),
+        (Token.Punctuation, ')'),
+        (Token.Punctuation, ';'),
+        (Token.Text, '\n'),
+    ]
+    assert list(lexer.get_tokens(fragment)) == tokens
+
+
+def test_keywords_ended_by_newline(lexer):
+    """Test that keywords on their own line are marked as keywords."""
+    fragment = "if x > 100\n    disp('x > 100')\nelse\n    disp('x < 100')\nend\n"
+    tokens = [
+        (Token.Keyword, 'if'),
+        (Token.Text, ' '),
+        (Token.Name, 'x'),
+        (Token.Text, ' '),
+        (Token.Operator, '>'),
+        (Token.Text, ' '),
+        (Token.Literal.Number.Integer, '100'),
+        (Token.Text, '\n'),
+        (Token.Text, ' '),
+        (Token.Text, ' '),
+        (Token.Text, ' '),
+        (Token.Text, ' '),
+        (Token.Name.Builtin, 'disp'),
+        (Token.Punctuation, '('),
+        (Token.Literal.String, "'"),
+        (Token.Literal.String, "x > 100'"),
+        (Token.Punctuation, ')'),
+        (Token.Text, '\n'),
+        (Token.Keyword, 'else'),
+        (Token.Text, '\n'),
+        (Token.Text, ' '),
+        (Token.Text, ' '),
+        (Token.Text, ' '),
+        (Token.Text, ' '),
+        (Token.Name.Builtin, 'disp'),
+        (Token.Punctuation, '('),
+        (Token.Literal.String, "'"),
+        (Token.Literal.String, "x < 100'"),
+        (Token.Punctuation, ')'),
+        (Token.Text, '\n'),
+        (Token.Keyword, 'end'),
+        (Token.Text, '\n'),
+    ]
+    assert list(lexer.get_tokens(fragment)) == tokens
+
+
+def test_comment_after_continuation(lexer):
+    """
+    Test that text after the line continuation ellipses is marked as a comment.
+    """
+    fragment = "set('T',300,... a comment\n'P',101325);\n"
+    tokens = [
+        (Token.Name, 'set'),
+        (Token.Punctuation, '('),
+        (Token.Literal.String, "'"),
+        (Token.Literal.String, "T'"),
+        (Token.Punctuation, ','),
+        (Token.Literal.Number.Integer, '300'),
+        (Token.Punctuation, ','),
+        (Token.Keyword, '...'),
+        (Token.Comment, ' a comment'),
+        (Token.Text, '\n'),
+        (Token.Literal.String, "'"),
+        (Token.Literal.String, "P'"),
+        (Token.Punctuation, ','),
+        (Token.Literal.Number.Integer, '101325'),
+        (Token.Punctuation, ')'),
+        (Token.Punctuation, ';'),
+        (Token.Text, '\n'),
+    ]
+    assert list(lexer.get_tokens(fragment)) == tokens
+
+
+def test_multiple_spaces_variable_assignment(lexer):
+    """
+    Test that multiple spaces with an equal sign doesn't get formatted to a string.
+    """
+    fragment = 'x  = 100;\n'
+    tokens = [
+        (Token.Name, 'x'),
+        (Token.Text, ' '),
+        (Token.Text, ' '),
+        (Token.Punctuation, '='),
+        (Token.Text, ' '),
+        (Token.Literal.Number.Integer, '100'),
+        (Token.Punctuation, ';'),
+        (Token.Text, '\n'),
+    ]
+    assert list(lexer.get_tokens(fragment)) == tokens
+
+
+def test_operator_multiple_space(lexer):
+    """
+    Test that multiple spaces with an operator doesn't get formatted to a string.
+    """
+    fragment = 'x  > 100;\n'
+    tokens = [
+        (Token.Name, 'x'),
+        (Token.Text, ' '),
+        (Token.Text, ' '),
+        (Token.Operator, '>'),
+        (Token.Text, ' '),
+        (Token.Literal.Number.Integer, '100'),
+        (Token.Punctuation, ';'),
+        (Token.Text, '\n'),
+    ]
+    assert list(lexer.get_tokens(fragment)) == tokens
+
+
+def test_one_space_assignment(lexer):
+    """Test that one space before an equal sign is formatted correctly."""
+    fragment = 'x = 100;\n'
+    tokens = [
+        (Token.Name, 'x'),
+        (Token.Text, ' '),
+        (Token.Punctuation, '='),
+        (Token.Text, ' '),
+        (Token.Literal.Number.Integer, '100'),
+        (Token.Punctuation, ';'),
+        (Token.Text, '\n'),
+    ]
+    assert list(lexer.get_tokens(fragment)) == tokens
+
+
+def test_command_mode(lexer):
+    """
+    MATLAB allows char function arguments to not be enclosed by parentheses
+    or contain quote characters, as long as they are space separated. Test
+    that one common such function is formatted appropriately.
+    """
+    fragment = 'help sin\n'
+    tokens = [
+        (Token.Name, 'help'),
+        (Token.Text, ' '),
+        (Token.Literal.String, 'sin'),
+        (Token.Punctuation, ''),
+        (Token.Text, '\n'),
+    ]
+    assert list(lexer.get_tokens(fragment)) == tokens