Update SystemVerilog literal constants (#1460)

The original implementation was missing some of the more arcane features
such as underbars, the character 's' for signed/unsigned, support for
spaces before/after the base specifier, capital letter base specifiers
(ie 'B 'D 'H), and the 4-state 'xXzZ?' characters.

For regular integers, the 'l' and 'L' suffixes are not valid.
That is, unlike C, in Verilog '42L' is not a valid int literal.

Create a new test that exercises most of the interesting kinds of
SystemVerilog numbers.

This fixes a couple minor issues with what type of number the lexer
returns. For example, Numbers like '42' used to return Integer.Hex,
but now return Integer.Decimal.

2 files changed, 158 insertions, 14 deletions

diff --git a/pygments/lexers/hdl.py b/pygments/lexers/hdl.py
index 2e64b8e9..1472b5df 100644
--- a/pygments/lexers/hdl.py
+++ b/pygments/lexers/hdl.py
@@ -162,14 +162,22 @@ class SystemVerilogLexer(RegexLexer):
             (r'[{}#@]', Punctuation),
             (r'L?"', String, 'string'),
             (r"L?'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char),
+
             (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[lL]?', Number.Float),
             (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float),
-            (r'([0-9]+)|(\'h)[0-9a-fA-F]+', Number.Hex),
-            (r'([0-9]+)|(\'b)[01]+', Number.Bin),
-            (r'([0-9]+)|(\'d)[0-9]+', Number.Integer),
-            (r'([0-9]+)|(\'o)[0-7]+', Number.Oct),
-            (r'\'[01xz]', Number),
-            (r'\d+[Ll]?', Number.Integer),
+
+            (r'([1-9][_0-9]*)?\s*\'[sS]?[bB]\s*[xXzZ?01][_xXzZ?01]*',
+             Number.Bin),
+            (r'([1-9][_0-9]*)?\s*\'[sS]?[oO]\s*[xXzZ?0-7][_xXzZ?0-7]*',
+             Number.Oct),
+            (r'([1-9][_0-9]*)?\s*\'[sS]?[dD]\s*[xXzZ?0-9][_xXzZ?0-9]*',
+             Number.Integer),
+            (r'([1-9][_0-9]*)?\s*\'[sS]?[hH]\s*[xXzZ?0-9a-fA-F][_xXzZ?0-9a-fA-F]*',
+             Number.Hex),
+
+            (r'\'[01xXzZ]', Number),
+            (r'[0-9][_0-9]*', Number.Integer),
+
             (r'\*/', Error),
             (r'[~!%^&*+=|?:<>/-]', Operator),
             (r'[()\[\],.;\']', Punctuation),
diff --git a/tests/test_hdl.py b/tests/test_hdl.py
index 8e986ef0..ec4028ec 100644
--- a/tests/test_hdl.py
+++ b/tests/test_hdl.py
@@ -64,8 +64,7 @@ def test_systemverilog_basic(lexer):
         (Text, ' '),
         (Operator, '='),
         (Text, ' '),
-        # Note: This should be Number.Integer
-        (Number.Hex, '42'),
+        (Number.Integer, '42'),
         (Text, '\n'),
 
         (Punctuation, ')'),
@@ -81,9 +80,9 @@ def test_systemverilog_basic(lexer):
         (Punctuation, '['),
         (Name, 'N'),
         (Operator, '-'),
-        (Number.Hex, '1'),
+        (Number.Integer, '1'),
         (Operator, ':'),
-        (Number.Hex, '0'),
+        (Number.Integer, '0'),
         (Punctuation, ']'),
         (Text, ' '),
         (Name, 'y'),
@@ -108,10 +107,10 @@ def test_systemverilog_basic(lexer):
         (Punctuation, '['),
         (Name, 'N'),
         (Operator, '-'),
-        (Number.Hex, '1'),
+        (Number.Integer, '1'),
         # Note: This ':' should be Punctuation
         (Operator, ':'),
-        (Number.Hex, '0'),
+        (Number.Integer, '0'),
         (Punctuation, ']'),
         (Text, ' '),
         (Name, 'a'),
@@ -126,9 +125,9 @@ def test_systemverilog_basic(lexer):
         (Punctuation, '['),
         (Name, 'N'),
         (Operator, '-'),
-        (Number.Hex, '1'),
+        (Number.Integer, '1'),
         (Operator, ':'),
-        (Number.Hex, '0'),
+        (Number.Integer, '0'),
         (Punctuation, ']'),
         (Text, ' '),
         (Name, 'b'),
@@ -205,3 +204,140 @@ def test_systemverilog_basic(lexer):
         (Text, '\n'),
     ]
     assert list(lexer.get_tokens(SYSTEMVERILOG_BASIC_FRAGMENT)) == tokens
+
+
+# Believe it or not, SystemVerilog supports spaces before and after the base
+# specifier (ie 'b, 'd, 'h). See IEEE 1800-2017 Section 5.7.1 for examples.
+SVNUMS = """
+8'b10101010
+8 'b10101010
+8'b 10101010
+8'sb10101010
+8'Sb10101010
+8'B10101010
+8'b1010_1010
+8'b10xXzZ?10
+
+24'o01234567
+24 'o01234567
+24'o 01234567
+24'so01234567
+24'So01234567
+24'O01234567
+24'o0123_4567
+24'o01xXzZ?7
+
+32'd27182818
+32 'd27182818
+32'd 27182818
+32'sd27182818
+32'Sd27182818
+32'D27182818
+32'd2718_2818
+32'd27xXzZ?8
+
+32'hdeadbeef
+32 'hdeadbeef
+32'h deadbeef
+32'shdeadbeef
+32'Shdeadbeef
+32'Hdeadbeef
+32'hdead_beef
+32'hdexXzZ?f
+
+'0 '1 'x 'X 'z 'Z
+
+42 1234_5678
+"""
+
+def test_systemverilog_numbers(lexer):
+    """Test most types of numbers"""
+
+    tokens = [
+        (Number.Bin, "8'b10101010"),
+        (Text, '\n'),
+        (Number.Bin, "8 'b10101010"),
+        (Text, '\n'),
+        (Number.Bin, "8'b 10101010"),
+        (Text, '\n'),
+        (Number.Bin, "8'sb10101010"),
+        (Text, '\n'),
+        (Number.Bin, "8'Sb10101010"),
+        (Text, '\n'),
+        (Number.Bin, "8'B10101010"),
+        (Text, '\n'),
+        (Number.Bin, "8'b1010_1010"),
+        (Text, '\n'),
+        (Number.Bin, "8'b10xXzZ?10"),
+        (Text, '\n'),
+        (Text, '\n'),
+        (Number.Oct, "24'o01234567"),
+        (Text, '\n'),
+        (Number.Oct, "24 'o01234567"),
+        (Text, '\n'),
+        (Number.Oct, "24'o 01234567"),
+        (Text, '\n'),
+        (Number.Oct, "24'so01234567"),
+        (Text, '\n'),
+        (Number.Oct, "24'So01234567"),
+        (Text, '\n'),
+        (Number.Oct, "24'O01234567"),
+        (Text, '\n'),
+        (Number.Oct, "24'o0123_4567"),
+        (Text, '\n'),
+        (Number.Oct, "24'o01xXzZ?7"),
+        (Text, '\n'),
+        (Text, '\n'),
+        (Number.Integer, "32'd27182818"),
+        (Text, '\n'),
+        (Number.Integer, "32 'd27182818"),
+        (Text, '\n'),
+        (Number.Integer, "32'd 27182818"),
+        (Text, '\n'),
+        (Number.Integer, "32'sd27182818"),
+        (Text, '\n'),
+        (Number.Integer, "32'Sd27182818"),
+        (Text, '\n'),
+        (Number.Integer, "32'D27182818"),
+        (Text, '\n'),
+        (Number.Integer, "32'd2718_2818"),
+        (Text, '\n'),
+        (Number.Integer, "32'd27xXzZ?8"),
+        (Text, '\n'),
+        (Text, '\n'),
+        (Number.Hex, "32'hdeadbeef"),
+        (Text, '\n'),
+        (Number.Hex, "32 'hdeadbeef"),
+        (Text, '\n'),
+        (Number.Hex, "32'h deadbeef"),
+        (Text, '\n'),
+        (Number.Hex, "32'shdeadbeef"),
+        (Text, '\n'),
+        (Number.Hex, "32'Shdeadbeef"),
+        (Text, '\n'),
+        (Number.Hex, "32'Hdeadbeef"),
+        (Text, '\n'),
+        (Number.Hex, "32'hdead_beef"),
+        (Text, '\n'),
+        (Number.Hex, "32'hdexXzZ?f"),
+        (Text, '\n'),
+        (Text, '\n'),
+        (Number, "'0"),
+        (Text, ' '),
+        (Number, "'1"),
+        (Text, ' '),
+        (Number, "'x"),
+        (Text, ' '),
+        (Number, "'X"),
+        (Text, ' '),
+        (Number, "'z"),
+        (Text, ' '),
+        (Number, "'Z"),
+        (Text, '\n'),
+        (Text, '\n'),
+        (Number.Integer, '42'),
+        (Text, ' '),
+        (Number.Integer, '1234_5678'),
+        (Text, '\n'),
+    ]
+    assert list(lexer.get_tokens(SVNUMS)) == tokens