Improve numeric literal support in Java lexer

Previously unsupported literals (binary, octal, hex floats) are now
recognized, and all regexes better match the Java Language
Specification (especially concerning underscores, suffixes, and the
many ways to write decimal floats).

2 files changed, 49 insertions, 4 deletions

diff --git a/pygments/lexers/jvm.py b/pygments/lexers/jvm.py
index 14647616..41fc0fdb 100644
--- a/pygments/lexers/jvm.py
+++ b/pygments/lexers/jvm.py
@@ -66,10 +66,19 @@ class JavaLexer(RegexLexer):
             (r'(\.)((?:[^\W\d]|\$)[\w$]*)', bygroups(Operator, Name.Attribute)),
             (r'^\s*([^\W\d]|\$)[\w$]*:', Name.Label),
             (r'([^\W\d]|\$)[\w$]*', Name),
+            (r'([0-9](_*[0-9]+)*\.([0-9](_*[0-9]+)*)?|'
+             r'([0-9](_*[0-9]+)*)?\.[0-9](_*[0-9]+)*)'
+             r'([eE][+\-]?[0-9](_*[0-9]+)*)?[fFdD]?|'
+             r'[0-9][eE][+\-]?[0-9](_*[0-9]+)*[fFdD]?|'
+             r'[0-9]([eE][+\-]?[0-9](_*[0-9]+)*)?[fFdD]|'
+             r'0[xX]([0-9a-fA-F](_*[0-9a-fA-F]+)*\.?|'
+             r'([0-9a-fA-F](_*[0-9a-fA-F]+)*)?\.[0-9a-fA-F](_*[0-9a-fA-F]+)*)'
+             r'[pP][+\-]?[0-9](_*[0-9]+)*[fFdD]?', Number.Float),
+            (r'0[xX][0-9a-fA-F](_*[0-9a-fA-F]+)*[lL]?', Number.Hex),
+            (r'0[bB][01](_*[01]+)*[lL]?', Number.Bin),
+            (r'0(_*[0-7]+)+[lL]?', Number.Oct),
+            (r'0|[1-9](_*[0-9]+)*[lL]?', Number.Integer),
             (r'[~^*!%&\[\](){}<>|+=:;,./?-]', Operator),
-            (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
-            (r'0x[0-9a-fA-F]+', Number.Hex),
-            (r'[0-9]+(_+[0-9]+)*L?', Number.Integer),
             (r'\n', Text)
         ],
         'class': [
diff --git a/tests/test_java.py b/tests/test_java.py
index 33a64e99..f4096647 100644
--- a/tests/test_java.py
+++ b/tests/test_java.py
@@ -9,7 +9,7 @@
 
 import unittest
 
-from pygments.token import Text, Name, Operator, Keyword
+from pygments.token import Text, Name, Operator, Keyword, Number
 from pygments.lexers import JavaLexer
 
 
@@ -40,3 +40,39 @@ class JavaTest(unittest.TestCase):
         ]
         self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
 
+    def testNumericLiterals(self):
+        fragment = '0 5L 9__542_72l 0xbEEf 0X9_A 0_35 01 0b0___101_0'
+        fragment += ' 0. .7_17F 3e-1_3d 1f 6_01.9e+3 0x.1Fp3 0XEP8D\n'
+        tokens = [
+            (Number.Integer, '0'),
+            (Text, ' '),
+            (Number.Integer, '5L'),
+            (Text, ' '),
+            (Number.Integer, '9__542_72l'),
+            (Text, ' '),
+            (Number.Hex, '0xbEEf'),
+            (Text, ' '),
+            (Number.Hex, '0X9_A'),
+            (Text, ' '),
+            (Number.Oct, '0_35'),
+            (Text, ' '),
+            (Number.Oct, '01'),
+            (Text, ' '),
+            (Number.Bin, '0b0___101_0'),
+            (Text, ' '),
+            (Number.Float, '0.'),
+            (Text, ' '),
+            (Number.Float, '.7_17F'),
+            (Text, ' '),
+            (Number.Float, '3e-1_3d'),
+            (Text, ' '),
+            (Number.Float, '1f'),
+            (Text, ' '),
+            (Number.Float, '6_01.9e+3'),
+            (Text, ' '),
+            (Number.Float, '0x.1Fp3'),
+            (Text, ' '),
+            (Number.Float, '0XEP8D'),
+            (Text, '\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))