Improve Pig lexer formatting and completness

1 files changed, 36 insertions, 20 deletions

diff --git a/pygments/lexers/jvm.py b/pygments/lexers/jvm.py
index 12d9890c..735729ea 100644
--- a/pygments/lexers/jvm.py
+++ b/pygments/lexers/jvm.py
@@ -1068,51 +1068,67 @@ class XtendLexer(RegexLexer):
     }
 
 class PigLexer(RegexLexer):
+    """
+    For `Pig Latin <https://pig.apache.org/>`_ source code.
+
+    .. versionadded:: 2.0
+    """
+
     name = 'Pig'
     aliases = ['pig']
     filenames = ['*.pig']
     mimetypes = ['text/x-pig']
 
-    flags = re.MULTILINE|re.IGNORECASE
+    flags = re.MULTILINE | re.IGNORECASE
 
     tokens = {
         'root': [
             (r'\s+', Text),
             (r'--.*', Comment),
-            (r'/\*\*([^*][^/]*)/', Comment.Multiline),
-            (r'/\*\*.*\*\*/$', Comment),
+            (r'/\*[\w\W]*?\*/', Comment.Multiline),
             (r'\\\n', Text),
             (r'\\', Text),
-            (r'\'[^\'^\n]+\'', String),
-            (r'\"[^\"^\n]+\"', String),
+            (r'\'(?:\\[ntbrf\\\']|\\u[0-9a-f]{4}|[^\'\\\n\r])*\'', String),
             include('keywords'),
             include('types'),
             include('builtins'),
-            (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
-            (r'0x[0-9a-fA-F]+', Number.Hex),
+            include('punct'),
+            include('operators'),
+            (r'[0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float),
+            (r'0x[0-9a-f]+', Number.Hex),
             (r'[0-9]+L?', Number.Integer),
             (r'\n', Text),
-            (r'([a-zA-Z_][a-zA-Z0-9_]*)(\s*)(\()',
+            (r'([a-z_][a-z0-9_]*)(\s*)(\()',
              bygroups(Name.Function, Text, Punctuation)),
             (r'[()#:]', Text),
-            (r'[^(:\n#\'\")\s]+', Text),
-            (r'\S+\s+', Text)
+            (r'[^(:#\'\")\s]+', Text),
+            (r'\S+\s+', Text) # TODO: make tests pass without \s+
         ],
         'keywords': [
             (r'(assert|and|any|all|arrange|as|asc|bag|by|cache|CASE|cat|cd|cp|'
              r'%declare|%default|define|dense|desc|describe|distinct|du|dump|'
-             r'eval|exex|explain|filter|flatten|foreach|full|generate|group|help|'
-             r'if|illustrate|import|inner|input|into|is|join|kill|left|limit|load|'
-             r'ls|map|matches|mkdir|mv|not|null|onschema|or|order|outer|output|'
-             r'parallel|pig|pwd|quit|register|returns|right|rm|rmf|rollup|run|sample|'
-             r'set|ship|split|stderr|stdin|stdout|store|stream|through|union|using|void)\b', Keyword)
+             r'eval|exex|explain|filter|flatten|foreach|full|generate|group|'
+             r'help|if|illustrate|import|inner|input|into|is|join|kill|left|'
+             r'limit|load|ls|map|matches|mkdir|mv|not|null|onschema|or|order|'
+             r'outer|output|parallel|pig|pwd|quit|register|returns|right|rm|'
+             r'rmf|rollup|run|sample|set|ship|split|stderr|stdin|stdout|store|'
+             r'stream|through|union|using|void)\b', Keyword)
         ],
         'builtins': [
-             (r'(AVG|BinStorage|cogroup|CONCAT|copyFromLocal|copyToLocal|COUNT|'
-              r'cross|DIFF|MAX|MIN|PigDump|PigStorage|SIZE|SUM|TextLoader|TOKENIZE)\b', Name.Builtin)
+            (r'(AVG|BinStorage|cogroup|CONCAT|copyFromLocal|copyToLocal|COUNT|'
+             r'cross|DIFF|MAX|MIN|PigDump|PigStorage|SIZE|SUM|TextLoader|'
+             r'TOKENIZE)\b', Name.Builtin)
+        ],
+        'types': [
+            (r'(bytearray|BIGINTEGER|BIGDECIMAL|chararray|datetime|double|float|'
+             r'int|long|tuple)\b', Keyword.Type)
+        ],
+        'punct': [
+            (r'[;(){}\[\]]', Punctuation),
         ],
-        'types':[
-             (r'(bytearray|BIGINTEGER|BIGDECIMAL|chararray|datetime|double|float|'
-              r'int|long|tuple)\b', Keyword.Type)
+        'operators': [
+            (r'[#=,./%+\-?]', Operator),
+            (r'(eq|gt|lt|gte|lte|neq|matches)\b', Operator),
+            (r'(==|<=|<|>=|>|!=)', Operator),
         ],
     }