Merged in jaingaurav2/pygments-main (pull request #321)

If encoding fails use ? as ANSI fallback character

13 files changed, 75 insertions, 23 deletions

diff --git a/pygments/lexers/agile.py b/pygments/lexers/agile.py
index 0a780a3e..b575f29c 100644
--- a/pygments/lexers/agile.py
+++ b/pygments/lexers/agile.py
@@ -1720,7 +1720,7 @@ class FactorLexer(RegexLexer):
             (r'"""\s+(?:.|\n)*?\s+"""', String),
             (r'"(?:\\\\|\\"|[^"])*"', String),
             (r'\S+"\s+(?:\\\\|\\"|[^"])*"', String),
-            (r'CHAR:\s+(\\[\\abfnrstv]|[^\\]\S+)\s', String.Char),
+            (r'CHAR:\s+(?:\\[\\abfnrstv]|[^\\]\S*)\s', String.Char),
 
             # comments
             (r'!\s+.*$', Comment),
@@ -1728,21 +1728,21 @@ class FactorLexer(RegexLexer):
             (r'/\*\s+(?:.|\n)*?\s\*/\s', Comment),
 
             # boolean constants
-            (r'(t|f)\s', Name.Constant),
+            (r'[tf]\s', Name.Constant),
 
             # symbols and literals
             (r'[\\$]\s+\S+', Name.Constant),
             (r'M\\\s+\S+\s+\S+', Name.Constant),
 
             # numbers
-            (r'[+-]?([\d,]*\d)?\.(\d([\d,]*\d)?)?([eE][+-]?\d+)?\s', Number),
-            (r'[+-]?\d([\d,]*\d)?([eE][+-]?\d+)?\s', Number),
-            (r'0x[a-fA-F\d]([a-fA-F\d,]*[a-fA-F\d])?(p\d([\d,]*\d)?)?\s', Number),
-            (r'NAN:\s+[a-fA-F\d]([a-fA-F\d,]*[a-fA-F\d])?(p\d([\d,]*\d)?)?\s', Number),
+            (r'[+-]?(?:[\d,]*\d)?\.(?:\d([\d,]*\d)?)?(?:[eE][+-]?\d+)?\s', Number),
+            (r'[+-]?\d(?:[\d,]*\d)?(?:[eE][+-]?\d+)?\s', Number),
+            (r'0x[a-fA-F\d](?:[a-fA-F\d,]*[a-fA-F\d])?(?:p\d([\d,]*\d)?)?\s', Number),
+            (r'NAN:\s+[a-fA-F\d](?:[a-fA-F\d,]*[a-fA-F\d])?(?:p\d([\d,]*\d)?)?\s', Number),
             (r'0b[01]+\s', Number),
             (r'0o[0-7]+\s', Number),
-            (r'(\d([\d,]*\d)?)?\+\d([\d,]*\d)?/\d([\d,]*\d)?\s', Number),
-            (r'(\-\d([\d,]*\d)?)?\-\d([\d,]*\d)?/\d([\d,]*\d)?\s', Number),
+            (r'(?:\d([\d,]*\d)?)?\+\d(?:[\d,]*\d)?/\d(?:[\d,]*\d)?\s', Number),
+            (r'(?:\-\d([\d,]*\d)?)?\-\d(?:[\d,]*\d)?/\d(?:[\d,]*\d)?\s', Number),
 
             # keywords
             (r'(?:deprecated|final|foldable|flushable|inline|recursive)\s',
diff --git a/pygments/lexers/compiled.py b/pygments/lexers/compiled.py
index 947282dd..82af5426 100644
--- a/pygments/lexers/compiled.py
+++ b/pygments/lexers/compiled.py
@@ -1622,9 +1622,9 @@ class FortranLexer(RegexLexer):
         ],
 
         'nums': [
-            (r'\d+(?![.Ee])', Number.Integer),
-            (r'[+-]?\d*\.\d+([eE][-+]?\d+)?', Number.Float),
-            (r'[+-]?\d+\.\d*([eE][-+]?\d+)?', Number.Float),
+            (r'\d+(?![.Ee])(_[a-z][a-z0-9_]+)?', Number.Integer),
+            (r'[+-]?\d*\.\d+([eE][-+]?\d+)?(_[a-z][a-z0-9_]+)?', Number.Float),
+            (r'[+-]?\d+\.\d*([eE][-+]?\d+)?(_[a-z][a-z0-9_]+)?', Number.Float),
         ],
     }
 
diff --git a/pygments/lexers/functional.py b/pygments/lexers/functional.py
index 122114fa..af9918a0 100644
--- a/pygments/lexers/functional.py
+++ b/pygments/lexers/functional.py
@@ -977,7 +977,7 @@ class HaskellLexer(RegexLexer):
             (r'\s+', Text),
             (r'[A-Z][a-zA-Z0-9_]*', Keyword.Type),
             (r'(_[\w\']+|[a-z][\w\']*)', Name.Function),
-            (r'--.*$', Comment.Single),
+            (r'--(?![!#$%&*+./<=>?@\^|_~:\\]).*?$', Comment.Single),
             (r'{-', Comment.Multiline, 'comment'),
             (r',', Punctuation),
             (r'[:!#$%&*+.\\/<=>?@^|~-]+', Operator),
diff --git a/pygments/lexers/other.py b/pygments/lexers/other.py
index c308c884..306ae326 100644
--- a/pygments/lexers/other.py
+++ b/pygments/lexers/other.py
@@ -1309,8 +1309,6 @@ class ModelicaLexer(RegexLexer):
              r'([\[\d,:\]]*)'
              r'(\.([a-zA-Z_][\w]*|[\'][^\']+[\']))+'
              r'([\[\d,:\]]*)', Name.Class),
-            (r'([a-zA-Z_][\w]*|[\'][^\']+[\'])'
-             r'([\[\d,:\]]+)', Name.Class),
             (r'(\'[\w\+\-\*\/\^]+\'|\w+)', Name),
             (r'[()\[\]{},.;]', Punctuation),
             (r'\'', Name, 'quoted_ident'),
@@ -1335,9 +1333,11 @@ class ModelicaLexer(RegexLexer):
         ],
         'functions': [
             (r'(abs|acos|acosh|asin|asinh|atan|atan2|atan3|ceil|cos|cosh|'
-             r'cross|div|exp|floor|getInstanceName|log|log10|mod|rem|'
-             r'semiLinear|sign|sin|sinh|size|spatialDistribution|sqrt|tan|'
-             r'tanh|zeros)\b', Name.Function),
+             r'cross|diagonal|div|exp|fill|floor|getInstanceName|identity|'
+             r'linspace|log|log10|matrix|mod|max|min|ndims|ones|outerProduct|'
+             r'product|rem|scalar|semiLinear|skew|sign|sin|sinh|size|'
+             r'spatialDistribution|sum|sqrt|symmetric|tan|tanh|transpose|'
+             r'vector|zeros)\b', Name.Function),
         ],
         'operators': [
             (r'(actualStream|and|assert|backSample|cardinality|change|Clock|'
@@ -1347,8 +1347,9 @@ class ModelicaLexer(RegexLexer):
              r'terminate)\b', Name.Builtin),
         ],
         'classes': [
-            (r'(operator)?(\s+)?(block|class|connector|end|function|model|operator|package|'
-             r'record|type)(\s+)((?!if|when|while)[A-Za-z_]\w*|[\'][^\']+[\'])([;]?)',
+            (r'(operator)?(\s+)?(block|class|connector|end|function|model|'
+             r'operator|package|record|type)(\s+)'
+             r'((?!if|for|when|while)[A-Za-z_]\w*|[\'][^\']+[\'])([;]?)',
              bygroups(Keyword, Text, Keyword, Text, Name.Class, Text))
         ],
         'quoted_ident': [
diff --git a/pygments/lexers/shell.py b/pygments/lexers/shell.py
index b069b375..c07ff6ee 100644
--- a/pygments/lexers/shell.py
+++ b/pygments/lexers/shell.py
@@ -71,8 +71,8 @@ class BashLexer(RegexLexer):
             (r'&', Punctuation),
             (r'\|', Punctuation),
             (r'\s+', Text),
-            (r'[^=\s\[\]{}()$"\'`\\<&|;]+', Text),
             (r'\d+(?= |\Z)', Number),
+            (r'[^=\s\[\]{}()$"\'`\\<&|;]+', Text),
             (r'\$#?(\w+|.)', Name.Variable),
             (r'<', Text),
         ],
@@ -91,6 +91,8 @@ class BashLexer(RegexLexer):
         'math': [
             (r'\)\)', Keyword, '#pop'),
             (r'[-+*/%^|&]|\*\*|\|\|', Operator),
+            (r'\d+#\d+', Number),
+            (r'\d+#(?! )', Number),
             (r'\d+', Number),
             include('root'),
         ],
diff --git a/pygments/lexers/text.py b/pygments/lexers/text.py
index db3200b3..6c2a4119 100644
--- a/pygments/lexers/text.py
+++ b/pygments/lexers/text.py
@@ -1681,6 +1681,12 @@ class CMakeLexer(RegexLexer):
         ]
     }
 
+    def analyse_text(text):
+        exp = r'^ *CMAKE_MINIMUM_REQUIRED *\( *VERSION *\d(\.\d)* *( FATAL_ERROR)? *\) *$'
+        if re.search(exp, text, flags=re.MULTILINE | re.IGNORECASE):
+            return 0.8
+        return 0.0
+
 
 class HttpLexer(RegexLexer):
     """
diff --git a/pygments/lexers/web.py b/pygments/lexers/web.py
index f8c89204..180c8e8a 100644
--- a/pygments/lexers/web.py
+++ b/pygments/lexers/web.py
@@ -2710,7 +2710,7 @@ class LiveScriptLexer(RegexLexer):
             ("'''", String, 'tsqs'),
             ('"', String, 'dqs'),
             ("'", String, 'sqs'),
-            (r'\\[\w$-]+', String),
+            (r'\\\S+', String),
             (r'<\[.*?\]>', String),
         ],
         'strings': [
diff --git a/tests/examplefiles/example.f90 b/tests/examplefiles/example.f90
new file mode 100644
index 00000000..40462189
--- /dev/null
+++ b/tests/examplefiles/example.f90
@@ -0,0 +1,8 @@
+program main
+  integer, parameter :: mykind = selected_real_kind()
+  print *, 1
+  print *, 1_mykind
+  print *, 1.
+  print *, 1._mykind
+  print *, (1., 1._mykind)
+end program main
diff --git a/tests/examplefiles/example.sh b/tests/examplefiles/example.sh
new file mode 100644
index 00000000..2112cdd1
--- /dev/null
+++ b/tests/examplefiles/example.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+printf "%d %s\n" 10 "foo"
+printf "%d %s\n" $((10#1)) "bar"
+
+let "m = 10#${1:1:2}"
+echo $m
+
+m=$((10#${1:4:3} + 10#${1:1:3}))
+echo $m
+
+m=$((10#${1:4:3}))
+echo $m
+
+m=$((10#$1))
+echo $m
+
+m=$((10#1))
+echo $m
+
+m=$((10))
+echo $m
diff --git a/tests/examplefiles/import.hs b/tests/examplefiles/import.hs
index 09058ae6..f266f62e 100644
--- a/tests/examplefiles/import.hs
+++ b/tests/examplefiles/import.hs
@@ -1,4 +1,15 @@
-import "mtl" Control.Monad.Trans
+import "base" Data.Char
+import "base" Data.Char (isControl, isSpace)
+import "base" Data.Char (isControl, --isSpace)
+ isSpace)
+import "base" Data.Char (isControl, -- isSpace)
+ isSpace)
+
+(-->) :: Num a => a -- signature
+(-->) = 2 -- >implementation
+
+--test comment
+-- test comment
 
 main :: IO ()
 main = putStrLn "hello world"
diff --git a/tests/examplefiles/livescript-demo.ls b/tests/examplefiles/livescript-demo.ls
index 16d1894a..03cbcc99 100644
--- a/tests/examplefiles/livescript-demo.ls
+++ b/tests/examplefiles/livescript-demo.ls
@@ -7,7 +7,7 @@ dashes-identifiers = ->
 underscores_i$d = ->
   /regexp1/
   //regexp2//g
-  'strings' and "strings" and \strings
+  'strings' and "strings" and \strings and \#$-"\'strings
 
 another-word-list = <[ more words ]>
 
diff --git a/tests/examplefiles/main.cmake b/tests/examplefiles/main.cmake
index dac3da43..71dc3ce7 100644
--- a/tests/examplefiles/main.cmake
+++ b/tests/examplefiles/main.cmake
@@ -1,3 +1,5 @@
+CMAKE_MINIMUM_REQUIRED(VERSION 2.6 FATAL_ERROR)
+
 SET( SOURCES back.c io.c main.c )
 MESSAGE( ${SOURCES}   )      # three arguments, prints "back.cio.cmain.c"
 MESSAGE( "${SOURCES}" )      # one argument,    prints "back.c;io.c;main.c"
diff --git a/tests/examplefiles/example.p b/tests/examplefiles/openedge_example
index e8c17e33..e8c17e33 100644
--- a/tests/examplefiles/example.p
+++ b/tests/examplefiles/openedge_example