Merged in kevinastone/pygments-main (pull request #720)

Added pep 515 support to the python lexer

10 files changed, 1077 insertions, 507 deletions

diff --git a/tests/examplefiles/StdGeneric.icl b/tests/examplefiles/StdGeneric.icl
index 2e6c3931..891b510a 100644
--- a/tests/examplefiles/StdGeneric.icl
+++ b/tests/examplefiles/StdGeneric.icl
@@ -1,5 +1,13 @@
 implementation module StdGeneric
 
+/**
+ * NOTE: this is a collection of different tricky parts of Clean modules (even
+ * though the file is simply called StdGeneric.icl). The code is taken from:
+ *
+ * - StdGeneric (StdEnv)
+ * - Graphics.Scalable.Image (Platform)
+ */
+
 import StdInt, StdMisc, StdClass, StdFunc
 
 generic bimap a b :: Bimap .a .b
@@ -89,4 +97,38 @@ where
 			= [ ConsLeft : doit i (n/2) ]
 		| otherwise
 			= [ ConsRight : doit (i - (n/2)) (n - (n/2)) ]
-			  	 							 	
\ No newline at end of file
+			  	 							 	
+:: NoAttr          m = NoAttr
+:: DashAttr        m = { dash        :: ![Int]    }
+:: FillAttr        m = { fill        :: !SVGColor }
+:: LineEndMarker   m = { endmarker   :: !Image m  }
+:: LineMidMarker   m = { midmarker   :: !Image m  }
+:: LineStartMarker m = { startmarker :: !Image m  }
+:: MaskAttr        m = { mask        :: !Image m  }
+:: OpacityAttr     m = { opacity     :: !Real     }
+:: StrokeAttr      m = { stroke      :: !SVGColor }
+:: StrokeWidthAttr m = { strokewidth :: !Span     }
+:: XRadiusAttr     m = { xradius     :: !Span     }
+:: YRadiusAttr     m = { yradius     :: !Span     }
+
+
+instance tuneImage NoAttr          where tuneImage image _    = image
+instance tuneImage DashAttr        where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgDashAttr attr.DashAttr.dash)) image
+instance tuneImage FillAttr        where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgFillAttr attr.FillAttr.fill)) image
+instance tuneImage LineEndMarker   where tuneImage image attr = Attr` (LineMarkerAttr` {LineMarkerAttr | markerImg = attr.LineEndMarker.endmarker, markerPos = LineMarkerEnd}) image
+instance tuneImage LineMidMarker   where tuneImage image attr = Attr` (LineMarkerAttr` {LineMarkerAttr | markerImg = attr.LineMidMarker.midmarker, markerPos = LineMarkerMid}) image
+instance tuneImage LineStartMarker where tuneImage image attr = Attr` (LineMarkerAttr` {LineMarkerAttr | markerImg = attr.LineStartMarker.startmarker, markerPos = LineMarkerStart}) image
+instance tuneImage MaskAttr        where tuneImage image attr = Attr` (MaskAttr` attr.MaskAttr.mask) image
+instance tuneImage OpacityAttr     where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgFillOpacityAttr attr.OpacityAttr.opacity)) image
+instance tuneImage StrokeAttr      where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgStrokeAttr      attr.StrokeAttr.stroke)) image
+instance tuneImage StrokeWidthAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgStrokeWidthAttr attr.StrokeWidthAttr.strokewidth)) image
+instance tuneImage XRadiusAttr     where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgXRadiusAttr     attr.XRadiusAttr.xradius)) image
+instance tuneImage YRadiusAttr     where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgYRadiusAttr     attr.YRadiusAttr.yradius)) image
+
+instance tuneImage DraggableAttr   where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerDraggableAttr   attr)) image
+instance tuneImage OnClickAttr     where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnClickAttr     attr)) image
+instance tuneImage OnMouseDownAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseDownAttr attr)) image
+instance tuneImage OnMouseMoveAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseMoveAttr attr)) image
+instance tuneImage OnMouseOutAttr  where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseOutAttr  attr)) image
+instance tuneImage OnMouseOverAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseOverAttr attr)) image
+instance tuneImage OnMouseUpAttr   where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseUpAttr   attr)) image
diff --git a/tests/examplefiles/docker.docker b/tests/examplefiles/docker.docker
index d65385b6..1ae3c3a1 100644
--- a/tests/examplefiles/docker.docker
+++ b/tests/examplefiles/docker.docker
@@ -1,5 +1,34 @@
-maintainer First O'Last
+FROM alpine:3.5
+MAINTAINER First O'Last
 
+# comment
 run echo \
   123 $bar
-# comment
+RUN apk --update add rsync dumb-init
+
+# Test env with both syntax
+ENV FOO = "BAR"
+ENV FOO \
+	"BAR"
+
+COPY foo "bar"
+COPY foo \
+	"bar"
+
+HEALTHCHECK \
+    --interval=5m --timeout=3s \
+    CMD curl -f http://localhost/ || exit 1
+
+# ONBUILD keyword, then with linebreak
+ONBUILD ADD . /app/src
+ONBUILD \
+	RUN echo 123 $bar
+
+# Potential JSON array parsing, mixed with linebreaks
+VOLUME \
+    /foo
+VOLUME \
+    ["/bar"]
+VOLUME ["/bar"]
+VOLUME /foo
+CMD ["foo", "bar"]
diff --git a/tests/examplefiles/example.hlsl b/tests/examplefiles/example.hlsl
new file mode 100644
index 00000000..21d0a672
--- /dev/null
+++ b/tests/examplefiles/example.hlsl
@@ -0,0 +1,168 @@
+// A few random snippets of HLSL shader code I gathered...

+

+[numthreads(256, 1, 1)]

+void cs_main(uint3 threadId : SV_DispatchThreadID)

+{

+	// Seed the PRNG using the thread ID

+	rng_state = threadId.x;

+

+	// Generate a few numbers...

+	uint r0 = rand_xorshift();

+	uint r1 = rand_xorshift();

+	// Do some stuff with them...

+

+	// Generate a random float in [0, 1)...

+	float f0 = float(rand_xorshift()) * (1.0 / 4294967296.0);

+

+	// ...etc.

+}

+

+// Constant buffer of parameters

+cbuffer IntegratorParams : register(b0)

+{

+	float2 specPow;		// Spec powers in XY directions (equal for isotropic BRDFs)

+	float3 L;			// Unit vector toward light 

+	int2 cThread;		// Total threads launched in XY dimensions

+	int2 xyOutput;		// Where in the output buffer to store the result

+}

+

+static const float pi = 3.141592654;

+

+float AshikhminShirleyNDF(float3 H)

+{

+	float normFactor = sqrt((specPow.x + 2.0f) * (specPow.y + 2.0)) * (0.5f / pi);

+	float NdotH = H.z;

+	float2 Hxy = normalize(H.xy);

+	return normFactor * pow(NdotH, dot(specPow, Hxy * Hxy));

+}

+

+float BeckmannNDF(float3 H)

+{

+	float glossFactor = specPow.x * 0.5f + 1.0f;	// This is 1/m^2 in the usual Beckmann formula

+	float normFactor = glossFactor * (1.0f / pi);

+	float NdotHSq = H.z * H.z;

+	return normFactor / (NdotHSq * NdotHSq) * exp(glossFactor * (1.0f - 1.0f / NdotHSq));

+}

+

+// Output buffer for compute shader (actually float, but must be declared as uint

+// for atomic operations to work)

+globallycoherent RWTexture2D<uint> o_data : register(u0);

+

+// Sum up the outputs of all threads and store to the output location

+static const uint threadGroupSize2D = 16;

+static const uint threadGroupSize1D = threadGroupSize2D * threadGroupSize2D;

+groupshared float g_partialSums[threadGroupSize1D];

+void SumAcrossThreadsAndStore(float value, uint iThreadInGroup)

+{

+	// First reduce within the threadgroup: partial sums of 2, 4, 8... elements

+	// are calculated by 1/2, 1/4, 1/8... of the threads, always keeping the

+	// active threads at the front of the group to minimize divergence.

+

+	// NOTE: there are faster ways of doing this...but this is simple to code

+	// and good enough.

+

+	g_partialSums[iThreadInGroup] = value;

+	GroupMemoryBarrierWithGroupSync();

+

+	[unroll] for (uint i = threadGroupSize1D / 2; i > 0; i /= 2)

+	{

+		if (iThreadInGroup < i)

+		{

+			g_partialSums[iThreadInGroup] += g_partialSums[iThreadInGroup + i];

+		}

+		GroupMemoryBarrierWithGroupSync();

+	}

+

+	// Then reduce across threadgroups: one thread from each group adds the group

+	// total to the final output location, using a software transactional memory

+	// style since D3D11 doesn't support atomic add on floats.

+	// (Assumes the output value has been cleared to zero beforehand.)

+

+	if (iThreadInGroup == 0)

+	{

+		float threadGroupSum = g_partialSums[0];

+		uint outputValueRead = o_data[xyOutput];

+		while (true)

+		{

+			uint newOutputValue = asuint(asfloat(outputValueRead) + threadGroupSum);

+			uint previousOutputValue;

+			InterlockedCompareExchange(

+				o_data[xyOutput], outputValueRead, newOutputValue, previousOutputValue);

+			if (previousOutputValue == outputValueRead)

+				break;

+			outputValueRead = previousOutputValue;

+		}

+	}

+}

+

+void main(

+	in Vertex i_vtx,

+	out Vertex o_vtx,

+	out float3 o_vecCamera : CAMERA,

+	out float4 o_uvzwShadow : UVZW_SHADOW,

+	out float4 o_posClip : SV_Position)

+{

+	o_vtx = i_vtx;

+	o_vecCamera = g_posCamera - i_vtx.m_pos;

+	o_uvzwShadow = mul(float4(i_vtx.m_pos, 1.0), g_matWorldToUvzwShadow);

+	o_posClip = mul(float4(i_vtx.m_pos, 1.0), g_matWorldToClip);

+}

+

+#pragma pack_matrix(row_major)

+

+struct Vertex

+{

+	float3		m_pos		: POSITION;

+	float3		m_normal	: NORMAL;

+	float2		m_uv		: UV;

+};

+

+cbuffer CBFrame : CB_FRAME					// matches struct CBFrame in test.cpp

+{

+	float4x4	g_matWorldToClip;

+	float4x4	g_matWorldToUvzwShadow;

+	float3x3	g_matWorldToUvzShadowNormal;

+	float3		g_posCamera;

+

+	float3		g_vecDirectionalLight;

+	float3		g_rgbDirectionalLight;

+

+	float2		g_dimsShadowMap;

+	float		g_normalOffsetShadow;

+	float		g_shadowSharpening;

+

+	float		g_exposure;					// Exposure multiplier

+}

+

+Texture2D<float3> g_texDiffuse : register(t0);

+SamplerState g_ss : register(s0);

+

+void main(

+	in Vertex i_vtx,

+	in float3 i_vecCamera : CAMERA,

+	in float4 i_uvzwShadow : UVZW_SHADOW,

+	out float3 o_rgb : SV_Target)

+{

+	float3 normal = normalize(i_vtx.m_normal);

+

+	// Sample shadow map

+	float shadow = EvaluateShadow(i_uvzwShadow, normal);

+

+	// Evaluate diffuse lighting

+	float3 diffuseColor = g_texDiffuse.Sample(g_ss, i_vtx.m_uv);

+	float3 diffuseLight = g_rgbDirectionalLight * (shadow * saturate(dot(normal, g_vecDirectionalLight)));

+	diffuseLight += SimpleAmbient(normal);

+

+	o_rgb = diffuseColor * diffuseLight;

+}

+

+[domain("quad")]

+void ds(

+	in float edgeFactors[4] : SV_TessFactor,

+	in float insideFactors[2] : SV_InsideTessFactor,

+	in OutputPatch<VData, 4> inp,

+	in float2 uv : SV_DomainLocation,

+	out float4 o_pos : SV_Position)

+{

+	o_pos = lerp(lerp(inp[0].pos, inp[1].pos, uv.x), lerp(inp[2].pos, inp[3].pos, uv.x), uv.y);

+}

diff --git a/tests/examplefiles/fennelview.fnl b/tests/examplefiles/fennelview.fnl
new file mode 100644
index 00000000..fd0fc648
--- /dev/null
+++ b/tests/examplefiles/fennelview.fnl
@@ -0,0 +1,156 @@
+;; A pretty-printer that outputs tables in Fennel syntax.
+;; Loosely based on inspect.lua: http://github.com/kikito/inspect.lua
+
+(local quote (fn [str] (.. '"' (: str :gsub '"' '\\"') '"')))
+
+(local short-control-char-escapes
+       {"\a" "\\a" "\b" "\\b" "\f" "\\f" "\n" "\\n"
+        "\r" "\\r" "\t" "\\t" "\v" "\\v"})
+
+(local long-control-char-esapes
+       (let [long {}]
+         (for [i 0 31]
+           (let [ch (string.char i)]
+             (when (not (. short-control-char-escapes ch))
+               (tset short-control-char-escapes ch (.. "\\" i))
+               (tset long ch (: "\\%03d" :format i)))))
+         long))
+
+(fn escape [str]
+  (let [str (: str :gsub "\\" "\\\\")
+        str (: str :gsub "(%c)%f[0-9]" long-control-char-esapes)]
+    (: str :gsub "%c" short-control-char-escapes)))
+
+(fn sequence-key? [k len]
+  (and (= (type k) "number")
+       (<= 1 k)
+       (<= k len)
+       (= (math.floor k) k)))
+
+(local type-order {:number 1 :boolean 2 :string 3 :table 4
+                   :function 5 :userdata 6 :thread 7})
+
+(fn sort-keys [a b]
+  (let [ta (type a) tb (type b)]
+    (if (and (= ta tb) (~= ta "boolean")
+             (or (= ta "string") (= ta "number")))
+        (< a b)
+        (let [dta (. type-order a)
+              dtb (. type-order b)]
+          (if (and dta dtb)
+              (< dta dtb)
+              dta true
+              dtb false
+              :else (< ta tb))))))
+
+(fn get-sequence-length [t]
+  (var len 1)
+  (each [i (ipairs t)] (set len i))
+  len)
+
+(fn get-nonsequential-keys [t]
+  (let [keys {}
+        sequence-length (get-sequence-length t)]
+    (each [k (pairs t)]
+      (when (not (sequence-key? k sequence-length))
+        (table.insert keys k)))
+    (table.sort keys sort-keys)
+    (values keys sequence-length)))
+
+(fn count-table-appearances [t appearances]
+  (if (= (type t) "table")
+      (when (not (. appearances t))
+        (tset appearances t 1)
+        (each [k v (pairs t)]
+          (count-table-appearances k appearances)
+          (count-table-appearances v appearances)))
+      (when (and t (= t t)) ; no nans please
+        (tset appearances t (+ (or (. appearances t) 0) 1))))
+  appearances)
+
+
+
+(var put-value nil) ; mutual recursion going on; defined below
+
+(fn puts [self ...]
+  (each [_ v (ipairs [...])]
+    (table.insert self.buffer v)))
+
+(fn tabify [self] (puts self "\n" (: self.indent :rep self.level)))
+
+(fn already-visited? [self v] (~= (. self.ids v) nil))
+
+(fn get-id [self v]
+  (var id (. self.ids v))
+  (when (not id)
+    (let [tv (type v)]
+      (set id (+ (or (. self.max-ids tv) 0) 1))
+      (tset self.max-ids tv id)
+      (tset self.ids v id)))
+  (tostring id))
+
+(fn put-sequential-table [self t length]
+  (puts self "[")
+  (set self.level (+ self.level 1))
+  (for [i 1 length]
+    (puts self " ")
+    (put-value self (. t i)))
+  (set self.level (- self.level 1))
+  (puts self " ]"))
+
+(fn put-key [self k]
+  (if (and (= (type k) "string")
+           (: k :find "^[-%w?\\^_`!#$%&*+./@~:|<=>]+$"))
+      (puts self ":" k)
+      (put-value self k)))
+
+(fn put-kv-table [self t]
+  (puts self "{")
+  (set self.level (+ self.level 1))
+  (each [k v (pairs t)]
+    (tabify self)
+    (put-key self k)
+    (puts self " ")
+    (put-value self v))
+  (set self.level (- self.level 1))
+  (tabify self)
+  (puts self "}"))
+
+(fn put-table [self t]
+  (if (already-visited? self t)
+      (puts self "#<table " (get-id self t) ">")
+      (>= self.level self.depth)
+      (puts self "{...}")
+      :else
+      (let [(non-seq-keys length) (get-nonsequential-keys t)
+            id (get-id self t)]
+        (if (> (. self.appearances t) 1)
+            (puts self "#<" id ">")
+            (and (= (# non-seq-keys) 0) (= (# t) 0))
+            (puts self "{}")
+            (= (# non-seq-keys) 0)
+            (put-sequential-table self t length)
+            :else
+            (put-kv-table self t)))))
+
+(set put-value (fn [self v]
+                 (let [tv (type v)]
+                   (if (= tv "string")
+                       (puts self (quote (escape v)))
+                       (or (= tv "number") (= tv "boolean") (= tv "nil"))
+                       (puts self (tostring v))
+                       (= tv "table")
+                       (put-table self v)
+                       :else
+                       (puts self "#<" (tostring v) ">")))))
+
+
+
+(fn fennelview [root options]
+  (let [options (or options {})
+        inspector {:appearances (count-table-appearances root {})
+                   :depth (or options.depth 128)
+                   :level 0 :buffer {} :ids {} :max-ids {}
+                   :indent (or options.indent "  ")}]
+    (put-value inspector root)
+    (table.concat inspector.buffer)))
diff --git a/tests/examplefiles/test.csd b/tests/examplefiles/test.csd
index 9122309b..6512d99e 100644
--- a/tests/examplefiles/test.csd
+++ b/tests/examplefiles/test.csd
@@ -1,260 +1,18 @@
+/*
+ * comment
+ */
+; comment
+// comment
+/
 <CsoundSynthesizer>
 <CsInstruments>
-// This is a Csound orchestra file for testing a Pygments <http://pygments.org>
-// lexer. Csound single-line comments can be preceded by a pair of forward
-// slashes...
-; ...or a semicolon.
-
-/* Block comments begin with /* and end with */
-
-// Orchestras begin with a header of audio parameters.
-nchnls = 1
-nchnls_i = 1
-sr = 44100
 0dbfs = 1
-ksmps = 10
-
-// The control rate kr = sr / ksmps can be omitted when the number of audio
-// samples in a control period (ksmps) is set, but kr may appear in older
-// orchestras.
-kr = 4410
-
-// Orchestras contain instruments. These begin with the keyword instr followed
-// by a comma-separated list of numbers or names of the instrument. Instruments
-// end at the endin keyword and cannot be nested.
-instr 1, N_a_M_e_, +Name
-  // Instruments contain statements. Here is a typical statement:
-  aSignal oscil 0dbfs, 440, 1
-  // Statements are terminated with a newline (possibly preceded by a comment).
-  // To write a statement on several lines, precede the newline with a
-  // backslash.
-  prints \
-    "hello, world\n";comment
-
-  // Csound 6 introduced function syntax for opcodes with one or zero outputs.
-  // The oscil statement above is the same as
-  aSignal = oscil(0dbfs, 440, 1)
-
-  // Instruments can contain control structures.
-  kNote = p3
-  if (kNote == 0) then
-    kFrequency = 220
-  elseif kNote == 1 then // Parentheses around binary expressions are optional.
-    kFrequency = 440
-  endif
-
-  // Csound 6 introduced looping structures.
-  iIndex = 0
-  while iIndex < 5 do
-    print iIndex
-    iIndex += 1
-  od
-  iIndex = 0
-  until iIndex >= 5 do
-    print iIndex
-    iIndex += 1
-  enduntil
-  // Both kinds of loops can be terminated by either od or enduntil.
-
-  // Single-line strings are enclosed in double-quotes.
-  prints "string\\\r\n\t\""
-  // Multi-line strings are enclosed in pairs of curly braces.
-  prints {{
-    hello,
-
-    world
-  }}
-
-  // Instruments often end with a statement containing an output opcode.
-  outc aSignal
-endin
-
-// Orchestras can also contain user-defined opcodes (UDOs). Here is an
-// oscillator with one audio-rate output and two control-rate inputs:
-opcode anOscillator, a, kk
-  kAmplitude, kFrequency xin
-  aSignal vco2 kAmplitude, kFrequency
-  xout aSignal
-endop
-instr TestOscillator
-  outc(anOscillator(0dbfs, 110))
-endin
-
-// Python can be executed in Csound
-// <http://www.csounds.com/manual/html/pyrun.html>. So can Lua
-// <http://www.csounds.com/manual/html/lua.html>.
-pyruni {{
-import random
-
-pool = [(1 + i / 10.0) ** 1.2 for i in range(100)]
-
-def get_number_from_pool(n, p):
-  if random.random() < p:
-    i = int(random.random() * len(pool))
-    pool[i] = n
-  return random.choice(pool)
-}}
-
-// The Csound preprocessor supports conditional compilation and including files.
-#ifdef DEBUG
-#undef DEBUG
-#include "filename.orc"
-#endif
-
-// The preprocessor also supports object- and function-like macros. This is an
-// object-like macro that defines a number:
-#define A_HZ #440#
-
-// This is a function-like macro:
-#define OSCIL_MACRO(VOLUME'FREQUENCY'TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE#
-
-// Bodies of macros are enclosed in # and can contain newlines. The arguments of
-// function-like macros are separated by single-quotes. Uses of macros are
-// prefixed with a dollar sign.
-instr TestMacro
-  aSignal $OSCIL_MACRO(1'$A_HZ'1)
-  // Not unlike PHP, macros expand in double-quoted strings.
-  prints "The frequency of the oscillator is $A_HZ Hz.\n"
-  out aSignal
-endin
-
-// Here are other things to note about Csound.
-
-// There are two bitwise NOT operators, ~ and ¬ (U+00AC). The latter is common
-// on keyboards in the United Kingdom
-// <https://en.wikipedia.org/wiki/British_and_American_keyboards>.
-instr TestBitwiseNOT
-  print ~42
-  print ¬42
-endin
-
-// Csound uses # for bitwise XOR, which the Csound manual calls bitwise
-// non-equivalence <http://www.csounds.com/manual/html/opnonequiv.html>.
-instr TestBitwiseXOR
-  print 0 # 0
-  print 0 # 1
-  print 1 # 0
-  print 1 # 1
-endin
-
-// Loops and if-then statements are relatively recent additions to Csound. There
-// are many flow-control opcodes that involve goto and labels.
-instr TestGoto
-  // This...
-  if p3 > 0 goto if_label
-  goto else_label
-if_label:
-  prints "if branch\n"
-  goto endif_label
-else_label:
-  prints "else branch\n"
-endif_label:
-
-  // ...is the same as this.
-  if p3 > 0 then
-    prints "if branch\n"
-  else
-    prints "else branch\n"
-  endif
-
-  // This...
-  iIndex = 0
-loop_label:
-  print iIndex
-  iIndex += 1
-  if iIndex < 10 goto loop_label
-
-  // ...is the same as this...
-  iIndex = 0
-loop_lt_label:
-  print iIndex
-  loop_lt iIndex, 1, 10, loop_lt_label
-
-  // ...and this.
-  iIndex = 0
-  while iIndex < 10 do
-    print iIndex
-    iIndex += 1
-  od
-endin
-
-// The prints and printks opcodes
-// <https://github.com/csound/csound/blob/develop/OOps/ugrw1.c#L831>, arguably
-// the primary methods of logging output, treat certain sequences of characters
-// different from printf in C.
-instr TestPrints
-  // ^ prints an ESCAPE character (U+001B), not a CIRCUMFLEX ACCENT character
-  // (U+005E). ^^ prints a CIRCUMFLEX ACCENT.
-  prints "^^\n"
-  // ~ prints an ESCAPE character (U+001B) followed by a [, not a TILDE
-  // character (U+007E). ~~ prints a TILDE.
-  prints "~~\n"
-  // \A, \B, \N, \R, and \T correspond to the escaped lowercase characters (that
-  // is, BELL (U+0007), BACKSPACE (U+0008), new line (U+000A), CARRIAGE RETURN
-  // (U+000D), and tab (U+0009)).
-  prints "\T\R\N"
-  // %n, %r, and %t are the same as \n, \r, and \t, as are %N, %R, and %T.
-  prints "%t%r%n"
-  // %! prints a semicolon. This is a hold-over from old versions of Csound that
-  // allowed comments to begin in strings.
-  prints "; %!\n"
-endin
-
-// The arguments of function-like macros can be separated by # instead of '.
-// These two lines define the same macro.
-#define OSCIL_MACRO(VOLUME'FREQUENCY'TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE#
-#define OSCIL_MACRO(VOLUME#FREQUENCY#TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE#
-
-// Uses of macros can optionally be suffixed with a period.
-instr TestMacroPeriodSuffix
-  aSignal $OSCIL_MACRO.(1'$A_HZ'1)
-  prints "The frequency of the oscillator is $A_HZ.Hz.\n"
-  out aSignal
-endin
-
-// Csound has @ and @@ operator-like macros that, when followed by a literal
-// non-negative integer, expand to the next power of 2 and the next power of 2
-// plus 1:
-//    @x = 2^(ceil(log2(x + 1))), x >= 0
-//   @@0 = 2
-//   @@x = 2^(ceil(log2(x))) + 1, x > 0
-// These macros are in
-// <https://github.com/csound/csound/blob/develop/Engine/csound_orc.l#L542> (and
-// <https://github.com/csound/csound/blob/develop/Engine/csound_sco.lex#L202>)
-// and are described at <http://www.csounds.com/manual/html/ScoreEval.html>.
-instr TestAt
-  prints "%d  %2d  %2d\n", 0, @0, @@0
-  prints "%d  %2d  %2d\n", 1, @1, @@1
-  prints "%d  %2d  %2d\n", 2, @2, @@2
-  prints "%d  %2d  %2d\n", 3, @3, @@3
-  prints "%d  %2d  %2d\n", 4, @4, @@4
-  prints "%d  %2d  %2d\n", 5, @5, @@5
-  prints "%d  %2d  %2d\n", 6, @6, @@6
-  prints "%d  %2d  %2d\n", 7, @7, @@7
-  prints "%d  %2d  %2d\n", 8, @8, @@8
-  prints "%d  %2d  %2d\n", 9, @9, @@9
-endin
-
-// Including newlines in macros can lead to confusing code, but it tests the
-// lexer.
-instr MacroAbuse
-  if 1 == 1 then
-    prints "on\n"
-#define FOO#
-BAR
-#endif // This ends the if statement. It is not a preprocessor directive.
-endin
+prints "hello, world\n"
 </CsInstruments>
 <CsScore>
-f 1 0 16384 10 1
-i "N_a_M_e_" 0 2
-i "TestOscillator" 2 2
-i "TestBitwiseNOT" 0 1
-i "TestBitwiseXOR" 0 1
-i "TestGoto" 0 1
-i "TestMacroPeriodSuffix" 4 1
-i "TestAt" 0 1
-i "MacroAbuse" 0 1
-e
+i 1 0 0
 </CsScore>
+<html>
+<!DOCTYPE html>
+</html>
 </CsoundSynthesizer>
diff --git a/tests/examplefiles/test.orc b/tests/examplefiles/test.orc
index 36725342..d113303e 100644
--- a/tests/examplefiles/test.orc
+++ b/tests/examplefiles/test.orc
@@ -1,257 +1,81 @@
-// This is a Csound orchestra file for testing a Pygments <http://pygments.org>
-// lexer. Csound single-line comments can be preceded by a pair of forward
-// slashes...
-; ...or a semicolon.
-
-/* Block comments begin with /* and end with */
-
-// Orchestras begin with a header of audio parameters.
-nchnls = 1
-nchnls_i = 1
-sr = 44100
-0dbfs = 1
-ksmps = 10
-
-// The control rate kr = sr / ksmps can be omitted when the number of audio
-// samples in a control period (ksmps) is set, but kr may appear in older
-// orchestras.
-kr = 4410
-
-// Orchestras contain instruments. These begin with the keyword instr followed
-// by a comma-separated list of numbers or names of the instrument. Instruments
-// end at the endin keyword and cannot be nested.
-instr 1, N_a_M_e_, +Name
-  // Instruments contain statements. Here is a typical statement:
-  aSignal oscil 0dbfs, 440, 1
-  // Statements are terminated with a newline (possibly preceded by a comment).
-  // To write a statement on several lines, precede the newline with a
-  // backslash.
-  prints \
-    "hello, world\n";comment
-
-  // Csound 6 introduced function syntax for opcodes with one or zero outputs.
-  // The oscil statement above is the same as
-  aSignal = oscil(0dbfs, 440, 1)
-
-  // Instruments can contain control structures.
-  kNote = p3
-  if (kNote == 0) then
-    kFrequency = 220
-  elseif kNote == 1 then // Parentheses around binary expressions are optional.
-    kFrequency = 440
-  endif
-
-  // Csound 6 introduced looping structures.
-  iIndex = 0
-  while iIndex < 5 do
-    print iIndex
-    iIndex += 1
-  od
-  iIndex = 0
-  until iIndex >= 5 do
-    print iIndex
-    iIndex += 1
-  enduntil
-  // Both kinds of loops can be terminated by either od or enduntil.
-
-  // Single-line strings are enclosed in double-quotes.
-  prints "string\\\r\n\t\""
-  // Multi-line strings are enclosed in pairs of curly braces.
-  prints {{
-    hello,
-
-    world
-  }}
-
-  // Instruments often end with a statement containing an output opcode.
-  outc aSignal
+/*
+ * comment
+ */
+; comment
+// comment
+
+instr/**/1,/**/N_a_M_e_,/**/+Name/**///
+  iDuration = p3
+  outc:a(aSignal)
 endin
 
-// Orchestras can also contain user-defined opcodes (UDOs). Here is an
-// oscillator with one audio-rate output and two control-rate inputs:
-opcode anOscillator, a, kk
-  kAmplitude, kFrequency xin
-  aSignal vco2 kAmplitude, kFrequency
-  xout aSignal
+opcode/**/aUDO,/**/i[],/**/aik//
+  aUDO
 endop
-instr TestOscillator
-  outc(anOscillator(0dbfs, 110))
-endin
 
-// Python can be executed in Csound
-// <http://www.csounds.com/manual/html/pyrun.html>. So can Lua
-// <http://www.csounds.com/manual/html/lua.html>.
-pyruni {{
-import random
+123 0123456789
+0xabcdef0123456789 0XABCDEF
+1e2 3e+4 5e-6 7E8 9E+0 1E-2 3. 4.56 .789
 
-pool = [(1 + i / 10.0) ** 1.2 for i in range(100)]
+"characters$MACRO."
+"\\\a\b\n\r\t\012\345\67\""
 
-def get_number_from_pool(n, p):
-  if random.random() < p:
-    i = int(random.random() * len(pool))
-    pool[i] = n
-  return random.choice(pool)
+{{
+characters$MACRO.
 }}
+{{\\\a\b\n\r\t\"\012\345\67}}
 
-// The Csound preprocessor supports conditional compilation and including files.
-#ifdef DEBUG
-#undef DEBUG
-#include "filename.orc"
-#endif
-
-// The preprocessor also supports object- and function-like macros. This is an
-// object-like macro that defines a number:
-#define A_HZ #440#
-
-// This is a function-like macro:
-#define OSCIL_MACRO(VOLUME'FREQUENCY'TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE#
-
-// Bodies of macros are enclosed in # and can contain newlines. The arguments of
-// function-like macros are separated by single-quotes. Uses of macros are
-// prefixed with a dollar sign.
-instr TestMacro
-  aSignal $OSCIL_MACRO(1'$A_HZ'1)
-  // Not unlike PHP, macros expand in double-quoted strings.
-  prints "The frequency of the oscillator is $A_HZ Hz.\n"
-  out aSignal
-endin
-
-// Here are other things to note about Csound.
++ - ~ ¬ ! * / ^ % << >> < > <= >= == != & # | && || ? : += -= *= /=
 
-// There are two bitwise NOT operators, ~ and ¬ (U+00AC). The latter is common
-// on keyboards in the United Kingdom
-// <https://en.wikipedia.org/wiki/British_and_American_keyboards>.
-instr TestBitwiseNOT
-  print ~42
-  print ¬42
-endin
-
-// Csound uses # for bitwise XOR, which the Csound manual calls bitwise
-// non-equivalence <http://www.csounds.com/manual/html/opnonequiv.html>.
-instr TestBitwiseXOR
-  print 0 # 0
-  print 0 # 1
-  print 1 # 0
-  print 1 # 1
-endin
+0dbfs A4 kr ksmps nchnls nchnls_i sr
 
-// Loops and if-then statements are relatively recent additions to Csound. There
-// are many flow-control opcodes that involve goto and labels.
-instr TestGoto
-  // This...
-  if p3 > 0 goto if_label
-  goto else_label
-if_label:
-  prints "if branch\n"
-  goto endif_label
-else_label:
-  prints "else branch\n"
-endif_label:
+do else elseif endif enduntil fi if ithen kthen od then until while
+return rireturn
 
-  // ...is the same as this.
-  if p3 > 0 then
-    prints "if branch\n"
-  else
-    prints "else branch\n"
-  endif
+aLabel:
+ label2:
 
-  // This...
-  iIndex = 0
-loop_label:
-  print iIndex
-  iIndex += 1
-  if iIndex < 10 goto loop_label
+goto aLabel
+reinit aLabel
+cggoto 1==0, aLabel
+timout 0, 0, aLabel
+loop_ge 0, 0, 0, aLabel
 
-  // ...is the same as this...
-  iIndex = 0
-loop_lt_label:
-  print iIndex
-  loop_lt iIndex, 1, 10, loop_lt_label
-
-  // ...and this.
-  iIndex = 0
-  while iIndex < 10 do
-    print iIndex
-    iIndex += 1
-  od
-endin
+prints "%! %% %n%N %r%R %t%T \\a\\A \\b\\B \\n\\N \\r\\R \\t\\T"
+prints Soutput
 
-// The prints and printks opcodes
-// <https://github.com/csound/csound/blob/develop/OOps/ugrw1.c#L831>, arguably
-// the primary methods of logging output, treat certain sequences of characters
-// different from printf in C.
-instr TestPrints
-  // ^ prints an ESCAPE character (U+001B), not a CIRCUMFLEX ACCENT character
-  // (U+005E). ^^ prints a CIRCUMFLEX ACCENT.
-  prints "^^\n"
-  // ~ prints an ESCAPE character (U+001B) followed by a [, not a TILDE
-  // character (U+007E). ~~ prints a TILDE.
-  prints "~~\n"
-  // \A, \B, \N, \R, and \T correspond to the escaped lowercase characters (that
-  // is, BELL (U+0007), BACKSPACE (U+0008), new line (U+000A), CARRIAGE RETURN
-  // (U+000D), and tab (U+0009)).
-  prints "\T\R\N"
-  // %n, %r, and %t are the same as \n, \r, and \t, as are %N, %R, and %T.
-  prints "%t%r%n"
-  // %! prints a semicolon. This is a hold-over from old versions of Csound that
-  // allowed comments to begin in strings.
-  prints "; %!\n"
-endin
-
-// The arguments of function-like macros can be separated by # instead of '.
-// These two lines define the same macro.
-#define OSCIL_MACRO(VOLUME'FREQUENCY'TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE#
-#define OSCIL_MACRO(VOLUME#FREQUENCY#TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE#
-
-// Uses of macros can optionally be suffixed with a period.
-instr TestMacroPeriodSuffix
-  aSignal $OSCIL_MACRO.(1'$A_HZ'1)
-  prints "The frequency of the oscillator is $A_HZ.Hz.\n"
-  out aSignal
-endin
-
-// Csound has @ and @@ operator-like macros that, when followed by a literal
-// non-negative integer, expand to the next power of 2 and the next power of 2
-// plus 1:
-//    @x = 2^(ceil(log2(x + 1))), x >= 0
-//   @@0 = 2
-//   @@x = 2^(ceil(log2(x))) + 1, x > 0
-// These macros are in
-// <https://github.com/csound/csound/blob/develop/Engine/csound_orc.l#L542> (and
-// <https://github.com/csound/csound/blob/develop/Engine/csound_sco.lex#L202>)
-// and are described at <http://www.csounds.com/manual/html/ScoreEval.html>.
-instr TestAt
-  prints "%d  %2d  %2d\n", 0, @0, @@0
-  prints "%d  %2d  %2d\n", 1, @1, @@1
-  prints "%d  %2d  %2d\n", 2, @2, @@2
-  prints "%d  %2d  %2d\n", 3, @3, @@3
-  prints "%d  %2d  %2d\n", 4, @4, @@4
-  prints "%d  %2d  %2d\n", 5, @5, @@5
-  prints "%d  %2d  %2d\n", 6, @6, @@6
-  prints "%d  %2d  %2d\n", 7, @7, @@7
-  prints "%d  %2d  %2d\n", 8, @8, @@8
-  prints "%d  %2d  %2d\n", 9, @9, @@9
-endin
+readscore {{
+i 1 0 0
+}}
+pyrun {{
+# Python
+}}
+lua_exec {{
+-- Lua
+}}
 
-// Including newlines in macros can lead to confusing code, but it tests the
-// lexer.
-instr MacroAbuse
-  if 1 == 1 then
-    prints "on\n"
-#define FOO#
-BAR
-#endif // This ends the if statement. It is not a preprocessor directive.
-endin
+#include/**/"file.udo"
+#include/**/|file.udo|
 
-scoreline_i {{
-f 1 0 16384 10 1
-i "N_a_M_e_" 0 2
-i "TestOscillator" 2 2
-i "TestBitwiseNOT" 0 1
-i "TestBitwiseXOR" 0 1
-i "TestGoto" 0 1
-i "TestMacroPeriodSuffix" 4 1
-i "TestAt" 0 1
-i "MacroAbuse" 0 1
-e
-}}
+#ifdef MACRO
+#else
+#ifndef MACRO
+#endif
+#undef MACRO
+
+#   define MACRO#macro_body#
+#define/**/
+MACRO/**/
+#\#macro
+body\##
+
+#define MACRO(ARG1#ARG2) #macro_body#
+#define/**/
+MACRO(ARG1'ARG2'ARG3)/**/
+#\#macro
+body\##
+
+$MACRO $MACRO.
+$MACRO(x)
+@0
+@@ 1
diff --git a/tests/examplefiles/test.sco b/tests/examplefiles/test.sco
index a0b39251..d997c1b3 100644
--- a/tests/examplefiles/test.sco
+++ b/tests/examplefiles/test.sco
@@ -1,10 +1,22 @@
-f 1 0 16384 10 1
-i "N_a_M_e_" 0 2
-i "TestOscillator" 2 2
-i "TestBitwiseNOT" 0 1
-i "TestBitwiseXOR" 0 1
-i "TestGoto" 0 1
-i "TestMacroPeriodSuffix" 4 1
-i "TestAt" 0 1
-i "MacroAbuse" 0 1
-e
+/*
+ * comment
+ */
+; comment
+// comment
+a b C d e f i q s t v x y
+z
+np0 nP1 Np2 NP3
+m/**/label;
+n label
+123 0123456789
+0xabcdef0123456789 0XABCDEF
+1e2 3e+4 5e-6 7E8 9E+0 1E-2 3. 4.56 .789
+"characters$MACRO."
+{ 1 I
+  { 2 J
+    { 3 K
+      $I $J $K
+    }
+  }
+}
+#include "score.sco"
diff --git a/tests/test_csound.py b/tests/test_csound.py
new file mode 100644
index 00000000..4d10c267
--- /dev/null
+++ b/tests/test_csound.py
@@ -0,0 +1,480 @@
+# -*- coding: utf-8 -*-
+"""
+    Csound lexer tests
+    ~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import unittest
+from textwrap import dedent
+
+from pygments.token import Comment, Error, Keyword, Name, Number, Operator, Punctuation, \
+    String, Text
+from pygments.lexers import CsoundOrchestraLexer
+
+
+class CsoundOrchestraTest(unittest.TestCase):
+
+    def setUp(self):
+        self.lexer = CsoundOrchestraLexer()
+        self.maxDiff = None
+
+    def testComments(self):
+        fragment = dedent('''\
+            /*
+             * comment
+             */
+            ; comment
+            // comment
+        ''')
+        tokens = [
+            (Comment.Multiline, u'/*\n * comment\n */'),
+            (Text, u'\n'),
+            (Comment.Single, u'; comment'),
+            (Text, u'\n'),
+            (Comment.Single, u'// comment'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testInstrumentBlocks(self):
+        fragment = dedent('''\
+            instr/**/1,/**/N_a_M_e_,/**/+Name/**///
+              iDuration = p3
+              outc:a(aSignal)
+            endin
+        ''')
+        tokens = [
+            (Keyword.Declaration, u'instr'),
+            (Comment.Multiline, u'/**/'),
+            (Name.Function, u'1'),
+            (Punctuation, u','),
+            (Comment.Multiline, u'/**/'),
+            (Name.Function, u'N_a_M_e_'),
+            (Punctuation, u','),
+            (Comment.Multiline, u'/**/'),
+            (Punctuation, u'+'),
+            (Name.Function, u'Name'),
+            (Comment.Multiline, u'/**/'),
+            (Comment.Single, u'//'),
+            (Text, u'\n'),
+            (Text, u'  '),
+            (Keyword.Type, u'i'),
+            (Name, u'Duration'),
+            (Text, u' '),
+            (Operator, u'='),
+            (Text, u' '),
+            (Name.Variable.Instance, u'p3'),
+            (Text, u'\n'),
+            (Text, u'  '),
+            (Name.Builtin, u'outc'),
+            (Punctuation, u':'),
+            (Keyword.Type, u'a'),
+            (Punctuation, u'('),
+            (Keyword.Type, u'a'),
+            (Name, u'Signal'),
+            (Punctuation, u')'),
+            (Text, u'\n'),
+            (Keyword.Declaration, u'endin'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testUserDefinedOpcodes(self):
+        fragment = dedent('''\
+            opcode/**/aUDO,/**/i[],/**/aik//
+              aUDO
+            endop
+        ''')
+        tokens = [
+            (Keyword.Declaration, u'opcode'),
+            (Comment.Multiline, u'/**/'),
+            (Name.Function, u'aUDO'),
+            (Punctuation, u','),
+            (Comment.Multiline, u'/**/'),
+            (Keyword.Type, u'i[]'),
+            (Punctuation, u','),
+            (Comment.Multiline, u'/**/'),
+            (Keyword.Type, u'aik'),
+            (Comment.Single, u'//'),
+            (Text, u'\n'),
+            (Text, u'  '),
+            (Name.Function, u'aUDO'),
+            (Text, u'\n'),
+            (Keyword.Declaration, u'endop'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testNumbers(self):
+        fragment = '123 0123456789'
+        tokens = [
+            (Number.Integer, u'123'),
+            (Text, u' '),
+            (Number.Integer, u'0123456789'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        fragment = '0xabcdef0123456789 0XABCDEF'
+        tokens = [
+            (Keyword.Type, u'0x'),
+            (Number.Hex, u'abcdef0123456789'),
+            (Text, u' '),
+            (Keyword.Type, u'0X'),
+            (Number.Hex, u'ABCDEF'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        fragments = ['1e2', '3e+4', '5e-6', '7E8', '9E+0', '1E-2', '3.', '4.56', '.789']
+        for fragment in fragments:
+            tokens = [
+                (Number.Float, fragment),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testQuotedStrings(self):
+        fragment = '"characters$MACRO."'
+        tokens = [
+            (String, u'"'),
+            (String, u'characters'),
+            (Comment.Preproc, u'$MACRO.'),
+            (String, u'"'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testBracedStrings(self):
+        fragment = dedent('''\
+            {{
+            characters$MACRO.
+            }}
+        ''')
+        tokens = [
+            (String, u'{{'),
+            (String, u'\ncharacters$MACRO.\n'),
+            (String, u'}}'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testEscapeSequences(self):
+        for character in ['\\', 'a', 'b', 'n', 'r', 't', '"', '012', '345', '67']:
+            escapedCharacter = '\\' + character
+            fragment = '"' + escapedCharacter + '"'
+            tokens = [
+                (String, u'"'),
+                (String.Escape, escapedCharacter),
+                (String, u'"'),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+            fragment = '{{' + escapedCharacter + '}}'
+            tokens = [
+                (String, u'{{'),
+                (String.Escape, escapedCharacter),
+                (String, u'}}'),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testOperators(self):
+        fragments = ['+', '-', '~', u'¬', '!', '*', '/', '^', '%', '<<', '>>', '<', '>',
+                     '<=', '>=', '==', '!=', '&', '#', '|', '&&', '||', '?', ':', '+=',
+                     '-=', '*=', '/=']
+        for fragment in fragments:
+            tokens = [
+                (Operator, fragment),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testGlobalValueIdentifiers(self):
+        for fragment in ['0dbfs', 'A4', 'kr', 'ksmps', 'nchnls', 'nchnls_i', 'sr']:
+            tokens = [
+                (Name.Variable.Global, fragment),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testKeywords(self):
+        fragments = ['do', 'else', 'elseif', 'endif', 'enduntil', 'fi', 'if', 'ithen',
+                     'kthen', 'od', 'then', 'until', 'while']
+        for fragment in fragments:
+            tokens = [
+                (Keyword, fragment),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        for fragment in ['return', 'rireturn']:
+            tokens = [
+                (Keyword.Pseudo, fragment),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testLabels(self):
+        fragment = dedent('''\
+            aLabel:
+             label2:
+        ''')
+        tokens = [
+            (Name.Label, u'aLabel'),
+            (Punctuation, u':'),
+            (Text, u'\n'),
+            (Text, u' '),
+            (Name.Label, u'label2'),
+            (Punctuation, u':'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testPrintksAndPrintsEscapeSequences(self):
+        escapedCharacters = ['%!', '%%', '%n', '%N', '%r', '%R', '%t', '%T', '\\\\a',
+                             '\\\\A', '\\\\b', '\\\\B', '\\\\n', '\\\\N', '\\\\r',
+                             '\\\\R', '\\\\t', '\\\\T']
+        for opcode in ['printks', 'prints']:
+            for escapedCharacter in escapedCharacters:
+                fragment = opcode + ' "' + escapedCharacter + '"'
+                tokens = [
+                    (Name.Builtin, opcode),
+                    (Text, u' '),
+                    (String, u'"'),
+                    (String.Escape, escapedCharacter),
+                    (String, u'"'),
+                    (Text, u'\n')
+                ]
+                self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testGotoStatements(self):
+        for keyword in ['goto', 'igoto', 'kgoto']:
+            fragment = keyword + ' aLabel'
+            tokens = [
+                (Keyword, keyword),
+                (Text, u' '),
+                (Name.Label, u'aLabel'),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        for opcode in ['reinit', 'rigoto', 'tigoto']:
+            fragment = opcode + ' aLabel'
+            tokens = [
+                (Keyword.Pseudo, opcode),
+                (Text, u' '),
+                (Name.Label, u'aLabel'),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        for opcode in ['cggoto', 'cigoto', 'cingoto', 'ckgoto', 'cngoto', 'cnkgoto']:
+            fragment = opcode + ' 1==0, aLabel'
+            tokens = [
+                (Keyword.Pseudo, opcode),
+                (Text, u' '),
+                (Number.Integer, u'1'),
+                (Operator, u'=='),
+                (Number.Integer, u'0'),
+                (Punctuation, u','),
+                (Text, u' '),
+                (Name.Label, u'aLabel'),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        fragment = 'timout 0, 0, aLabel'
+        tokens = [
+            (Keyword.Pseudo, 'timout'),
+            (Text, u' '),
+            (Number.Integer, u'0'),
+            (Punctuation, u','),
+            (Text, u' '),
+            (Number.Integer, u'0'),
+            (Punctuation, u','),
+            (Text, u' '),
+            (Name.Label, u'aLabel'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        for opcode in ['loop_ge', 'loop_gt', 'loop_le', 'loop_lt']:
+            fragment = opcode + ' 0, 0, 0, aLabel'
+            tokens = [
+                (Keyword.Pseudo, opcode),
+                (Text, u' '),
+                (Number.Integer, u'0'),
+                (Punctuation, u','),
+                (Text, u' '),
+                (Number.Integer, u'0'),
+                (Punctuation, u','),
+                (Text, u' '),
+                (Number.Integer, u'0'),
+                (Punctuation, u','),
+                (Text, u' '),
+                (Name.Label, u'aLabel'),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testIncludeDirectives(self):
+        for character in ['"', '|']:
+            fragment = '#include/**/' + character + 'file.udo' + character
+            tokens = [
+                (Comment.Preproc, u'#include'),
+                (Comment.Multiline, u'/**/'),
+                (String, character + u'file.udo' + character),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testObjectLikeMacroDefinitions(self):
+        fragment = dedent('''\
+            # \tdefine MACRO#macro_body#
+            #define/**/
+            MACRO/**/
+            #\\#macro
+            body\\##
+        ''')
+        tokens = [
+            (Comment.Preproc, u'# \tdefine'),
+            (Text, u' '),
+            (Comment.Preproc, u'MACRO'),
+            (Punctuation, u'#'),
+            (Comment.Preproc, u'macro_body'),
+            (Punctuation, u'#'),
+            (Text, u'\n'),
+            (Comment.Preproc, u'#define'),
+            (Comment.Multiline, u'/**/'),
+            (Text, u'\n'),
+            (Comment.Preproc, u'MACRO'),
+            (Comment.Multiline, u'/**/'),
+            (Text, u'\n'),
+            (Punctuation, u'#'),
+            (Comment.Preproc, u'\\#'),
+            (Comment.Preproc, u'macro\nbody'),
+            (Comment.Preproc, u'\\#'),
+            (Punctuation, u'#'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testFunctionLikeMacroDefinitions(self):
+        fragment = dedent('''\
+            #define MACRO(ARG1#ARG2) #macro_body#
+            #define/**/
+            MACRO(ARG1'ARG2' ARG3)/**/
+            #\\#macro
+            body\\##
+        ''')
+        tokens = [
+            (Comment.Preproc, u'#define'),
+            (Text, u' '),
+            (Comment.Preproc, u'MACRO'),
+            (Punctuation, u'('),
+            (Comment.Preproc, u'ARG1'),
+            (Punctuation, u'#'),
+            (Comment.Preproc, u'ARG2'),
+            (Punctuation, u')'),
+            (Text, u' '),
+            (Punctuation, u'#'),
+            (Comment.Preproc, u'macro_body'),
+            (Punctuation, u'#'),
+            (Text, u'\n'),
+            (Comment.Preproc, u'#define'),
+            (Comment.Multiline, u'/**/'),
+            (Text, u'\n'),
+            (Comment.Preproc, u'MACRO'),
+            (Punctuation, u'('),
+            (Comment.Preproc, u'ARG1'),
+            (Punctuation, u"'"),
+            (Comment.Preproc, u'ARG2'),
+            (Punctuation, u"'"),
+            (Text, u' '),
+            (Comment.Preproc, u'ARG3'),
+            (Punctuation, u')'),
+            (Comment.Multiline, u'/**/'),
+            (Text, u'\n'),
+            (Punctuation, u'#'),
+            (Comment.Preproc, u'\\#'),
+            (Comment.Preproc, u'macro\nbody'),
+            (Comment.Preproc, u'\\#'),
+            (Punctuation, u'#'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testMacroPreprocessorDirectives(self):
+        for directive in ['#ifdef', '#ifndef', '#undef']:
+            fragment = directive + ' MACRO'
+            tokens = [
+                (Comment.Preproc, directive),
+                (Text, u' '),
+                (Comment.Preproc, u'MACRO'),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testOtherPreprocessorDirectives(self):
+        fragment = dedent('''\
+            #else
+            #end
+            #endif
+            ###
+            @ \t12345
+            @@ \t67890
+        ''')
+        tokens = [
+            (Comment.Preproc, u'#else'),
+            (Text, u'\n'),
+            (Comment.Preproc, u'#end'),
+            (Text, u'\n'),
+            (Comment.Preproc, u'#endif'),
+            (Text, u'\n'),
+            (Comment.Preproc, u'###'),
+            (Text, u'\n'),
+            (Comment.Preproc, u'@ \t12345'),
+            (Text, u'\n'),
+            (Comment.Preproc, u'@@ \t67890'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testFunctionLikeMacros(self):
+        fragment = "$MACRO.(((x#y\\)))' \"(#'x)\\)x\\))\"# {{x\\))x)\\)(#'}});"
+        tokens = [
+            (Comment.Preproc, u'$MACRO.'),
+            (Punctuation, u'('),
+            (Comment.Preproc, u'('),
+            (Comment.Preproc, u'('),
+            (Comment.Preproc, u'x#y\\)'),
+            (Comment.Preproc, u')'),
+            (Comment.Preproc, u')'),
+            (Punctuation, u"'"),
+            (Comment.Preproc, u' '),
+            (String, u'"'),
+            (Error, u'('),
+            (Error, u'#'),
+            (Error, u"'"),
+            (String, u'x'),
+            (Error, u')'),
+            (Comment.Preproc, u'\\)'),
+            (String, u'x'),
+            (Comment.Preproc, u'\\)'),
+            (Error, u')'),
+            (String, u'"'),
+            (Punctuation, u'#'),
+            (Comment.Preproc, u' '),
+            (String, u'{{'),
+            (String, u'x'),
+            (Comment.Preproc, u'\\)'),
+            (Error, u')'),
+            (String, u'x'),
+            (Error, u')'),
+            (Comment.Preproc, u'\\)'),
+            (Error, u'('),
+            (Error, u'#'),
+            (Error, u"'"),
+            (String, u'}}'),
+            (Punctuation, u')'),
+            (Comment.Single, u';'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
diff --git a/tests/test_markdown_lexer.py b/tests/test_markdown_lexer.py
new file mode 100644
index 00000000..16d1f28d
--- /dev/null
+++ b/tests/test_markdown_lexer.py
@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+"""
+    Pygments regex lexer tests
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+import unittest
+
+from pygments.lexers.markup import MarkdownLexer
+
+
+class SameTextTests(unittest.TestCase):
+
+    lexer = MarkdownLexer()
+
+    def assert_same_text(self, text):
+        """Show that lexed markdown does not remove any content. """
+        tokens = list(self.lexer.get_tokens_unprocessed(text))
+        output = ''.join(t[2] for t in tokens)
+        self.assertEqual(text, output)
+
+    def test_code_fence(self):
+        self.assert_same_text(r'```\nfoo\n```\n')
+
+    def test_code_fence_gsm(self):
+        self.assert_same_text(r'```markdown\nfoo\n```\n')
+
+    def test_code_fence_gsm_with_no_lexer(self):
+        self.assert_same_text(r'```invalid-lexer\nfoo\n```\n')
diff --git a/tests/test_r.py b/tests/test_r.py
new file mode 100644
index 00000000..d0e9090b
--- /dev/null
+++ b/tests/test_r.py
@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+"""
+    R Tests
+    ~~~~~~~~~
+
+    :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import unittest
+
+from pygments.lexers import SLexer
+from pygments.token import Token, Name, Punctuation
+
+
+class RTest(unittest.TestCase):
+    def setUp(self):
+        self.lexer = SLexer()
+
+    def testCall(self):
+        fragment = u'f(1, a)\n'
+        tokens = [
+            (Name.Function, u'f'),
+            (Punctuation, u'('),
+            (Token.Literal.Number, u'1'),
+            (Punctuation, u','),
+            (Token.Text, u' '),
+            (Token.Name, u'a'),
+            (Punctuation, u')'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testName1(self):
+        fragment = u'._a_2.c'
+        tokens = [
+            (Name, u'._a_2.c'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testName2(self):
+        # Invalid names are valid if backticks are used
+        fragment = u'`.1 blah`'
+        tokens = [
+            (Name, u'`.1 blah`'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testName3(self):
+        # Internal backticks can be escaped
+        fragment = u'`.1 \` blah`'
+        tokens = [
+            (Name, u'`.1 \` blah`'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testCustomOperator(self):
+        fragment = u'7 % and % 8'
+        tokens = [
+            (Token.Literal.Number, u'7'),
+            (Token.Text, u' '),
+            (Token.Operator, u'% and %'),
+            (Token.Text, u' '),
+            (Token.Literal.Number, u'8'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))