Merged in roskakori/pygments-vbscript (pull request #673)

Added lexer for VBScript
author: Anteru <bitbucket@ca.sh13.net> 2019-03-06 18:58:58 +0000
committer: Anteru <bitbucket@ca.sh13.net> 2019-03-06 18:58:58 +0000
commit: 9b9886734a7e9a33f268716212ee8c2159fb4242 (patch)
tree: 670f7ac89b2fca92b9c5f10dd49d28cdb7f574e0 /tests
parent: 590580e08ee53b7934a7886bf43f42878d77ce5c (diff)
parent: 30a5c5fd20f8062f63db7315c644c10b47f4b102 (diff)
download: pygments-9b9886734a7e9a33f268716212ee8c2159fb4242.tar.gz
61 files changed, 1602 insertions, 571 deletions
diff --git a/tests/examplefiles/99_bottles_of_beer.chpl b/tests/examplefiles/99_bottles_of_beer.chpl
index cdc1e650..ff50b294 100644
--- a/tests/examplefiles/99_bottles_of_beer.chpl
+++ b/tests/examplefiles/99_bottles_of_beer.chpl
@@ -177,3 +177,30 @@ private module M3 {
   private var x: int;
 
 }
+prototype module X {
+
+  proc f() throws {
+    throw new Error();
+  }
+
+  proc g() {
+    try {
+      f();
+      try! f();
+    } catch e {
+      writeln("Caught ", e);
+    }
+  }
+
+  proc int.add() { }
+
+  g();
+
+  override proc test() throws {
+    var a = new borrowed IntPair();
+    var b = new owned IntPair();
+    var c = new shared IntPair();
+    throw new unmanaged Error();
+  }
+}
+
diff --git a/tests/examplefiles/Charmci.ci b/tests/examplefiles/Charmci.ci
new file mode 100644
index 00000000..2e5cd5c6
--- /dev/null
+++ b/tests/examplefiles/Charmci.ci
@@ -0,0 +1,20 @@
+module CkCallback {
+	readonly CProxy_ckcallback_group _ckcallbackgroup;
+	message CkCcsRequestMsg {
+		char data[];
+ 	};
+	message CkDataMsg {
+		char data[];
+	};
+	
+	mainchare ckcallback_main {
+		entry ckcallback_main(CkArgMsg *m);
+	};
+	group [migratable] ckcallback_group : IrrGroup {
+		entry ckcallback_group();
+		entry void registerCcsCallback(char name[strlen(name)+1],
+			CkCallback cb);
+		entry void call(CkCallback c,CkMarshalledMessage msg);
+		entry void call(CkCallback c, int length, char data[length]);
+	};
+};
diff --git a/tests/examplefiles/StdGeneric.icl b/tests/examplefiles/StdGeneric.icl
index 2e6c3931..891b510a 100644
--- a/tests/examplefiles/StdGeneric.icl
+++ b/tests/examplefiles/StdGeneric.icl
@@ -1,5 +1,13 @@
 implementation module StdGeneric
 
+/**
+ * NOTE: this is a collection of different tricky parts of Clean modules (even
+ * though the file is simply called StdGeneric.icl). The code is taken from:
+ *
+ * - StdGeneric (StdEnv)
+ * - Graphics.Scalable.Image (Platform)
+ */
+
 import StdInt, StdMisc, StdClass, StdFunc
 
 generic bimap a b :: Bimap .a .b
@@ -89,4 +97,38 @@ where
 			= [ ConsLeft : doit i (n/2) ]
 		| otherwise
 			= [ ConsRight : doit (i - (n/2)) (n - (n/2)) ]
-			  	 							 	
-\ No newline at end of file
+			  	 							 	
+:: NoAttr          m = NoAttr
+:: DashAttr        m = { dash        :: ![Int]    }
+:: FillAttr        m = { fill        :: !SVGColor }
+:: LineEndMarker   m = { endmarker   :: !Image m  }
+:: LineMidMarker   m = { midmarker   :: !Image m  }
+:: LineStartMarker m = { startmarker :: !Image m  }
+:: MaskAttr        m = { mask        :: !Image m  }
+:: OpacityAttr     m = { opacity     :: !Real     }
+:: StrokeAttr      m = { stroke      :: !SVGColor }
+:: StrokeWidthAttr m = { strokewidth :: !Span     }
+:: XRadiusAttr     m = { xradius     :: !Span     }
+:: YRadiusAttr     m = { yradius     :: !Span     }
+
+
+instance tuneImage NoAttr          where tuneImage image _    = image
+instance tuneImage DashAttr        where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgDashAttr attr.DashAttr.dash)) image
+instance tuneImage FillAttr        where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgFillAttr attr.FillAttr.fill)) image
+instance tuneImage LineEndMarker   where tuneImage image attr = Attr` (LineMarkerAttr` {LineMarkerAttr | markerImg = attr.LineEndMarker.endmarker, markerPos = LineMarkerEnd}) image
+instance tuneImage LineMidMarker   where tuneImage image attr = Attr` (LineMarkerAttr` {LineMarkerAttr | markerImg = attr.LineMidMarker.midmarker, markerPos = LineMarkerMid}) image
+instance tuneImage LineStartMarker where tuneImage image attr = Attr` (LineMarkerAttr` {LineMarkerAttr | markerImg = attr.LineStartMarker.startmarker, markerPos = LineMarkerStart}) image
+instance tuneImage MaskAttr        where tuneImage image attr = Attr` (MaskAttr` attr.MaskAttr.mask) image
+instance tuneImage OpacityAttr     where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgFillOpacityAttr attr.OpacityAttr.opacity)) image
+instance tuneImage StrokeAttr      where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgStrokeAttr      attr.StrokeAttr.stroke)) image
+instance tuneImage StrokeWidthAttr where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgStrokeWidthAttr attr.StrokeWidthAttr.strokewidth)) image
+instance tuneImage XRadiusAttr     where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgXRadiusAttr     attr.XRadiusAttr.xradius)) image
+instance tuneImage YRadiusAttr     where tuneImage image attr = Attr` (BasicImageAttr` (BasicImgYRadiusAttr     attr.YRadiusAttr.yradius)) image
+
+instance tuneImage DraggableAttr   where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerDraggableAttr   attr)) image
+instance tuneImage OnClickAttr     where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnClickAttr     attr)) image
+instance tuneImage OnMouseDownAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseDownAttr attr)) image
+instance tuneImage OnMouseMoveAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseMoveAttr attr)) image
+instance tuneImage OnMouseOutAttr  where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseOutAttr  attr)) image
+instance tuneImage OnMouseOverAttr where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseOverAttr attr)) image
+instance tuneImage OnMouseUpAttr   where tuneImage image attr = Attr` (HandlerAttr` (ImgEventhandlerOnMouseUpAttr   attr)) image
diff --git a/tests/examplefiles/demo.hbs b/tests/examplefiles/demo.hbs
index 1b9ed5a7..ae80cc1b 100644
--- a/tests/examplefiles/demo.hbs
+++ b/tests/examplefiles/demo.hbs
@@ -10,3 +10,25 @@
 {{else}}
   <button {{action expand}}>Show More...</button>
 {{/if}}
+
+{{> myPartial}}
+{{> myPartial var="value" }}
+{{> myPartial var=../value}}
+{{> (myPartial)}}
+{{> (myPartial) var="value"}}
+{{> (lookup . "myPartial")}}
+{{> ( lookup . "myPartial" ) var="value" }}
+{{> (lookup ../foo "myPartial") var="value" }}
+{{> @partial-block}}
+
+{{#>myPartial}}
+...
+{{/myPartial}}
+
+{{#*inline "myPartial"}}
+...
+{{/inline}}
+
+{{../name}}
+{{./name}}
+{{this/name}}
diff --git a/tests/examplefiles/docker.docker b/tests/examplefiles/docker.docker
index d65385b6..1ae3c3a1 100644
--- a/tests/examplefiles/docker.docker
+++ b/tests/examplefiles/docker.docker
@@ -1,5 +1,34 @@
-maintainer First O'Last
+FROM alpine:3.5
+MAINTAINER First O'Last
 
+# comment
 run echo \
   123 $bar
-# comment
+RUN apk --update add rsync dumb-init
+
+# Test env with both syntax
+ENV FOO = "BAR"
+ENV FOO \
+	"BAR"
+
+COPY foo "bar"
+COPY foo \
+	"bar"
+
+HEALTHCHECK \
+    --interval=5m --timeout=3s \
+    CMD curl -f http://localhost/ || exit 1
+
+# ONBUILD keyword, then with linebreak
+ONBUILD ADD . /app/src
+ONBUILD \
+	RUN echo 123 $bar
+
+# Potential JSON array parsing, mixed with linebreaks
+VOLUME \
+    /foo
+VOLUME \
+    ["/bar"]
+VOLUME ["/bar"]
+VOLUME /foo
+CMD ["foo", "bar"]
diff --git a/tests/examplefiles/example.flo b/tests/examplefiles/example.flo
new file mode 100644
index 00000000..2d4ab5e7
--- /dev/null
+++ b/tests/examplefiles/example.flo
@@ -0,0 +1,40 @@
+#example mission box1.flo
+#from: https://github.com/ioflo/ioflo
+
+house box1
+
+   framer vehiclesim be active first vehicle_run
+      frame vehicle_run
+         do simulator motion uuv
+
+   framer mission be active first northleg
+      frame northleg
+         set elapsed with 20.0
+         set heading with 0.0
+         set depth with 5.0
+         set speed with 2.5
+         go next if elapsed >= goal
+
+      frame eastleg
+         set heading with 90.0
+         go next if elapsed >= goal
+
+      frame southleg
+         set heading with 180.0
+         go next if elapsed >= goal
+
+      frame westleg
+         set heading with 270.0
+         go next if elapsed >= goal
+
+      frame mission_stop
+         bid stop vehiclesim
+         bid stop autopilot
+         bid stop me
+
+   framer autopilot be active first autopilot_run
+      frame autopilot_run
+         do controller pid speed
+         do controller pid heading
+         do controller pid depth
+         do controller pid pitch
+\ No newline at end of file
diff --git a/tests/examplefiles/example.hlsl b/tests/examplefiles/example.hlsl
new file mode 100644
index 00000000..21d0a672
--- /dev/null
+++ b/tests/examplefiles/example.hlsl
@@ -0,0 +1,168 @@
+// A few random snippets of HLSL shader code I gathered...
+
+[numthreads(256, 1, 1)]
+void cs_main(uint3 threadId : SV_DispatchThreadID)
+{
+	// Seed the PRNG using the thread ID
+	rng_state = threadId.x;
+
+	// Generate a few numbers...
+	uint r0 = rand_xorshift();
+	uint r1 = rand_xorshift();
+	// Do some stuff with them...
+
+	// Generate a random float in [0, 1)...
+	float f0 = float(rand_xorshift()) * (1.0 / 4294967296.0);
+
+	// ...etc.
+}
+
+// Constant buffer of parameters
+cbuffer IntegratorParams : register(b0)
+{
+	float2 specPow;		// Spec powers in XY directions (equal for isotropic BRDFs)
+	float3 L;			// Unit vector toward light 
+	int2 cThread;		// Total threads launched in XY dimensions
+	int2 xyOutput;		// Where in the output buffer to store the result
+}
+
+static const float pi = 3.141592654;
+
+float AshikhminShirleyNDF(float3 H)
+{
+	float normFactor = sqrt((specPow.x + 2.0f) * (specPow.y + 2.0)) * (0.5f / pi);
+	float NdotH = H.z;
+	float2 Hxy = normalize(H.xy);
+	return normFactor * pow(NdotH, dot(specPow, Hxy * Hxy));
+}
+
+float BeckmannNDF(float3 H)
+{
+	float glossFactor = specPow.x * 0.5f + 1.0f;	// This is 1/m^2 in the usual Beckmann formula
+	float normFactor = glossFactor * (1.0f / pi);
+	float NdotHSq = H.z * H.z;
+	return normFactor / (NdotHSq * NdotHSq) * exp(glossFactor * (1.0f - 1.0f / NdotHSq));
+}
+
+// Output buffer for compute shader (actually float, but must be declared as uint
+// for atomic operations to work)
+globallycoherent RWTexture2D<uint> o_data : register(u0);
+
+// Sum up the outputs of all threads and store to the output location
+static const uint threadGroupSize2D = 16;
+static const uint threadGroupSize1D = threadGroupSize2D * threadGroupSize2D;
+groupshared float g_partialSums[threadGroupSize1D];
+void SumAcrossThreadsAndStore(float value, uint iThreadInGroup)
+{
+	// First reduce within the threadgroup: partial sums of 2, 4, 8... elements
+	// are calculated by 1/2, 1/4, 1/8... of the threads, always keeping the
+	// active threads at the front of the group to minimize divergence.
+
+	// NOTE: there are faster ways of doing this...but this is simple to code
+	// and good enough.
+
+	g_partialSums[iThreadInGroup] = value;
+	GroupMemoryBarrierWithGroupSync();
+
+	[unroll] for (uint i = threadGroupSize1D / 2; i > 0; i /= 2)
+	{
+		if (iThreadInGroup < i)
+		{
+			g_partialSums[iThreadInGroup] += g_partialSums[iThreadInGroup + i];
+		}
+		GroupMemoryBarrierWithGroupSync();
+	}
+
+	// Then reduce across threadgroups: one thread from each group adds the group
+	// total to the final output location, using a software transactional memory
+	// style since D3D11 doesn't support atomic add on floats.
+	// (Assumes the output value has been cleared to zero beforehand.)
+
+	if (iThreadInGroup == 0)
+	{
+		float threadGroupSum = g_partialSums[0];
+		uint outputValueRead = o_data[xyOutput];
+		while (true)
+		{
+			uint newOutputValue = asuint(asfloat(outputValueRead) + threadGroupSum);
+			uint previousOutputValue;
+			InterlockedCompareExchange(
+				o_data[xyOutput], outputValueRead, newOutputValue, previousOutputValue);
+			if (previousOutputValue == outputValueRead)
+				break;
+			outputValueRead = previousOutputValue;
+		}
+	}
+}
+
+void main(
+	in Vertex i_vtx,
+	out Vertex o_vtx,
+	out float3 o_vecCamera : CAMERA,
+	out float4 o_uvzwShadow : UVZW_SHADOW,
+	out float4 o_posClip : SV_Position)
+{
+	o_vtx = i_vtx;
+	o_vecCamera = g_posCamera - i_vtx.m_pos;
+	o_uvzwShadow = mul(float4(i_vtx.m_pos, 1.0), g_matWorldToUvzwShadow);
+	o_posClip = mul(float4(i_vtx.m_pos, 1.0), g_matWorldToClip);
+}
+
+#pragma pack_matrix(row_major)
+
+struct Vertex
+{
+	float3		m_pos		: POSITION;
+	float3		m_normal	: NORMAL;
+	float2		m_uv		: UV;
+};
+
+cbuffer CBFrame : CB_FRAME					// matches struct CBFrame in test.cpp
+{
+	float4x4	g_matWorldToClip;
+	float4x4	g_matWorldToUvzwShadow;
+	float3x3	g_matWorldToUvzShadowNormal;
+	float3		g_posCamera;
+
+	float3		g_vecDirectionalLight;
+	float3		g_rgbDirectionalLight;
+
+	float2		g_dimsShadowMap;
+	float		g_normalOffsetShadow;
+	float		g_shadowSharpening;
+
+	float		g_exposure;					// Exposure multiplier
+}
+
+Texture2D<float3> g_texDiffuse : register(t0);
+SamplerState g_ss : register(s0);
+
+void main(
+	in Vertex i_vtx,
+	in float3 i_vecCamera : CAMERA,
+	in float4 i_uvzwShadow : UVZW_SHADOW,
+	out float3 o_rgb : SV_Target)
+{
+	float3 normal = normalize(i_vtx.m_normal);
+
+	// Sample shadow map
+	float shadow = EvaluateShadow(i_uvzwShadow, normal);
+
+	// Evaluate diffuse lighting
+	float3 diffuseColor = g_texDiffuse.Sample(g_ss, i_vtx.m_uv);
+	float3 diffuseLight = g_rgbDirectionalLight * (shadow * saturate(dot(normal, g_vecDirectionalLight)));
+	diffuseLight += SimpleAmbient(normal);
+
+	o_rgb = diffuseColor * diffuseLight;
+}
+
+[domain("quad")]
+void ds(
+	in float edgeFactors[4] : SV_TessFactor,
+	in float insideFactors[2] : SV_InsideTessFactor,
+	in OutputPatch<VData, 4> inp,
+	in float2 uv : SV_DomainLocation,
+	out float4 o_pos : SV_Position)
+{
+	o_pos = lerp(lerp(inp[0].pos, inp[1].pos, uv.x), lerp(inp[2].pos, inp[3].pos, uv.x), uv.y);
+}
diff --git a/tests/examplefiles/example.hs b/tests/examplefiles/example.hs
index f5e2b555..764cab77 100644
--- a/tests/examplefiles/example.hs
+++ b/tests/examplefiles/example.hs
@@ -29,3 +29,13 @@ data ĈrazyThings =
 -- some char literals:
 
 charl = ['"', 'a', '\ESC', '\'', ' ']
+
+-- closed type families
+type family Fam (a :: Type) = r :: Type where
+  Fam Int = True
+  Fam a = False
+
+-- type literals
+type IntChar = '[Int, Char]
+type Falsy = 'False
+type Falsy = '(10, 20, 30)
diff --git a/tests/examplefiles/example.sgf b/tests/examplefiles/example.sgf
new file mode 100644
index 00000000..024a461e
--- /dev/null
+++ b/tests/examplefiles/example.sgf
@@ -0,0 +1,35 @@
+(;FF[4]GM[1]SZ[19]FG[257:Figure 1]PM[1]
+PB[Takemiya Masaki]BR[9 dan]PW[Cho Chikun]
+WR[9 dan]RE[W+Resign]KM[5.5]TM[28800]DT[1996-10-18,19]
+EV[21st Meijin]RO[2 (final)]SO[Go World #78]US[Arno Hollosi]
+;B[pd];W[dp];B[pp];W[dd];B[pj];W[nc];B[oe];W[qc];B[pc];W[qd]
+(;B[qf];W[rf];B[rg];W[re];B[qg];W[pb];B[ob];W[qb]
+(;B[mp];W[fq];B[ci];W[cg];B[dl];W[cn];B[qo];W[ec];B[jp];W[jd]
+;B[ei];W[eg];B[kk]LB[qq:a][dj:b][ck:c][qp:d]N[Figure 1]
+
+;W[me]FG[257:Figure 2];B[kf];W[ke];B[lf];W[jf];B[jg]
+(;W[mf];B[if];W[je];B[ig];W[mg];B[mj];W[mq];B[lq];W[nq]
+(;B[lr];W[qq];B[pq];W[pr];B[rq];W[rr];B[rp];W[oq];B[mr];W[oo];B[mn]
+(;W[nr];B[qp]LB[kd:a][kh:b]N[Figure 2]
+
+;W[pk]FG[257:Figure 3];B[pm];W[oj];B[ok];W[qr];B[os];W[ol];B[nk];W[qj]
+;B[pi];W[pl];B[qm];W[ns];B[sr];W[om];B[op];W[qi];B[oi]
+(;W[rl];B[qh];W[rm];B[rn];W[ri];B[ql];W[qk];B[sm];W[sk];B[sh];W[og]
+;B[oh];W[np];B[no];W[mm];B[nn];W[lp];B[kp];W[lo];B[ln];W[ko];B[mo]
+;W[jo];B[km]N[Figure 3])
+
+(;W[ql]VW[ja:ss]FG[257:Dia. 6]MN[1];B[rm];W[ph];B[oh];W[pg];B[og];W[pf]
+;B[qh];W[qe];B[sh];W[of];B[sj]TR[oe][pd][pc][ob]LB[pe:a][sg:b][si:c]
+N[Diagram 6]))
+
+(;W[no]VW[jj:ss]FG[257:Dia. 5]MN[1];B[pn]N[Diagram 5]))
+
+(;B[pr]FG[257:Dia. 4]MN[1];W[kq];B[lp];W[lr];B[jq];W[jr];B[kp];W[kr];B[ir]
+;W[hr]LB[is:a][js:b][or:c]N[Diagram 4]))
+
+(;W[if]FG[257:Dia. 3]MN[1];B[mf];W[ig];B[jh]LB[ki:a]N[Diagram 3]))
+
+(;W[oc]VW[aa:sk]FG[257:Dia. 2]MN[1];B[md];W[mc];B[ld]N[Diagram 2]))
+
+(;B[qe]VW[aa:sj]FG[257:Dia. 1]MN[1];W[re];B[qf];W[rf];B[qg];W[pb];B[ob]
+;W[qb]LB[rg:a]N[Diagram 1]))
diff --git a/tests/examplefiles/example.sl b/tests/examplefiles/example.sl
new file mode 100644
index 00000000..5fb430de
--- /dev/null
+++ b/tests/examplefiles/example.sl
@@ -0,0 +1,6 @@
+#!/bin/bash
+#SBATCH --partition=part
+#SBATCH --job-name=job
+#SBATCH --mem=1G
+#SBATCH --cpus-per-task=8
+srun /usr/bin/sleep
+\ No newline at end of file
diff --git a/tests/examplefiles/example.stan b/tests/examplefiles/example.stan
index 69c9ac70..03b7b1b5 100644
--- a/tests/examplefiles/example.stan
+++ b/tests/examplefiles/example.stan
@@ -16,7 +16,7 @@ functions {
 data {
   // valid name
   int abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_abc;
-  // all types should be highlighed
+  // all types should be highlighted
   int a3;
   real foo[2];
   vector[3] bar;
@@ -48,7 +48,7 @@ transformed data {
   thud <- -12309865;
   // ./ and .* should be recognized as operators
   grault2 <- grault .* garply ./ garply;
-  // ' and \ should be regognized as operators
+  // ' and \ should be recognized as operators
   qux2 <- qux' \ bar;
   
 }
diff --git a/tests/examplefiles/example.tf b/tests/examplefiles/example.tf
index d3f02779..4cbef52c 100644
--- a/tests/examplefiles/example.tf
+++ b/tests/examplefiles/example.tf
@@ -23,6 +23,14 @@ variable "aws_amis" {
 }
 
 
+resource "aws_internet_gateway" "base_igw" {
+  vpc_id = "${aws_vpc.something.id}"
+  tags {
+    Name = "igw-${var.something}-${var.something}"
+  }
+}
+
+
 
 
 
@@ -39,6 +47,16 @@ provider "aws" {
 */
 
 
+resource "aws_route53_record" "test" {
+  zone_id  = "zone"
+  name      = "name"
+  type        = "A"
+  alias {
+    name     = "alias name"
+  }
+}
+
+
 # Single line comment
 resource "aws_instance" "example" {
     ami = "ami-408c7f28"
@@ -160,3 +178,31 @@ resource "aws_instance" "web" {
   }
 }
 
+
+
+resource "aws_autoscaling_group" "bar" {
+  name                 = "terraform-asg-example"
+  launch_configuration = "${aws_launch_configuration.as_conf.name}"
+  min_size             = 1
+  max_size             = 2
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+
+resource "aws_db_instance" "timeout_example" {
+  allocated_storage = 10
+  engine            = "mysql"
+  engine_version    = "5.6.17"
+  instance_class    = "db.t1.micro"
+  name              = "mydb"
+
+  timeouts {
+    create = "60m"
+    delete = "2h"
+  }
+}
+
+
diff --git a/tests/examplefiles/fennelview.fnl b/tests/examplefiles/fennelview.fnl
new file mode 100644
index 00000000..fd0fc648
--- /dev/null
+++ b/tests/examplefiles/fennelview.fnl
@@ -0,0 +1,156 @@
+;; A pretty-printer that outputs tables in Fennel syntax.
+;; Loosely based on inspect.lua: http://github.com/kikito/inspect.lua
+
+(local quote (fn [str] (.. '"' (: str :gsub '"' '\\"') '"')))
+
+(local short-control-char-escapes
+       {"\a" "\\a" "\b" "\\b" "\f" "\\f" "\n" "\\n"
+        "\r" "\\r" "\t" "\\t" "\v" "\\v"})
+
+(local long-control-char-esapes
+       (let [long {}]
+         (for [i 0 31]
+           (let [ch (string.char i)]
+             (when (not (. short-control-char-escapes ch))
+               (tset short-control-char-escapes ch (.. "\\" i))
+               (tset long ch (: "\\%03d" :format i)))))
+         long))
+
+(fn escape [str]
+  (let [str (: str :gsub "\\" "\\\\")
+        str (: str :gsub "(%c)%f[0-9]" long-control-char-esapes)]
+    (: str :gsub "%c" short-control-char-escapes)))
+
+(fn sequence-key? [k len]
+  (and (= (type k) "number")
+       (<= 1 k)
+       (<= k len)
+       (= (math.floor k) k)))
+
+(local type-order {:number 1 :boolean 2 :string 3 :table 4
+                   :function 5 :userdata 6 :thread 7})
+
+(fn sort-keys [a b]
+  (let [ta (type a) tb (type b)]
+    (if (and (= ta tb) (~= ta "boolean")
+             (or (= ta "string") (= ta "number")))
+        (< a b)
+        (let [dta (. type-order a)
+              dtb (. type-order b)]
+          (if (and dta dtb)
+              (< dta dtb)
+              dta true
+              dtb false
+              :else (< ta tb))))))
+
+(fn get-sequence-length [t]
+  (var len 1)
+  (each [i (ipairs t)] (set len i))
+  len)
+
+(fn get-nonsequential-keys [t]
+  (let [keys {}
+        sequence-length (get-sequence-length t)]
+    (each [k (pairs t)]
+      (when (not (sequence-key? k sequence-length))
+        (table.insert keys k)))
+    (table.sort keys sort-keys)
+    (values keys sequence-length)))
+
+(fn count-table-appearances [t appearances]
+  (if (= (type t) "table")
+      (when (not (. appearances t))
+        (tset appearances t 1)
+        (each [k v (pairs t)]
+          (count-table-appearances k appearances)
+          (count-table-appearances v appearances)))
+      (when (and t (= t t)) ; no nans please
+        (tset appearances t (+ (or (. appearances t) 0) 1))))
+  appearances)
+
+
+
+(var put-value nil) ; mutual recursion going on; defined below
+
+(fn puts [self ...]
+  (each [_ v (ipairs [...])]
+    (table.insert self.buffer v)))
+
+(fn tabify [self] (puts self "\n" (: self.indent :rep self.level)))
+
+(fn already-visited? [self v] (~= (. self.ids v) nil))
+
+(fn get-id [self v]
+  (var id (. self.ids v))
+  (when (not id)
+    (let [tv (type v)]
+      (set id (+ (or (. self.max-ids tv) 0) 1))
+      (tset self.max-ids tv id)
+      (tset self.ids v id)))
+  (tostring id))
+
+(fn put-sequential-table [self t length]
+  (puts self "[")
+  (set self.level (+ self.level 1))
+  (for [i 1 length]
+    (puts self " ")
+    (put-value self (. t i)))
+  (set self.level (- self.level 1))
+  (puts self " ]"))
+
+(fn put-key [self k]
+  (if (and (= (type k) "string")
+           (: k :find "^[-%w?\\^_`!#$%&*+./@~:|<=>]+$"))
+      (puts self ":" k)
+      (put-value self k)))
+
+(fn put-kv-table [self t]
+  (puts self "{")
+  (set self.level (+ self.level 1))
+  (each [k v (pairs t)]
+    (tabify self)
+    (put-key self k)
+    (puts self " ")
+    (put-value self v))
+  (set self.level (- self.level 1))
+  (tabify self)
+  (puts self "}"))
+
+(fn put-table [self t]
+  (if (already-visited? self t)
+      (puts self "#<table " (get-id self t) ">")
+      (>= self.level self.depth)
+      (puts self "{...}")
+      :else
+      (let [(non-seq-keys length) (get-nonsequential-keys t)
+            id (get-id self t)]
+        (if (> (. self.appearances t) 1)
+            (puts self "#<" id ">")
+            (and (= (# non-seq-keys) 0) (= (# t) 0))
+            (puts self "{}")
+            (= (# non-seq-keys) 0)
+            (put-sequential-table self t length)
+            :else
+            (put-kv-table self t)))))
+
+(set put-value (fn [self v]
+                 (let [tv (type v)]
+                   (if (= tv "string")
+                       (puts self (quote (escape v)))
+                       (or (= tv "number") (= tv "boolean") (= tv "nil"))
+                       (puts self (tostring v))
+                       (= tv "table")
+                       (put-table self v)
+                       :else
+                       (puts self "#<" (tostring v) ">")))))
+
+
+
+(fn fennelview [root options]
+  (let [options (or options {})
+        inspector {:appearances (count-table-appearances root {})
+                   :depth (or options.depth 128)
+                   :level 0 :buffer {} :ids {} :max-ids {}
+                   :indent (or options.indent "  ")}]
+    (put-value inspector root)
+    (table.concat inspector.buffer)))
diff --git a/tests/examplefiles/test.csd b/tests/examplefiles/test.csd
index 9122309b..6512d99e 100644
--- a/tests/examplefiles/test.csd
+++ b/tests/examplefiles/test.csd
@@ -1,260 +1,18 @@
+/*
+ * comment
+ */
+; comment
+// comment
+/
 <CsoundSynthesizer>
 <CsInstruments>
-// This is a Csound orchestra file for testing a Pygments <http://pygments.org>
-// lexer. Csound single-line comments can be preceded by a pair of forward
-// slashes...
-; ...or a semicolon.
-
-/* Block comments begin with /* and end with */
-
-// Orchestras begin with a header of audio parameters.
-nchnls = 1
-nchnls_i = 1
-sr = 44100
 0dbfs = 1
-ksmps = 10
-
-// The control rate kr = sr / ksmps can be omitted when the number of audio
-// samples in a control period (ksmps) is set, but kr may appear in older
-// orchestras.
-kr = 4410
-
-// Orchestras contain instruments. These begin with the keyword instr followed
-// by a comma-separated list of numbers or names of the instrument. Instruments
-// end at the endin keyword and cannot be nested.
-instr 1, N_a_M_e_, +Name
-  // Instruments contain statements. Here is a typical statement:
-  aSignal oscil 0dbfs, 440, 1
-  // Statements are terminated with a newline (possibly preceded by a comment).
-  // To write a statement on several lines, precede the newline with a
-  // backslash.
-  prints \
-    "hello, world\n";comment
-
-  // Csound 6 introduced function syntax for opcodes with one or zero outputs.
-  // The oscil statement above is the same as
-  aSignal = oscil(0dbfs, 440, 1)
-
-  // Instruments can contain control structures.
-  kNote = p3
-  if (kNote == 0) then
-    kFrequency = 220
-  elseif kNote == 1 then // Parentheses around binary expressions are optional.
-    kFrequency = 440
-  endif
-
-  // Csound 6 introduced looping structures.
-  iIndex = 0
-  while iIndex < 5 do
-    print iIndex
-    iIndex += 1
-  od
-  iIndex = 0
-  until iIndex >= 5 do
-    print iIndex
-    iIndex += 1
-  enduntil
-  // Both kinds of loops can be terminated by either od or enduntil.
-
-  // Single-line strings are enclosed in double-quotes.
-  prints "string\\\r\n\t\""
-  // Multi-line strings are enclosed in pairs of curly braces.
-  prints {{
-    hello,
-
-    world
-  }}
-
-  // Instruments often end with a statement containing an output opcode.
-  outc aSignal
-endin
-
-// Orchestras can also contain user-defined opcodes (UDOs). Here is an
-// oscillator with one audio-rate output and two control-rate inputs:
-opcode anOscillator, a, kk
-  kAmplitude, kFrequency xin
-  aSignal vco2 kAmplitude, kFrequency
-  xout aSignal
-endop
-instr TestOscillator
-  outc(anOscillator(0dbfs, 110))
-endin
-
-// Python can be executed in Csound
-// <http://www.csounds.com/manual/html/pyrun.html>. So can Lua
-// <http://www.csounds.com/manual/html/lua.html>.
-pyruni {{
-import random
-
-pool = [(1 + i / 10.0) ** 1.2 for i in range(100)]
-
-def get_number_from_pool(n, p):
-  if random.random() < p:
-    i = int(random.random() * len(pool))
-    pool[i] = n
-  return random.choice(pool)
-}}
-
-// The Csound preprocessor supports conditional compilation and including files.
-#ifdef DEBUG
-#undef DEBUG
-#include "filename.orc"
-#endif
-
-// The preprocessor also supports object- and function-like macros. This is an
-// object-like macro that defines a number:
-#define A_HZ #440#
-
-// This is a function-like macro:
-#define OSCIL_MACRO(VOLUME'FREQUENCY'TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE#
-
-// Bodies of macros are enclosed in # and can contain newlines. The arguments of
-// function-like macros are separated by single-quotes. Uses of macros are
-// prefixed with a dollar sign.
-instr TestMacro
-  aSignal $OSCIL_MACRO(1'$A_HZ'1)
-  // Not unlike PHP, macros expand in double-quoted strings.
-  prints "The frequency of the oscillator is $A_HZ Hz.\n"
-  out aSignal
-endin
-
-// Here are other things to note about Csound.
-
-// There are two bitwise NOT operators, ~ and ¬ (U+00AC). The latter is common
-// on keyboards in the United Kingdom
-// <https://en.wikipedia.org/wiki/British_and_American_keyboards>.
-instr TestBitwiseNOT
-  print ~42
-  print ¬42
-endin
-
-// Csound uses # for bitwise XOR, which the Csound manual calls bitwise
-// non-equivalence <http://www.csounds.com/manual/html/opnonequiv.html>.
-instr TestBitwiseXOR
-  print 0 # 0
-  print 0 # 1
-  print 1 # 0
-  print 1 # 1
-endin
-
-// Loops and if-then statements are relatively recent additions to Csound. There
-// are many flow-control opcodes that involve goto and labels.
-instr TestGoto
-  // This...
-  if p3 > 0 goto if_label
-  goto else_label
-if_label:
-  prints "if branch\n"
-  goto endif_label
-else_label:
-  prints "else branch\n"
-endif_label:
-
-  // ...is the same as this.
-  if p3 > 0 then
-    prints "if branch\n"
-  else
-    prints "else branch\n"
-  endif
-
-  // This...
-  iIndex = 0
-loop_label:
-  print iIndex
-  iIndex += 1
-  if iIndex < 10 goto loop_label
-
-  // ...is the same as this...
-  iIndex = 0
-loop_lt_label:
-  print iIndex
-  loop_lt iIndex, 1, 10, loop_lt_label
-
-  // ...and this.
-  iIndex = 0
-  while iIndex < 10 do
-    print iIndex
-    iIndex += 1
-  od
-endin
-
-// The prints and printks opcodes
-// <https://github.com/csound/csound/blob/develop/OOps/ugrw1.c#L831>, arguably
-// the primary methods of logging output, treat certain sequences of characters
-// different from printf in C.
-instr TestPrints
-  // ^ prints an ESCAPE character (U+001B), not a CIRCUMFLEX ACCENT character
-  // (U+005E). ^^ prints a CIRCUMFLEX ACCENT.
-  prints "^^\n"
-  // ~ prints an ESCAPE character (U+001B) followed by a [, not a TILDE
-  // character (U+007E). ~~ prints a TILDE.
-  prints "~~\n"
-  // \A, \B, \N, \R, and \T correspond to the escaped lowercase characters (that
-  // is, BELL (U+0007), BACKSPACE (U+0008), new line (U+000A), CARRIAGE RETURN
-  // (U+000D), and tab (U+0009)).
-  prints "\T\R\N"
-  // %n, %r, and %t are the same as \n, \r, and \t, as are %N, %R, and %T.
-  prints "%t%r%n"
-  // %! prints a semicolon. This is a hold-over from old versions of Csound that
-  // allowed comments to begin in strings.
-  prints "; %!\n"
-endin
-
-// The arguments of function-like macros can be separated by # instead of '.
-// These two lines define the same macro.
-#define OSCIL_MACRO(VOLUME'FREQUENCY'TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE#
-#define OSCIL_MACRO(VOLUME#FREQUENCY#TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE#
-
-// Uses of macros can optionally be suffixed with a period.
-instr TestMacroPeriodSuffix
-  aSignal $OSCIL_MACRO.(1'$A_HZ'1)
-  prints "The frequency of the oscillator is $A_HZ.Hz.\n"
-  out aSignal
-endin
-
-// Csound has @ and @@ operator-like macros that, when followed by a literal
-// non-negative integer, expand to the next power of 2 and the next power of 2
-// plus 1:
-//    @x = 2^(ceil(log2(x + 1))), x >= 0
-//   @@0 = 2
-//   @@x = 2^(ceil(log2(x))) + 1, x > 0
-// These macros are in
-// <https://github.com/csound/csound/blob/develop/Engine/csound_orc.l#L542> (and
-// <https://github.com/csound/csound/blob/develop/Engine/csound_sco.lex#L202>)
-// and are described at <http://www.csounds.com/manual/html/ScoreEval.html>.
-instr TestAt
-  prints "%d  %2d  %2d\n", 0, @0, @@0
-  prints "%d  %2d  %2d\n", 1, @1, @@1
-  prints "%d  %2d  %2d\n", 2, @2, @@2
-  prints "%d  %2d  %2d\n", 3, @3, @@3
-  prints "%d  %2d  %2d\n", 4, @4, @@4
-  prints "%d  %2d  %2d\n", 5, @5, @@5
-  prints "%d  %2d  %2d\n", 6, @6, @@6
-  prints "%d  %2d  %2d\n", 7, @7, @@7
-  prints "%d  %2d  %2d\n", 8, @8, @@8
-  prints "%d  %2d  %2d\n", 9, @9, @@9
-endin
-
-// Including newlines in macros can lead to confusing code, but it tests the
-// lexer.
-instr MacroAbuse
-  if 1 == 1 then
-    prints "on\n"
-#define FOO#
-BAR
-#endif // This ends the if statement. It is not a preprocessor directive.
-endin
+prints "hello, world\n"
 </CsInstruments>
 <CsScore>
-f 1 0 16384 10 1
-i "N_a_M_e_" 0 2
-i "TestOscillator" 2 2
-i "TestBitwiseNOT" 0 1
-i "TestBitwiseXOR" 0 1
-i "TestGoto" 0 1
-i "TestMacroPeriodSuffix" 4 1
-i "TestAt" 0 1
-i "MacroAbuse" 0 1
-e
+i 1 0 0
 </CsScore>
+<html>
+<!DOCTYPE html>
+</html>
 </CsoundSynthesizer>
diff --git a/tests/examplefiles/test.orc b/tests/examplefiles/test.orc
index 36725342..d113303e 100644
--- a/tests/examplefiles/test.orc
+++ b/tests/examplefiles/test.orc
@@ -1,257 +1,81 @@
-// This is a Csound orchestra file for testing a Pygments <http://pygments.org>
-// lexer. Csound single-line comments can be preceded by a pair of forward
-// slashes...
-; ...or a semicolon.
-
-/* Block comments begin with /* and end with */
-
-// Orchestras begin with a header of audio parameters.
-nchnls = 1
-nchnls_i = 1
-sr = 44100
-0dbfs = 1
-ksmps = 10
-
-// The control rate kr = sr / ksmps can be omitted when the number of audio
-// samples in a control period (ksmps) is set, but kr may appear in older
-// orchestras.
-kr = 4410
-
-// Orchestras contain instruments. These begin with the keyword instr followed
-// by a comma-separated list of numbers or names of the instrument. Instruments
-// end at the endin keyword and cannot be nested.
-instr 1, N_a_M_e_, +Name
-  // Instruments contain statements. Here is a typical statement:
-  aSignal oscil 0dbfs, 440, 1
-  // Statements are terminated with a newline (possibly preceded by a comment).
-  // To write a statement on several lines, precede the newline with a
-  // backslash.
-  prints \
-    "hello, world\n";comment
-
-  // Csound 6 introduced function syntax for opcodes with one or zero outputs.
-  // The oscil statement above is the same as
-  aSignal = oscil(0dbfs, 440, 1)
-
-  // Instruments can contain control structures.
-  kNote = p3
-  if (kNote == 0) then
-    kFrequency = 220
-  elseif kNote == 1 then // Parentheses around binary expressions are optional.
-    kFrequency = 440
-  endif
-
-  // Csound 6 introduced looping structures.
-  iIndex = 0
-  while iIndex < 5 do
-    print iIndex
-    iIndex += 1
-  od
-  iIndex = 0
-  until iIndex >= 5 do
-    print iIndex
-    iIndex += 1
-  enduntil
-  // Both kinds of loops can be terminated by either od or enduntil.
-
-  // Single-line strings are enclosed in double-quotes.
-  prints "string\\\r\n\t\""
-  // Multi-line strings are enclosed in pairs of curly braces.
-  prints {{
-    hello,
-
-    world
-  }}
-
-  // Instruments often end with a statement containing an output opcode.
-  outc aSignal
+/*
+ * comment
+ */
+; comment
+// comment
+
+instr/**/1,/**/N_a_M_e_,/**/+Name/**///
+  iDuration = p3
+  outc:a(aSignal)
 endin
 
-// Orchestras can also contain user-defined opcodes (UDOs). Here is an
-// oscillator with one audio-rate output and two control-rate inputs:
-opcode anOscillator, a, kk
-  kAmplitude, kFrequency xin
-  aSignal vco2 kAmplitude, kFrequency
-  xout aSignal
+opcode/**/aUDO,/**/i[],/**/aik//
+  aUDO
 endop
-instr TestOscillator
-  outc(anOscillator(0dbfs, 110))
-endin
 
-// Python can be executed in Csound
-// <http://www.csounds.com/manual/html/pyrun.html>. So can Lua
-// <http://www.csounds.com/manual/html/lua.html>.
-pyruni {{
-import random
+123 0123456789
+0xabcdef0123456789 0XABCDEF
+1e2 3e+4 5e-6 7E8 9E+0 1E-2 3. 4.56 .789
 
-pool = [(1 + i / 10.0) ** 1.2 for i in range(100)]
+"characters$MACRO."
+"\\\a\b\n\r\t\012\345\67\""
 
-def get_number_from_pool(n, p):
-  if random.random() < p:
-    i = int(random.random() * len(pool))
-    pool[i] = n
-  return random.choice(pool)
+{{
+characters$MACRO.
 }}
+{{\\\a\b\n\r\t\"\012\345\67}}
 
-// The Csound preprocessor supports conditional compilation and including files.
-#ifdef DEBUG
-#undef DEBUG
-#include "filename.orc"
-#endif
-
-// The preprocessor also supports object- and function-like macros. This is an
-// object-like macro that defines a number:
-#define A_HZ #440#
-
-// This is a function-like macro:
-#define OSCIL_MACRO(VOLUME'FREQUENCY'TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE#
-
-// Bodies of macros are enclosed in # and can contain newlines. The arguments of
-// function-like macros are separated by single-quotes. Uses of macros are
-// prefixed with a dollar sign.
-instr TestMacro
-  aSignal $OSCIL_MACRO(1'$A_HZ'1)
-  // Not unlike PHP, macros expand in double-quoted strings.
-  prints "The frequency of the oscillator is $A_HZ Hz.\n"
-  out aSignal
-endin
-
-// Here are other things to note about Csound.
++ - ~ ¬ ! * / ^ % << >> < > <= >= == != & # | && || ? : += -= *= /=
 
-// There are two bitwise NOT operators, ~ and ¬ (U+00AC). The latter is common
-// on keyboards in the United Kingdom
-// <https://en.wikipedia.org/wiki/British_and_American_keyboards>.
-instr TestBitwiseNOT
-  print ~42
-  print ¬42
-endin
-
-// Csound uses # for bitwise XOR, which the Csound manual calls bitwise
-// non-equivalence <http://www.csounds.com/manual/html/opnonequiv.html>.
-instr TestBitwiseXOR
-  print 0 # 0
-  print 0 # 1
-  print 1 # 0
-  print 1 # 1
-endin
+0dbfs A4 kr ksmps nchnls nchnls_i sr
 
-// Loops and if-then statements are relatively recent additions to Csound. There
-// are many flow-control opcodes that involve goto and labels.
-instr TestGoto
-  // This...
-  if p3 > 0 goto if_label
-  goto else_label
-if_label:
-  prints "if branch\n"
-  goto endif_label
-else_label:
-  prints "else branch\n"
-endif_label:
+do else elseif endif enduntil fi if ithen kthen od then until while
+return rireturn
 
-  // ...is the same as this.
-  if p3 > 0 then
-    prints "if branch\n"
-  else
-    prints "else branch\n"
-  endif
+aLabel:
+ label2:
 
-  // This...
-  iIndex = 0
-loop_label:
-  print iIndex
-  iIndex += 1
-  if iIndex < 10 goto loop_label
+goto aLabel
+reinit aLabel
+cggoto 1==0, aLabel
+timout 0, 0, aLabel
+loop_ge 0, 0, 0, aLabel
 
-  // ...is the same as this...
-  iIndex = 0
-loop_lt_label:
-  print iIndex
-  loop_lt iIndex, 1, 10, loop_lt_label
-
-  // ...and this.
-  iIndex = 0
-  while iIndex < 10 do
-    print iIndex
-    iIndex += 1
-  od
-endin
+prints "%! %% %n%N %r%R %t%T \\a\\A \\b\\B \\n\\N \\r\\R \\t\\T"
+prints Soutput
 
-// The prints and printks opcodes
-// <https://github.com/csound/csound/blob/develop/OOps/ugrw1.c#L831>, arguably
-// the primary methods of logging output, treat certain sequences of characters
-// different from printf in C.
-instr TestPrints
-  // ^ prints an ESCAPE character (U+001B), not a CIRCUMFLEX ACCENT character
-  // (U+005E). ^^ prints a CIRCUMFLEX ACCENT.
-  prints "^^\n"
-  // ~ prints an ESCAPE character (U+001B) followed by a [, not a TILDE
-  // character (U+007E). ~~ prints a TILDE.
-  prints "~~\n"
-  // \A, \B, \N, \R, and \T correspond to the escaped lowercase characters (that
-  // is, BELL (U+0007), BACKSPACE (U+0008), new line (U+000A), CARRIAGE RETURN
-  // (U+000D), and tab (U+0009)).
-  prints "\T\R\N"
-  // %n, %r, and %t are the same as \n, \r, and \t, as are %N, %R, and %T.
-  prints "%t%r%n"
-  // %! prints a semicolon. This is a hold-over from old versions of Csound that
-  // allowed comments to begin in strings.
-  prints "; %!\n"
-endin
-
-// The arguments of function-like macros can be separated by # instead of '.
-// These two lines define the same macro.
-#define OSCIL_MACRO(VOLUME'FREQUENCY'TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE#
-#define OSCIL_MACRO(VOLUME#FREQUENCY#TABLE) #oscil $VOLUME, $FREQUENCY, $TABLE#
-
-// Uses of macros can optionally be suffixed with a period.
-instr TestMacroPeriodSuffix
-  aSignal $OSCIL_MACRO.(1'$A_HZ'1)
-  prints "The frequency of the oscillator is $A_HZ.Hz.\n"
-  out aSignal
-endin
-
-// Csound has @ and @@ operator-like macros that, when followed by a literal
-// non-negative integer, expand to the next power of 2 and the next power of 2
-// plus 1:
-//    @x = 2^(ceil(log2(x + 1))), x >= 0
-//   @@0 = 2
-//   @@x = 2^(ceil(log2(x))) + 1, x > 0
-// These macros are in
-// <https://github.com/csound/csound/blob/develop/Engine/csound_orc.l#L542> (and
-// <https://github.com/csound/csound/blob/develop/Engine/csound_sco.lex#L202>)
-// and are described at <http://www.csounds.com/manual/html/ScoreEval.html>.
-instr TestAt
-  prints "%d  %2d  %2d\n", 0, @0, @@0
-  prints "%d  %2d  %2d\n", 1, @1, @@1
-  prints "%d  %2d  %2d\n", 2, @2, @@2
-  prints "%d  %2d  %2d\n", 3, @3, @@3
-  prints "%d  %2d  %2d\n", 4, @4, @@4
-  prints "%d  %2d  %2d\n", 5, @5, @@5
-  prints "%d  %2d  %2d\n", 6, @6, @@6
-  prints "%d  %2d  %2d\n", 7, @7, @@7
-  prints "%d  %2d  %2d\n", 8, @8, @@8
-  prints "%d  %2d  %2d\n", 9, @9, @@9
-endin
+readscore {{
+i 1 0 0
+}}
+pyrun {{
+# Python
+}}
+lua_exec {{
+-- Lua
+}}
 
-// Including newlines in macros can lead to confusing code, but it tests the
-// lexer.
-instr MacroAbuse
-  if 1 == 1 then
-    prints "on\n"
-#define FOO#
-BAR
-#endif // This ends the if statement. It is not a preprocessor directive.
-endin
+#include/**/"file.udo"
+#include/**/|file.udo|
 
-scoreline_i {{
-f 1 0 16384 10 1
-i "N_a_M_e_" 0 2
-i "TestOscillator" 2 2
-i "TestBitwiseNOT" 0 1
-i "TestBitwiseXOR" 0 1
-i "TestGoto" 0 1
-i "TestMacroPeriodSuffix" 4 1
-i "TestAt" 0 1
-i "MacroAbuse" 0 1
-e
-}}
+#ifdef MACRO
+#else
+#ifndef MACRO
+#endif
+#undef MACRO
+
+#   define MACRO#macro_body#
+#define/**/
+MACRO/**/
+#\#macro
+body\##
+
+#define MACRO(ARG1#ARG2) #macro_body#
+#define/**/
+MACRO(ARG1'ARG2'ARG3)/**/
+#\#macro
+body\##
+
+$MACRO $MACRO.
+$MACRO(x)
+@0
+@@ 1
diff --git a/tests/examplefiles/test.sco b/tests/examplefiles/test.sco
index a0b39251..d997c1b3 100644
--- a/tests/examplefiles/test.sco
+++ b/tests/examplefiles/test.sco
@@ -1,10 +1,22 @@
-f 1 0 16384 10 1
-i "N_a_M_e_" 0 2
-i "TestOscillator" 2 2
-i "TestBitwiseNOT" 0 1
-i "TestBitwiseXOR" 0 1
-i "TestGoto" 0 1
-i "TestMacroPeriodSuffix" 4 1
-i "TestAt" 0 1
-i "MacroAbuse" 0 1
-e
+/*
+ * comment
+ */
+; comment
+// comment
+a b C d e f i q s t v x y
+z
+np0 nP1 Np2 NP3
+m/**/label;
+n label
+123 0123456789
+0xabcdef0123456789 0XABCDEF
+1e2 3e+4 5e-6 7E8 9E+0 1E-2 3. 4.56 .789
+"characters$MACRO."
+{ 1 I
+  { 2 J
+    { 3 K
+      $I $J $K
+    }
+  }
+}
+#include "score.sco"
diff --git a/tests/examplefiles/xorg.conf b/tests/examplefiles/xorg.conf
new file mode 100644
index 00000000..e1f7164b
--- /dev/null
+++ b/tests/examplefiles/xorg.conf
@@ -0,0 +1,48 @@
+Section "Files"
+	ModulePath "/usr/lib64/opengl/nvidia/extensions"
+	ModulePath "/usr/lib64/xorg/modules"
+EndSection
+
+Section "ServerLayout"
+	Identifier     "XFree86 Configured"
+	Screen "Screen"
+EndSection
+
+Section "ServerFlags"
+	Option "AutoAddDevices" "false"
+EndSection
+
+Section "Screen"
+        Identifier "Screen"
+	Device "Card0"
+	DefaultDepth    24
+	SubSection     "Display"
+		Depth       24
+	EndSubSection
+        Option "UseEDIDDpi" "False"
+        Option "DPI" "96 x 96"
+EndSection
+
+Section "Device"
+    Identifier  "Card0"
+    Driver      "nvidia"
+    VendorName  "NVIDIA Corporation" # inline comment
+    #Option      "RenderAccel" "true"
+    
+    #Option      "NvAgp" "3"
+    #Option      "AllowGLXWithComposite" "true"
+    #Option      "AddARGBGLXVisuals" "true"
+    #Option      "XAANoOffscreenPixmaps" "true"
+    #Option      "DRI" "true"
+    
+    #Option      "UseEvents" "false"
+    #Option      "TripleBuffer" "1"
+    #Option      "DamageEvents" "1"
+    ##Option      "BackingStore" "1"
+    #Option      "PixmapCacheSize" "70000"
+    #Option      "OnDemandVBlankInterrupts" "true"
+EndSection
+
+Section "Extensions"
+#    Option "Composite" "Disabled"
+EndSection
diff --git a/tests/run.py b/tests/run.py
index 8167b911..2e962f2f 100644
--- a/tests/run.py
+++ b/tests/run.py
@@ -8,7 +8,7 @@
         python run.py [testfile ...]
 
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -16,11 +16,19 @@ from __future__ import print_function
 
 import os
 import sys
+import warnings
 
 # only find tests in this directory
 if os.path.dirname(__file__):
     os.chdir(os.path.dirname(__file__))
 
+# make FutureWarnings (coming from Regex syntax most likely) and
+# DeprecationWarnings due to non-raw strings an error
+warnings.filterwarnings("error", module=r"pygments\..*",
+                        category=FutureWarning)
+warnings.filterwarnings("error", module=r".*pygments.*",
+                        category=DeprecationWarning)
+
 
 try:
     import nose
diff --git a/tests/string_asserts.py b/tests/string_asserts.py
index 11f5c7f0..05e95e6a 100644
--- a/tests/string_asserts.py
+++ b/tests/string_asserts.py
@@ -3,7 +3,7 @@
     Pygments string assert utility
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/support/empty.py b/tests/support/empty.py
new file mode 100644
index 00000000..40a96afc
--- /dev/null
+++ b/tests/support/empty.py
@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-
diff --git a/tests/support/html_formatter.py b/tests/support/html_formatter.py
new file mode 100644
index 00000000..169cd4af
--- /dev/null
+++ b/tests/support/html_formatter.py
@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+from pygments.formatters import HtmlFormatter
+
+
+class HtmlFormatterWrapper(HtmlFormatter):
+    name = 'HtmlWrapper'
diff --git a/tests/support/python_lexer.py b/tests/support/python_lexer.py
new file mode 100644
index 00000000..565ee674
--- /dev/null
+++ b/tests/support/python_lexer.py
@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+# pygments.lexers.python (as CustomLexer) for test_cmdline.py
+
+from pygments.lexers import PythonLexer
+
+
+class CustomLexer(PythonLexer):
+    name = 'PythonLexerWrapper'
+
+
+class LexerWrapper(CustomLexer):
+    name = 'PythonLexerWrapperWrapper'
diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py
index 022e6c55..ac3b4a51 100644
--- a/tests/test_basic_api.py
+++ b/tests/test_basic_api.py
@@ -3,7 +3,7 @@
     Pygments basic API tests
     ~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -161,8 +161,8 @@ def test_formatter_public_api():
 
         try:
             inst = formatter(opt1="val1")
-        except (ImportError, FontNotFound):
-            raise support.SkipTest
+        except (ImportError, FontNotFound) as e:
+            raise support.SkipTest(e)
 
         try:
             inst.get_style_defs()
@@ -209,9 +209,9 @@ def test_formatter_unicode_handling():
     def verify(formatter):
         try:
             inst = formatter(encoding=None)
-        except (ImportError, FontNotFound):
+        except (ImportError, FontNotFound) as e:
             # some dependency or font not installed
-            raise support.SkipTest
+            raise support.SkipTest(e)
 
         if formatter.name != 'Raw tokens':
             out = format(tokens, inst)
diff --git a/tests/test_bibtex.py b/tests/test_bibtex.py
index d007766d..5ad92db4 100644
--- a/tests/test_bibtex.py
+++ b/tests/test_bibtex.py
@@ -3,7 +3,7 @@
     BibTeX Test
     ~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_cfm.py b/tests/test_cfm.py
index 2585489a..0ff1b167 100644
--- a/tests/test_cfm.py
+++ b/tests/test_cfm.py
@@ -3,7 +3,7 @@
     Basic ColdfusionHtmlLexer Test
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_clexer.py b/tests/test_clexer.py
index fd7f58fc..5095b797 100644
--- a/tests/test_clexer.py
+++ b/tests/test_clexer.py
@@ -3,7 +3,7 @@
     Basic CLexer Test
     ~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_cmdline.py b/tests/test_cmdline.py
index 5883fb5c..a55e30ec 100644
--- a/tests/test_cmdline.py
+++ b/tests/test_cmdline.py
@@ -3,7 +3,7 @@
     Command line test
     ~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -28,6 +28,13 @@ def func(args):
 '''
 
 
+def _decode_output(text):
+    try:
+        return text.decode('utf-8')
+    except UnicodeEncodeError:  # implicit encode on Python 2 with data loss
+        return text
+
+
 def run_cmdline(*args, **kwds):
     saved_stdin = sys.stdin
     saved_stdout = sys.stdout
@@ -53,9 +60,9 @@ def run_cmdline(*args, **kwds):
         sys.stderr = saved_stderr
     new_stdout.flush()
     new_stderr.flush()
-    out, err = stdout_buffer.getvalue().decode('utf-8'), \
-        stderr_buffer.getvalue().decode('utf-8')
-    return (ret, out, err)
+    out, err = stdout_buffer.getvalue(), \
+        stderr_buffer.getvalue()
+    return (ret, _decode_output(out), _decode_output(err))
 
 
 class CmdLineTest(unittest.TestCase):
@@ -110,6 +117,32 @@ class CmdLineTest(unittest.TestCase):
             finally:
                 os.unlink(name)
 
+    def test_load_from_file(self):
+        lexer_file = os.path.join(TESTDIR, 'support', 'python_lexer.py')
+        formatter_file = os.path.join(TESTDIR, 'support', 'html_formatter.py')
+
+        # By default, use CustomLexer
+        o = self.check_success('-l', lexer_file, '-f', 'html',
+                               '-x', stdin=TESTCODE)
+        o = re.sub('<[^>]*>', '', o)
+        # rstrip is necessary since HTML inserts a \n after the last </div>
+        self.assertEqual(o.rstrip(), TESTCODE.rstrip())
+
+        # If user specifies a name, use it
+        o = self.check_success('-f', 'html', '-x', '-l',
+                               lexer_file + ':LexerWrapper', stdin=TESTCODE)
+        o = re.sub('<[^>]*>', '', o)
+        # rstrip is necessary since HTML inserts a \n after the last </div>
+        self.assertEqual(o.rstrip(), TESTCODE.rstrip())
+
+        # Should also work for formatters
+        o = self.check_success('-lpython', '-f',
+                               formatter_file + ':HtmlFormatterWrapper',
+                               '-x', stdin=TESTCODE)
+        o = re.sub('<[^>]*>', '', o)
+        # rstrip is necessary since HTML inserts a \n after the last </div>
+        self.assertEqual(o.rstrip(), TESTCODE.rstrip())
+
     def test_stream_opt(self):
         o = self.check_success('-lpython', '-s', '-fterminal', stdin=TESTCODE)
         o = re.sub(r'\x1b\[.*?m', '', o)
@@ -211,6 +244,20 @@ class CmdLineTest(unittest.TestCase):
         e = self.check_failure('-lfooo', TESTFILE)
         self.assertTrue('Error: no lexer for alias' in e)
 
+        # cannot load .py file without load_from_file flag
+        e = self.check_failure('-l', 'nonexistent.py', TESTFILE)
+        self.assertTrue('Error: no lexer for alias' in e)
+
+        # lexer file is missing/unreadable
+        e = self.check_failure('-l', 'nonexistent.py',
+                               '-x', TESTFILE)
+        self.assertTrue('Error: cannot read' in e)
+
+        # lexer file is malformed
+        e = self.check_failure('-l', 'support/empty.py',
+                               '-x', TESTFILE)
+        self.assertTrue('Error: no valid CustomLexer class found' in e)
+
         # formatter not found
         e = self.check_failure('-lpython', '-ffoo', TESTFILE)
         self.assertTrue('Error: no formatter found for name' in e)
@@ -219,6 +266,20 @@ class CmdLineTest(unittest.TestCase):
         e = self.check_failure('-ofoo.foo', TESTFILE)
         self.assertTrue('Error: no formatter found for file name' in e)
 
+        # cannot load .py file without load_from_file flag
+        e = self.check_failure('-f', 'nonexistent.py', TESTFILE)
+        self.assertTrue('Error: no formatter found for name' in e)
+
+        # formatter file is missing/unreadable
+        e = self.check_failure('-f', 'nonexistent.py',
+                               '-x', TESTFILE)
+        self.assertTrue('Error: cannot read' in e)
+
+        # formatter file is malformed
+        e = self.check_failure('-f', 'support/empty.py',
+                               '-x', TESTFILE)
+        self.assertTrue('Error: no valid CustomFormatter class found' in e)
+
         # output file not writable
         e = self.check_failure('-o', os.path.join('nonexistent', 'dir', 'out.html'),
                                '-lpython', TESTFILE)
diff --git a/tests/test_csound.py b/tests/test_csound.py
new file mode 100644
index 00000000..d493bd04
--- /dev/null
+++ b/tests/test_csound.py
@@ -0,0 +1,491 @@
+# -*- coding: utf-8 -*-
+"""
+    Csound lexer tests
+    ~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import unittest
+from textwrap import dedent
+
+from pygments.token import Comment, Error, Keyword, Name, Number, Operator, Punctuation, \
+    String, Text
+from pygments.lexers import CsoundOrchestraLexer
+
+
+class CsoundOrchestraTest(unittest.TestCase):
+
+    def setUp(self):
+        self.lexer = CsoundOrchestraLexer()
+        self.maxDiff = None
+
+    def testComments(self):
+        fragment = dedent('''\
+            /*
+             * comment
+             */
+            ; comment
+            // comment
+        ''')
+        tokens = [
+            (Comment.Multiline, u'/*\n * comment\n */'),
+            (Text, u'\n'),
+            (Comment.Single, u'; comment'),
+            (Text, u'\n'),
+            (Comment.Single, u'// comment'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testInstrumentBlocks(self):
+        fragment = dedent('''\
+            instr/**/1,/**/N_a_M_e_,/**/+Name/**///
+              iDuration = p3
+              outc:a(aSignal)
+            endin
+        ''')
+        tokens = [
+            (Keyword.Declaration, u'instr'),
+            (Comment.Multiline, u'/**/'),
+            (Name.Function, u'1'),
+            (Punctuation, u','),
+            (Comment.Multiline, u'/**/'),
+            (Name.Function, u'N_a_M_e_'),
+            (Punctuation, u','),
+            (Comment.Multiline, u'/**/'),
+            (Punctuation, u'+'),
+            (Name.Function, u'Name'),
+            (Comment.Multiline, u'/**/'),
+            (Comment.Single, u'//'),
+            (Text, u'\n'),
+            (Text, u'  '),
+            (Keyword.Type, u'i'),
+            (Name, u'Duration'),
+            (Text, u' '),
+            (Operator, u'='),
+            (Text, u' '),
+            (Name.Variable.Instance, u'p3'),
+            (Text, u'\n'),
+            (Text, u'  '),
+            (Name.Builtin, u'outc'),
+            (Punctuation, u':'),
+            (Keyword.Type, u'a'),
+            (Punctuation, u'('),
+            (Keyword.Type, u'a'),
+            (Name, u'Signal'),
+            (Punctuation, u')'),
+            (Text, u'\n'),
+            (Keyword.Declaration, u'endin'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testUserDefinedOpcodes(self):
+        fragment = dedent('''\
+            opcode/**/aUDO,/**/i[],/**/aik//
+              aUDO
+            endop
+        ''')
+        tokens = [
+            (Keyword.Declaration, u'opcode'),
+            (Comment.Multiline, u'/**/'),
+            (Name.Function, u'aUDO'),
+            (Punctuation, u','),
+            (Comment.Multiline, u'/**/'),
+            (Keyword.Type, u'i[]'),
+            (Punctuation, u','),
+            (Comment.Multiline, u'/**/'),
+            (Keyword.Type, u'aik'),
+            (Comment.Single, u'//'),
+            (Text, u'\n'),
+            (Text, u'  '),
+            (Name.Function, u'aUDO'),
+            (Text, u'\n'),
+            (Keyword.Declaration, u'endop'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testNumbers(self):
+        fragment = '123 0123456789'
+        tokens = [
+            (Number.Integer, u'123'),
+            (Text, u' '),
+            (Number.Integer, u'0123456789'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        fragment = '0xabcdef0123456789 0XABCDEF'
+        tokens = [
+            (Keyword.Type, u'0x'),
+            (Number.Hex, u'abcdef0123456789'),
+            (Text, u' '),
+            (Keyword.Type, u'0X'),
+            (Number.Hex, u'ABCDEF'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        fragments = ['1e2', '3e+4', '5e-6', '7E8', '9E+0', '1E-2', '3.', '4.56', '.789']
+        for fragment in fragments:
+            tokens = [
+                (Number.Float, fragment),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testQuotedStrings(self):
+        fragment = '"characters$MACRO."'
+        tokens = [
+            (String, u'"'),
+            (String, u'characters'),
+            (Comment.Preproc, u'$MACRO.'),
+            (String, u'"'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testBracedStrings(self):
+        fragment = dedent('''\
+            {{
+            characters$MACRO.
+            }}
+        ''')
+        tokens = [
+            (String, u'{{'),
+            (String, u'\ncharacters$MACRO.\n'),
+            (String, u'}}'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testEscapeSequences(self):
+        for character in ['\\', 'a', 'b', 'n', 'r', 't', '"', '012', '345', '67']:
+            escapedCharacter = '\\' + character
+            fragment = '"' + escapedCharacter + '"'
+            tokens = [
+                (String, u'"'),
+                (String.Escape, escapedCharacter),
+                (String, u'"'),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+            fragment = '{{' + escapedCharacter + '}}'
+            tokens = [
+                (String, u'{{'),
+                (String.Escape, escapedCharacter),
+                (String, u'}}'),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testOperators(self):
+        fragments = ['+', '-', '~', u'¬', '!', '*', '/', '^', '%', '<<', '>>', '<', '>',
+                     '<=', '>=', '==', '!=', '&', '#', '|', '&&', '||', '?', ':', '+=',
+                     '-=', '*=', '/=']
+        for fragment in fragments:
+            tokens = [
+                (Operator, fragment),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testGlobalValueIdentifiers(self):
+        for fragment in ['0dbfs', 'A4', 'kr', 'ksmps', 'nchnls', 'nchnls_i', 'sr']:
+            tokens = [
+                (Name.Variable.Global, fragment),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testKeywords(self):
+        fragments = ['do', 'else', 'elseif', 'endif', 'enduntil', 'fi', 'if', 'ithen',
+                     'kthen', 'od', 'then', 'until', 'while']
+        for fragment in fragments:
+            tokens = [
+                (Keyword, fragment),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        for fragment in ['return', 'rireturn']:
+            tokens = [
+                (Keyword.Pseudo, fragment),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testLabels(self):
+        fragment = dedent('''\
+            aLabel:
+             label2:
+        ''')
+        tokens = [
+            (Name.Label, u'aLabel'),
+            (Punctuation, u':'),
+            (Text, u'\n'),
+            (Text, u' '),
+            (Name.Label, u'label2'),
+            (Punctuation, u':'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testPrintksAndPrintsEscapeSequences(self):
+        escapedCharacters = ['%!', '%%', '%n', '%N', '%r', '%R', '%t', '%T', '\\\\a',
+                             '\\\\A', '\\\\b', '\\\\B', '\\\\n', '\\\\N', '\\\\r',
+                             '\\\\R', '\\\\t', '\\\\T']
+        for opcode in ['printks', 'prints']:
+            for escapedCharacter in escapedCharacters:
+                fragment = opcode + ' "' + escapedCharacter + '"'
+                tokens = [
+                    (Name.Builtin, opcode),
+                    (Text, u' '),
+                    (String, u'"'),
+                    (String.Escape, escapedCharacter),
+                    (String, u'"'),
+                    (Text, u'\n')
+                ]
+                self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testGotoStatements(self):
+        for keyword in ['goto', 'igoto', 'kgoto']:
+            fragment = keyword + ' aLabel'
+            tokens = [
+                (Keyword, keyword),
+                (Text, u' '),
+                (Name.Label, u'aLabel'),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        for opcode in ['reinit', 'rigoto', 'tigoto']:
+            fragment = opcode + ' aLabel'
+            tokens = [
+                (Keyword.Pseudo, opcode),
+                (Text, u' '),
+                (Name.Label, u'aLabel'),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        for opcode in ['cggoto', 'cigoto', 'cingoto', 'ckgoto', 'cngoto', 'cnkgoto']:
+            fragment = opcode + ' 1==0, aLabel'
+            tokens = [
+                (Keyword.Pseudo, opcode),
+                (Text, u' '),
+                (Number.Integer, u'1'),
+                (Operator, u'=='),
+                (Number.Integer, u'0'),
+                (Punctuation, u','),
+                (Text, u' '),
+                (Name.Label, u'aLabel'),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        fragment = 'timout 0, 0, aLabel'
+        tokens = [
+            (Keyword.Pseudo, 'timout'),
+            (Text, u' '),
+            (Number.Integer, u'0'),
+            (Punctuation, u','),
+            (Text, u' '),
+            (Number.Integer, u'0'),
+            (Punctuation, u','),
+            (Text, u' '),
+            (Name.Label, u'aLabel'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+        for opcode in ['loop_ge', 'loop_gt', 'loop_le', 'loop_lt']:
+            fragment = opcode + ' 0, 0, 0, aLabel'
+            tokens = [
+                (Keyword.Pseudo, opcode),
+                (Text, u' '),
+                (Number.Integer, u'0'),
+                (Punctuation, u','),
+                (Text, u' '),
+                (Number.Integer, u'0'),
+                (Punctuation, u','),
+                (Text, u' '),
+                (Number.Integer, u'0'),
+                (Punctuation, u','),
+                (Text, u' '),
+                (Name.Label, u'aLabel'),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testIncludeDirectives(self):
+        for character in ['"', '|']:
+            fragment = '#include/**/' + character + 'file.udo' + character
+            tokens = [
+                (Comment.Preproc, u'#include'),
+                (Comment.Multiline, u'/**/'),
+                (String, character + u'file.udo' + character),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testObjectLikeMacroDefinitions(self):
+        fragment = dedent('''\
+            # \tdefine MACRO#macro_body#
+            #define/**/
+            MACRO/**/
+            #\\#macro
+            body\\##
+        ''')
+        tokens = [
+            (Comment.Preproc, u'# \tdefine'),
+            (Text, u' '),
+            (Comment.Preproc, u'MACRO'),
+            (Punctuation, u'#'),
+            (Comment.Preproc, u'macro_body'),
+            (Punctuation, u'#'),
+            (Text, u'\n'),
+            (Comment.Preproc, u'#define'),
+            (Comment.Multiline, u'/**/'),
+            (Text, u'\n'),
+            (Comment.Preproc, u'MACRO'),
+            (Comment.Multiline, u'/**/'),
+            (Text, u'\n'),
+            (Punctuation, u'#'),
+            (Comment.Preproc, u'\\#'),
+            (Comment.Preproc, u'macro\nbody'),
+            (Comment.Preproc, u'\\#'),
+            (Punctuation, u'#'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testFunctionLikeMacroDefinitions(self):
+        fragment = dedent('''\
+            #define MACRO(ARG1#ARG2) #macro_body#
+            #define/**/
+            MACRO(ARG1'ARG2' ARG3)/**/
+            #\\#macro
+            body\\##
+        ''')
+        tokens = [
+            (Comment.Preproc, u'#define'),
+            (Text, u' '),
+            (Comment.Preproc, u'MACRO'),
+            (Punctuation, u'('),
+            (Comment.Preproc, u'ARG1'),
+            (Punctuation, u'#'),
+            (Comment.Preproc, u'ARG2'),
+            (Punctuation, u')'),
+            (Text, u' '),
+            (Punctuation, u'#'),
+            (Comment.Preproc, u'macro_body'),
+            (Punctuation, u'#'),
+            (Text, u'\n'),
+            (Comment.Preproc, u'#define'),
+            (Comment.Multiline, u'/**/'),
+            (Text, u'\n'),
+            (Comment.Preproc, u'MACRO'),
+            (Punctuation, u'('),
+            (Comment.Preproc, u'ARG1'),
+            (Punctuation, u"'"),
+            (Comment.Preproc, u'ARG2'),
+            (Punctuation, u"'"),
+            (Text, u' '),
+            (Comment.Preproc, u'ARG3'),
+            (Punctuation, u')'),
+            (Comment.Multiline, u'/**/'),
+            (Text, u'\n'),
+            (Punctuation, u'#'),
+            (Comment.Preproc, u'\\#'),
+            (Comment.Preproc, u'macro\nbody'),
+            (Comment.Preproc, u'\\#'),
+            (Punctuation, u'#'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testMacroPreprocessorDirectives(self):
+        for directive in ['#ifdef', '#ifndef', '#undef']:
+            fragment = directive + ' MACRO'
+            tokens = [
+                (Comment.Preproc, directive),
+                (Text, u' '),
+                (Comment.Preproc, u'MACRO'),
+                (Text, u'\n')
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testOtherPreprocessorDirectives(self):
+        fragment = dedent('''\
+            #else
+            #end
+            #endif
+            ###
+            @ \t12345
+            @@ \t67890
+        ''')
+        tokens = [
+            (Comment.Preproc, u'#else'),
+            (Text, u'\n'),
+            (Comment.Preproc, u'#end'),
+            (Text, u'\n'),
+            (Comment.Preproc, u'#endif'),
+            (Text, u'\n'),
+            (Comment.Preproc, u'###'),
+            (Text, u'\n'),
+            (Comment.Preproc, u'@ \t12345'),
+            (Text, u'\n'),
+            (Comment.Preproc, u'@@ \t67890'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testFunctionLikeMacros(self):
+        fragment = "$MACRO.(((x#y\\)))' \"(#'x)\\)x\\))\"# {{x\\))x)\\)(#'}});"
+        tokens = [
+            (Comment.Preproc, u'$MACRO.'),
+            (Punctuation, u'('),
+            (Comment.Preproc, u'('),
+            (Comment.Preproc, u'('),
+            (Comment.Preproc, u'x#y\\)'),
+            (Comment.Preproc, u')'),
+            (Comment.Preproc, u')'),
+            (Punctuation, u"'"),
+            (Comment.Preproc, u' '),
+            (String, u'"'),
+            (Error, u'('),
+            (Error, u'#'),
+            (Error, u"'"),
+            (String, u'x'),
+            (Error, u')'),
+            (Comment.Preproc, u'\\)'),
+            (String, u'x'),
+            (Comment.Preproc, u'\\)'),
+            (Error, u')'),
+            (String, u'"'),
+            (Punctuation, u'#'),
+            (Comment.Preproc, u' '),
+            (String, u'{{'),
+            (String, u'x'),
+            (Comment.Preproc, u'\\)'),
+            (Error, u')'),
+            (String, u'x'),
+            (Error, u')'),
+            (Comment.Preproc, u'\\)'),
+            (Error, u'('),
+            (Error, u'#'),
+            (Error, u"'"),
+            (String, u'}}'),
+            (Punctuation, u')'),
+            (Comment.Single, u';'),
+            (Text, u'\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testName(self):
+        fragment = 'kG:V'
+        tokens = [
+            (Keyword.Type, 'k'),
+            (Name, 'G'),
+            (Punctuation, ':'),
+            (Name, 'V'),
+            (Text, '\n')
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
diff --git a/tests/test_examplefiles.py b/tests/test_examplefiles.py
index f43abf9b..2fae1125 100644
--- a/tests/test_examplefiles.py
+++ b/tests/test_examplefiles.py
@@ -3,7 +3,7 @@
     Pygments tests with example files
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -89,7 +89,7 @@ def test_example_files():
 
 def check_lexer(lx, fn):
     if os.name == 'java' and fn in BAD_FILES_FOR_JYTHON:
-        raise support.SkipTest
+        raise support.SkipTest('%s is a known bad file on Jython' % fn)
     absfn = os.path.join(TESTDIR, 'examplefiles', fn)
     with open(absfn, 'rb') as fp:
         text = fp.read()
diff --git a/tests/test_html_formatter.py b/tests/test_html_formatter.py
index 596d9fbc..670a5be9 100644
--- a/tests/test_html_formatter.py
+++ b/tests/test_html_formatter.py
@@ -3,7 +3,7 @@
     Pygments HTML formatter tests
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -100,7 +100,7 @@ class HtmlFormatterTest(unittest.TestCase):
         fmt = HtmlFormatter(**optdict)
         fmt.format(tokensource, outfile)
         html = outfile.getvalue()
-        self.assertTrue(re.search("<pre>\s+1\s+2\s+3", html))
+        self.assertTrue(re.search(r"<pre>\s+1\s+2\s+3", html))
 
     def test_linenos_with_startnum(self):
         optdict = dict(linenos=True, linenostart=5)
@@ -108,7 +108,7 @@ class HtmlFormatterTest(unittest.TestCase):
         fmt = HtmlFormatter(**optdict)
         fmt.format(tokensource, outfile)
         html = outfile.getvalue()
-        self.assertTrue(re.search("<pre>\s+5\s+6\s+7", html))
+        self.assertTrue(re.search(r"<pre>\s+5\s+6\s+7", html))
 
     def test_lineanchors(self):
         optdict = dict(lineanchors="foo")
@@ -132,9 +132,8 @@ class HtmlFormatterTest(unittest.TestCase):
                             outencoding='utf-8')
 
         handle, pathname = tempfile.mkstemp('.html')
-        tfile = os.fdopen(handle, 'w+b')
-        fmt.format(tokensource, tfile)
-        tfile.close()
+        with os.fdopen(handle, 'w+b') as tfile:
+            fmt.format(tokensource, tfile)
         catname = os.path.join(TESTDIR, 'dtds', 'HTML4.soc')
         try:
             import subprocess
@@ -173,9 +172,8 @@ class HtmlFormatterTest(unittest.TestCase):
                             cssstyles=u'div:before { content: \'bäz\' }',
                             encoding='utf-8')
         handle, pathname = tempfile.mkstemp('.html')
-        tfile = os.fdopen(handle, 'w+b')
-        fmt.format(tokensource, tfile)
-        tfile.close()
+        with os.fdopen(handle, 'w+b') as tfile:
+            fmt.format(tokensource, tfile)
 
     def test_ctags(self):
         try:
diff --git a/tests/test_inherit.py b/tests/test_inherit.py
index 34033a08..5da57dd9 100644
--- a/tests/test_inherit.py
+++ b/tests/test_inherit.py
@@ -3,7 +3,7 @@
     Tests for inheritance in RegexLexer
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_irc_formatter.py b/tests/test_irc_formatter.py
index 16a8fd30..3b34f0bc 100644
--- a/tests/test_irc_formatter.py
+++ b/tests/test_irc_formatter.py
@@ -3,7 +3,7 @@
     Pygments IRC formatter tests
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_java.py b/tests/test_java.py
index f4096647..6e5e8992 100644
--- a/tests/test_java.py
+++ b/tests/test_java.py
@@ -3,7 +3,7 @@
     Basic JavaLexer Test
     ~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_javascript.py b/tests/test_javascript.py
index 59890659..21dff7c4 100644
--- a/tests/test_javascript.py
+++ b/tests/test_javascript.py
@@ -3,7 +3,7 @@
     Javascript tests
     ~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_julia.py b/tests/test_julia.py
index 08c420d3..ed46f27e 100644
--- a/tests/test_julia.py
+++ b/tests/test_julia.py
@@ -3,7 +3,7 @@
     Julia Tests
     ~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_latex_formatter.py b/tests/test_latex_formatter.py
index 05a6c3ac..ebed7964 100644
--- a/tests/test_latex_formatter.py
+++ b/tests/test_latex_formatter.py
@@ -3,7 +3,7 @@
     Pygments LaTeX formatter tests
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -42,9 +42,9 @@ class LatexFormatterTest(unittest.TestCase):
             ret = po.wait()
             output = po.stdout.read()
             po.stdout.close()
-        except OSError:
+        except OSError as e:
             # latex not available
-            raise support.SkipTest
+            raise support.SkipTest(e)
         else:
             if ret:
                 print(output)
diff --git a/tests/test_lexers_other.py b/tests/test_lexers_other.py
index 90d05ef8..3716fb72 100644
--- a/tests/test_lexers_other.py
+++ b/tests/test_lexers_other.py
@@ -3,7 +3,7 @@
     Tests for other lexers
     ~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 import glob
diff --git a/tests/test_markdown_lexer.py b/tests/test_markdown_lexer.py
new file mode 100644
index 00000000..16d1f28d
--- /dev/null
+++ b/tests/test_markdown_lexer.py
@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+"""
+    Pygments regex lexer tests
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+import unittest
+
+from pygments.lexers.markup import MarkdownLexer
+
+
+class SameTextTests(unittest.TestCase):
+
+    lexer = MarkdownLexer()
+
+    def assert_same_text(self, text):
+        """Show that lexed markdown does not remove any content. """
+        tokens = list(self.lexer.get_tokens_unprocessed(text))
+        output = ''.join(t[2] for t in tokens)
+        self.assertEqual(text, output)
+
+    def test_code_fence(self):
+        self.assert_same_text(r'```\nfoo\n```\n')
+
+    def test_code_fence_gsm(self):
+        self.assert_same_text(r'```markdown\nfoo\n```\n')
+
+    def test_code_fence_gsm_with_no_lexer(self):
+        self.assert_same_text(r'```invalid-lexer\nfoo\n```\n')
diff --git a/tests/test_modeline.py b/tests/test_modeline.py
new file mode 100644
index 00000000..efe038df
--- /dev/null
+++ b/tests/test_modeline.py
@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+"""
+    Tests for the vim modeline feature
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from __future__ import print_function
+
+from pygments import modeline
+
+
+def test_lexer_classes():
+    def verify(buf):
+        assert modeline.get_filetype_from_buffer(buf) == 'python'
+
+    for buf in [
+            'vi: ft=python' + '\n' * 8,
+            'vi: ft=python' + '\n' * 8,
+            '\n\n\n\nvi=8: syntax=python' + '\n' * 8,
+            '\n' * 8 + 'ex: filetype=python',
+            '\n' * 8 + 'vim: some,other,syn=python\n\n\n\n'
+    ]:
+        yield verify, buf
diff --git a/tests/test_objectiveclexer.py b/tests/test_objectiveclexer.py
index 90bd680f..aee7db66 100644
--- a/tests/test_objectiveclexer.py
+++ b/tests/test_objectiveclexer.py
@@ -3,7 +3,7 @@
     Basic CLexer Test
     ~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_perllexer.py b/tests/test_perllexer.py
index 26b2d0a7..102f0a9f 100644
--- a/tests/test_perllexer.py
+++ b/tests/test_perllexer.py
@@ -3,14 +3,14 @@
     Pygments regex lexer tests
     ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import time
 import unittest
 
-from pygments.token import String
+from pygments.token import Keyword, Name, String, Text
 from pygments.lexers.perl import PerlLexer
 
 
@@ -135,3 +135,23 @@ class RunawayRegexTest(unittest.TestCase):
     def test_substitution_with_parenthesis(self):
         self.assert_single_token(r's(aaa)', String.Regex)
         self.assert_fast_tokenization('s(' + '\\'*999)
+
+    ### Namespaces/modules
+
+    def test_package_statement(self):
+        self.assert_tokens(['package', ' ', 'Foo'], [Keyword, Text, Name.Namespace])
+        self.assert_tokens(['package', '  ', 'Foo::Bar'], [Keyword, Text, Name.Namespace])
+
+    def test_use_statement(self):
+        self.assert_tokens(['use', ' ', 'Foo'], [Keyword, Text, Name.Namespace])
+        self.assert_tokens(['use', '  ', 'Foo::Bar'], [Keyword, Text, Name.Namespace])
+
+    def test_no_statement(self):
+        self.assert_tokens(['no', ' ', 'Foo'], [Keyword, Text, Name.Namespace])
+        self.assert_tokens(['no', '  ', 'Foo::Bar'], [Keyword, Text, Name.Namespace])
+
+    def test_require_statement(self):
+        self.assert_tokens(['require', ' ', 'Foo'], [Keyword, Text, Name.Namespace])
+        self.assert_tokens(['require', '  ', 'Foo::Bar'], [Keyword, Text, Name.Namespace])
+        self.assert_tokens(['require', ' ', '"Foo/Bar.pm"'], [Keyword, Text, String])
+
diff --git a/tests/test_php.py b/tests/test_php.py
index 050ca70d..b4117381 100644
--- a/tests/test_php.py
+++ b/tests/test_php.py
@@ -3,7 +3,7 @@
     PHP Tests
     ~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_praat.py b/tests/test_praat.py
index 471d5e2c..1ca97d1e 100644
--- a/tests/test_praat.py
+++ b/tests/test_praat.py
@@ -3,7 +3,7 @@
     Praat lexer tests
     ~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_properties.py b/tests/test_properties.py
index 333f3d7a..562778ba 100644
--- a/tests/test_properties.py
+++ b/tests/test_properties.py
@@ -3,7 +3,7 @@
     Properties Tests
     ~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_python.py b/tests/test_python.py
index f5784cb1..6445022c 100644
--- a/tests/test_python.py
+++ b/tests/test_python.py
@@ -3,7 +3,7 @@
     Python Tests
     ~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -111,3 +111,23 @@ class Python3Test(unittest.TestCase):
             (Token.Text, u'\n'),
         ]
         self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def test_pep_515(self):
+        """
+        Tests that the lexer can parse numeric literals with underscores
+        """
+        fragments = (
+            (Token.Literal.Number.Integer, u'1_000_000'),
+            (Token.Literal.Number.Float, u'1_000.000_001'),
+            (Token.Literal.Number.Float, u'1_000e1_000j'),
+            (Token.Literal.Number.Hex, u'0xCAFE_F00D'),
+            (Token.Literal.Number.Bin, u'0b_0011_1111_0100_1110'),
+            (Token.Literal.Number.Oct, u'0o_777_123'),
+        )
+
+        for token, fragment in fragments:
+            tokens = [
+                (token, fragment),
+                (Token.Text, u'\n'),
+            ]
+            self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
diff --git a/tests/test_qbasiclexer.py b/tests/test_qbasiclexer.py
index 8b790cee..0ea221a1 100644
--- a/tests/test_qbasiclexer.py
+++ b/tests/test_qbasiclexer.py
@@ -3,7 +3,7 @@
     Tests for QBasic
     ~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_r.py b/tests/test_r.py
new file mode 100644
index 00000000..70148e53
--- /dev/null
+++ b/tests/test_r.py
@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+"""
+    R Tests
+    ~~~~~~~~~
+
+    :copyright: Copyright 2006-2016 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import unittest
+
+from pygments.lexers import SLexer
+from pygments.token import Token, Name, Punctuation
+
+
+class RTest(unittest.TestCase):
+    def setUp(self):
+        self.lexer = SLexer()
+
+    def testCall(self):
+        fragment = u'f(1, a)\n'
+        tokens = [
+            (Name.Function, u'f'),
+            (Punctuation, u'('),
+            (Token.Literal.Number, u'1'),
+            (Punctuation, u','),
+            (Token.Text, u' '),
+            (Token.Name, u'a'),
+            (Punctuation, u')'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testName1(self):
+        fragment = u'._a_2.c'
+        tokens = [
+            (Name, u'._a_2.c'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testName2(self):
+        # Invalid names are valid if backticks are used
+        fragment = u'`.1 blah`'
+        tokens = [
+            (Name, u'`.1 blah`'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testName3(self):
+        # Internal backticks can be escaped
+        fragment = u'`.1 \\` blah`'
+        tokens = [
+            (Name, u'`.1 \\` blah`'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+    def testCustomOperator(self):
+        fragment = u'7 % and % 8'
+        tokens = [
+            (Token.Literal.Number, u'7'),
+            (Token.Text, u' '),
+            (Token.Operator, u'% and %'),
+            (Token.Text, u' '),
+            (Token.Literal.Number, u'8'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
diff --git a/tests/test_regexlexer.py b/tests/test_regexlexer.py
index eb25be61..d919a950 100644
--- a/tests/test_regexlexer.py
+++ b/tests/test_regexlexer.py
@@ -3,7 +3,7 @@
     Pygments regex lexer tests
     ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_regexopt.py b/tests/test_regexopt.py
index 6322c735..5cfb62a3 100644
--- a/tests/test_regexopt.py
+++ b/tests/test_regexopt.py
@@ -3,7 +3,7 @@
     Tests for pygments.regexopt
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_rtf_formatter.py b/tests/test_rtf_formatter.py
index 25784743..44da5768 100644
--- a/tests/test_rtf_formatter.py
+++ b/tests/test_rtf_formatter.py
@@ -3,7 +3,7 @@
     Pygments RTF formatter tests
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -80,7 +80,7 @@ class RtfFormatterTest(StringTests, unittest.TestCase):
         self.assertEndsWith(result, expected+self.foot, msg)
 
     def test_escape_characters(self):
-        t = u'\ {{'
+        t = u'\\ {{'
         result = self.format_rtf(t)
         expected = (r'\\ \{\{')
         if not result.endswith(self.foot):
diff --git a/tests/test_ruby.py b/tests/test_ruby.py
index ab210bad..b7d4110a 100644
--- a/tests/test_ruby.py
+++ b/tests/test_ruby.py
@@ -3,7 +3,7 @@
     Basic RubyLexer Test
     ~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_shell.py b/tests/test_shell.py
index 6faac9fd..e283793e 100644
--- a/tests/test_shell.py
+++ b/tests/test_shell.py
@@ -3,7 +3,7 @@
     Basic Shell Tests
     ~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_smarty.py b/tests/test_smarty.py
index 450e4e6b..e1e079d9 100644
--- a/tests/test_smarty.py
+++ b/tests/test_smarty.py
@@ -3,7 +3,7 @@
     Basic SmartyLexer Test
     ~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_sql.py b/tests/test_sql.py
index c5f5c758..6be34006 100644
--- a/tests/test_sql.py
+++ b/tests/test_sql.py
@@ -8,7 +8,10 @@
 """
 import unittest
 
-from pygments.lexers.sql import TransactSqlLexer
+from pygments.lexers.sql import name_between_bracket_re, \
+    name_between_backtick_re, tsql_go_re, tsql_declare_re, \
+    tsql_variable_re, MySqlLexer, SqlLexer, TransactSqlLexer
+
 from pygments.token import Comment, Name, Number, Punctuation, Whitespace
 
 
@@ -72,3 +75,44 @@ class TransactSqlLexerTest(unittest.TestCase):
             (Comment.Multiline, '*/'),
             (Comment.Multiline, '*/'),
         ))
+
+
+class SqlAnalyzeTextTest(unittest.TestCase):
+    def test_can_match_analyze_text_res(self):
+        self.assertEqual(['`a`', '`bc`'],
+            name_between_backtick_re.findall('select `a`, `bc` from some'))
+        self.assertEqual(['[a]', '[bc]'],
+            name_between_bracket_re.findall('select [a], [bc] from some'))
+        self.assertTrue(tsql_declare_re.search('--\nDeClaRe @some int;'))
+        self.assertTrue(tsql_go_re.search('select 1\ngo\n--'))
+        self.assertTrue(tsql_variable_re.search(
+            'create procedure dbo.usp_x @a int, @b int'))
+
+    def test_can_analyze_text(self):
+        mysql_lexer = MySqlLexer()
+        sql_lexer = SqlLexer()
+        tsql_lexer = TransactSqlLexer()
+        code_to_expected_lexer_map = {
+            'select `a`, `bc` from some': mysql_lexer,
+            'select a, bc from some': sql_lexer,
+            'select [a], [bc] from some': tsql_lexer,
+            '-- `a`, `bc`\nselect [a], [bc] from some': tsql_lexer,
+            '-- `a`, `bc`\nselect [a], [bc] from some; go': tsql_lexer,
+        }
+        sql_lexers = set(code_to_expected_lexer_map.values())
+        for code, expected_lexer in code_to_expected_lexer_map.items():
+            ratings_and_lexers = list((lexer.analyse_text(code), lexer.name) for lexer in sql_lexers)
+            best_rating, best_lexer_name  = sorted(ratings_and_lexers, reverse=True)[0]
+            expected_rating = expected_lexer.analyse_text(code)
+            message = (
+                'lexer must be %s (rating %.2f) instead of '
+                '%s (rating %.2f) for analyse_text() on code:\n%s') % (
+                expected_lexer.name,
+                expected_rating,
+                best_lexer_name,
+                best_rating,
+                code
+            )
+            self.assertEqual(
+                expected_lexer.name, best_lexer_name, message
+            )
diff --git a/tests/test_string_asserts.py b/tests/test_string_asserts.py
index ba7b37fa..5e9e5617 100644
--- a/tests/test_string_asserts.py
+++ b/tests/test_string_asserts.py
@@ -3,7 +3,7 @@
     Pygments string assert utility tests
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_terminal_formatter.py b/tests/test_terminal_formatter.py
index cb5c6c44..1f44807d 100644
--- a/tests/test_terminal_formatter.py
+++ b/tests/test_terminal_formatter.py
@@ -3,7 +3,7 @@
     Pygments terminal formatter tests
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -61,10 +61,10 @@ class TerminalFormatterTest(unittest.TestCase):
 
 class MyStyle(Style):
     styles = {
-        Token.Comment:    '#ansidarkgray',
-        Token.String:     '#ansiblue bg:#ansidarkred',
-        Token.Number:     '#ansigreen bg:#ansidarkgreen',
-        Token.Number.Hex: '#ansidarkgreen bg:#ansired',
+        Token.Comment:    'ansibrightblack',
+        Token.String:     'ansibrightblue bg:ansired',
+        Token.Number:     'ansibrightgreen bg:ansigreen',
+        Token.Number.Hex: 'ansigreen bg:ansibrightred',
     }
 
 
@@ -90,7 +90,7 @@ async def function(a,b,c, *d, **kwarg:Bool)->Bool:
 
     def test_256esc_seq(self):
         """
-        test that a few escape sequences are actualy used when using #ansi<> color codes
+        test that a few escape sequences are actualy used when using ansi<> color codes
         """
         def termtest(x):
             return highlight(x, Python3Lexer(),
diff --git a/tests/test_textfmts.py b/tests/test_textfmts.py
index d355ab68..453dd61f 100644
--- a/tests/test_textfmts.py
+++ b/tests/test_textfmts.py
@@ -3,7 +3,7 @@
     Basic Tests for textfmts
     ~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_token.py b/tests/test_token.py
index 0c6b02bf..94522373 100644
--- a/tests/test_token.py
+++ b/tests/test_token.py
@@ -3,7 +3,7 @@
     Test suite for the token module
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_unistring.py b/tests/test_unistring.py
index a414347c..c56b68c7 100644
--- a/tests/test_unistring.py
+++ b/tests/test_unistring.py
@@ -3,7 +3,7 @@
     Test suite for the unistring module
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_using_api.py b/tests/test_using_api.py
index 16d865e6..7517ce7d 100644
--- a/tests/test_using_api.py
+++ b/tests/test_using_api.py
@@ -3,7 +3,7 @@
     Pygments tests for using()
     ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
diff --git a/tests/test_util.py b/tests/test_util.py
index 720b384a..cdb58b3f 100644
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -3,7 +3,7 @@
     Test suite for the util module
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
author	Anteru <bitbucket@ca.sh13.net>	2019-03-06 18:58:58 +0000
committer	Anteru <bitbucket@ca.sh13.net>	2019-03-06 18:58:58 +0000
commit	9b9886734a7e9a33f268716212ee8c2159fb4242 (patch)
tree	670f7ac89b2fca92b9c5f10dd49d28cdb7f574e0 /tests
parent	590580e08ee53b7934a7886bf43f42878d77ce5c (diff)
parent	30a5c5fd20f8062f63db7315c644c10b47f4b102 (diff)
download	pygments-9b9886734a7e9a33f268716212ee8c2159fb4242.tar.gz