Merged in jaingaurav2/pygments-main (pull request #314)

20 files changed, 1236 insertions, 6 deletions

diff --git a/tests/examplefiles/core.cljs b/tests/examplefiles/core.cljs
new file mode 100644
index 00000000..f135b832
--- /dev/null
+++ b/tests/examplefiles/core.cljs
@@ -0,0 +1,52 @@
+
+(ns bounder.core
+  (:require [bounder.html :as html]
+            [domina :refer [value set-value! single-node]]
+            [domina.css :refer [sel]]
+            [lowline.functions :refer [debounce]]
+            [enfocus.core :refer [at]]
+            [cljs.reader :as reader]
+            [clojure.string :as s])
+  (:require-macros [enfocus.macros :as em]))
+
+(def filter-input 
+  (single-node 
+    (sel ".search input")))
+
+(defn project-matches [query project]
+  (let [words (cons (:name project)
+                    (map name (:categories project)))
+        to-match (->> words
+                   (s/join "")
+                   (s/lower-case))]
+    (<= 0 (.indexOf to-match (s/lower-case query)))))
+
+(defn apply-filter-for [projects]
+ (let [query (value filter-input)]
+   (html/render-projects 
+     (filter (partial project-matches query)
+             projects))))
+
+(defn filter-category [projects evt]
+  (let [target (.-currentTarget evt)]
+    (set-value! filter-input 
+                (.-innerHTML target))
+    (apply-filter-for projects)))
+
+(defn init-listeners [projects]
+  (at js/document
+    ["input"] (em/listen
+                :keyup
+                (debounce
+                  (partial apply-filter-for projects)
+                  500))
+    [".category-links li"] (em/listen
+                             :click
+                             (partial filter-category projects))))
+
+(defn init [projects-edn]
+  (let [projects (reader/read-string projects-edn)]
+    (init-listeners projects)
+    (html/render-projects projects)
+    (html/loaded)))
+
diff --git a/tests/examplefiles/demo.hbs b/tests/examplefiles/demo.hbs
new file mode 100644
index 00000000..1b9ed5a7
--- /dev/null
+++ b/tests/examplefiles/demo.hbs
@@ -0,0 +1,12 @@
+<!-- post.handlebars -->
+
+<div class='intro'>
+  {{intro}}
+</div>
+
+{{#if isExpanded}}
+  <div class='body'>{{body}}</div>
+  <button {{action contract}}>Contract</button>
+{{else}}
+  <button {{action expand}}>Show More...</button>
+{{/if}}
diff --git a/tests/examplefiles/ember.handlebars b/tests/examplefiles/ember.handlebars
new file mode 100644
index 00000000..515dffbd
--- /dev/null
+++ b/tests/examplefiles/ember.handlebars
@@ -0,0 +1,33 @@
+{{#view EmberFirebaseChat.ChatView class="chat-container"}}
+  <div class="chat-messages-container">
+    <ul class="chat-messages">
+      {{#each message in content}}
+      <li>
+        [{{formatTimestamp "message.timestamp" fmtString="h:mm:ss A"}}] 
+        <strong>{{message.sender}}</strong>: {{message.content}}
+      </li>
+      {{/each}}
+    </ul>
+  </div>
+
+  {{! Comment }}
+  {{{unescaped value}}}
+
+  {{#view EmberFirebaseChat.InputView class="chat-input-container"}}
+    <form class="form-inline">
+      {{#if "auth.authed"}}
+        {{#if "auth.hasName"}}
+          <input type="text" id="message" placeholder="Message">
+          <button {{action "postMessage" target="view"}} class="btn">Send</button>
+        {{else}}
+          <input type="text" id="username" placeholder="Enter your username...">
+          <button {{action "pickName" target="view"}} class="btn">Send</button>
+        {{/if}}
+      {{else}}
+        <input type="text" placeholder="Log in with Persona to chat!" disabled="disabled">
+        <button {{action "login"}} class="btn">Login</button>
+      {{/if}}
+    </form>
+  {{/view}}
+{{/view}}
+
diff --git a/tests/examplefiles/example.gd b/tests/examplefiles/example.gd
new file mode 100644
index 00000000..c285ea32
--- /dev/null
+++ b/tests/examplefiles/example.gd
@@ -0,0 +1,23 @@
+#############################################################################
+##
+#W  example.gd
+##
+##  This file contains a sample of a GAP declaration file.
+##
+DeclareProperty( "SomeProperty", IsLeftModule );
+DeclareGlobalFunction( "SomeGlobalFunction" );
+
+
+#############################################################################
+##
+#C  IsQuuxFrobnicator(<R>)
+##
+##  <ManSection>
+##  <Filt Name="IsQuuxFrobnicator" Arg='R' Type='Category'/>
+##
+##  <Description>
+##  Tests whether R is a quux frobnicator.
+##  </Description>
+##  </ManSection>
+##
+DeclareSynonym( "IsQuuxFrobnicator", IsField and IsGroup );
diff --git a/tests/examplefiles/example.gi b/tests/examplefiles/example.gi
new file mode 100644
index 00000000..c9c5e55d
--- /dev/null
+++ b/tests/examplefiles/example.gi
@@ -0,0 +1,64 @@
+#############################################################################
+##
+#W  example.gd
+##
+##  This file contains a sample of a GAP implementation file.
+##
+
+
+#############################################################################
+##
+#M  SomeOperation( <val> )
+##
+##  performs some operation on <val>
+##
+InstallMethod( SomeProperty,
+    "for left modules",
+    [ IsLeftModule ], 0,
+    function( M )
+    if IsFreeLeftModule( M ) and not IsTrivial( M ) then
+      return true;
+    fi;
+    TryNextMethod();
+    end );
+
+
+
+#############################################################################
+##
+#F  SomeGlobalFunction( )
+##
+##  A global variadic funfion.
+##
+InstallGlobalFunction( SomeGlobalFunction, function( arg )
+    if Length( arg ) = 3 then
+      return arg[1] + arg[2] * arg[3];
+    elif Length( arg ) = 2 then
+      return arg[1] - arg[2]
+    else
+      Error( "usage: SomeGlobalFunction( <x>, <y>[, <z>] )" );
+    fi;
+    end );
+
+
+#
+# A plain function.
+#
+SomeFunc := function(x, y)
+    local z, func, tmp, j;
+    z := x * 1.0;
+    y := 17^17 - y;
+    func := a -> a mod 5;
+    tmp := List( [1..50], func );
+    while y > 0 do
+        for j in tmp do
+            Print(j, "\n");
+        od;
+        repeat
+            y := y - 1;
+        until 0 < 1;
+        y := y -1;
+    od;
+    return z;
+end;
+        
\ No newline at end of file
diff --git a/tests/examplefiles/example.todotxt b/tests/examplefiles/example.todotxt
new file mode 100644
index 00000000..55ee5286
--- /dev/null
+++ b/tests/examplefiles/example.todotxt
@@ -0,0 +1,9 @@
+(A) Call Mom @Phone +Family
+(A) 2014-01-08 Schedule annual checkup +Health
+(B) Outline chapter 5 +Novel @Computer
+(C) Add cover sheets @Office +TPSReports
+Plan backyard herb garden @Home
+Pick up milk @GroceryStore
+Research self-publishing services +Novel @Computer
+x 2014-01-10 Download Todo.txt mobile app @Phone
+x 2014-01-10 2014-01-07 Download Todo.txt CLI @Computer
diff --git a/tests/examplefiles/hello.at b/tests/examplefiles/hello.at
new file mode 100644
index 00000000..23af2f2d
--- /dev/null
+++ b/tests/examplefiles/hello.at
@@ -0,0 +1,6 @@
+def me := object: {
+    def name := "Kevin";
+    def sayHello(peerName) {
+        system.println(peerName + " says hello!");
+    };
+};
diff --git a/tests/examplefiles/hello.golo b/tests/examplefiles/hello.golo
new file mode 100644
index 00000000..7e8ca214
--- /dev/null
+++ b/tests/examplefiles/hello.golo
@@ -0,0 +1,5 @@
+module hello.World
+
+function main = |args| {
+  println("Hello world!")
+}
diff --git a/tests/examplefiles/hello.lsl b/tests/examplefiles/hello.lsl
new file mode 100644
index 00000000..61697e7f
--- /dev/null
+++ b/tests/examplefiles/hello.lsl
@@ -0,0 +1,12 @@
+default
+{
+    state_entry()
+    {
+        llSay(0, "Hello, Avatar!");
+    }
+ 
+    touch_start(integer total_number)
+    {
+        llSay(0, "Touched.");
+    }
+}
diff --git a/tests/examplefiles/limbo.b b/tests/examplefiles/limbo.b
new file mode 100644
index 00000000..e55a0a62
--- /dev/null
+++ b/tests/examplefiles/limbo.b
@@ -0,0 +1,456 @@
+implement Ninewin;
+include "sys.m";
+	sys: Sys;
+include "draw.m";
+	draw: Draw;
+	Image, Display, Pointer: import draw;
+include "arg.m";
+include "keyboard.m";
+include "tk.m";
+include "wmclient.m";
+	wmclient: Wmclient;
+	Window: import wmclient;
+include "sh.m";
+	sh: Sh;
+
+# run a p9 graphics program (default rio) under inferno wm,
+# making available to it:
+# /dev/winname - naming the current inferno window (changing on resize)
+# /dev/mouse - pointer file + resize events; write to change position
+# /dev/cursor - change appearance of cursor.
+# /dev/draw - inferno draw device
+# /dev/cons - read keyboard events, write to 9win stdout.
+
+Ninewin: module {
+	init: fn(ctxt: ref Draw->Context, argv: list of string);
+};
+winname: string;
+
+init(ctxt: ref Draw->Context, argv: list of string)
+{
+	size := Draw->Point(500, 500);
+	sys = load Sys Sys->PATH;
+	draw = load Draw Draw->PATH;
+	wmclient = load Wmclient Wmclient->PATH;
+	wmclient->init();
+	sh = load Sh Sh->PATH;
+
+	buts := Wmclient->Resize;
+	if(ctxt == nil){
+		ctxt = wmclient->makedrawcontext();
+		buts = Wmclient->Plain;
+	}
+	arg := load Arg Arg->PATH;
+	arg->init(argv);
+	arg->setusage("9win [-s] [-x width] [-y height]");
+	exportonly := 0;
+	while(((opt := arg->opt())) != 0){
+		case opt {
+		's' =>
+			exportonly = 1;
+		'x' =>
+			size.x = int arg->earg();
+		'y' =>
+			size.y = int arg->earg();
+		* =>
+			arg->usage();
+		}
+	}
+	if(size.x < 1 || size.y < 1)
+		arg->usage();
+	argv = arg->argv();
+	if(argv != nil && hd argv == "-s"){
+		exportonly = 1;
+		argv = tl argv;
+	}
+	if(argv == nil && !exportonly)
+		argv = "rio" :: nil;
+	if(argv != nil && exportonly){
+		sys->fprint(sys->fildes(2), "9win: no command allowed with -s flag\n");
+		raise "fail:usage";
+	}
+	title := "9win";
+	if(!exportonly)
+		title += " " + hd argv;
+	w := wmclient->window(ctxt, title, buts);
+	w.reshape(((0, 0), size));
+	w.onscreen(nil);
+	if(w.image == nil){
+		sys->fprint(sys->fildes(2), "9win: cannot get image to draw on\n");
+		raise "fail:no window";
+	}
+
+	sys->pctl(Sys->FORKNS|Sys->NEWPGRP, nil);
+	ld := "/n/9win";
+	if(sys->bind("#s", ld, Sys->MREPL) == -1 &&
+			sys->bind("#s", ld = "/n/local", Sys->MREPL) == -1){
+		sys->fprint(sys->fildes(2), "9win: cannot bind files: %r\n");
+		raise "fail:error";
+	}
+	w.startinput("kbd" :: "ptr" :: nil);
+	spawn ptrproc(rq := chan of Sys->Rread, ptr := chan[10] of ref Pointer, reshape := chan[1] of int);
+
+		
+	fwinname := sys->file2chan(ld, "winname");
+	fconsctl := sys->file2chan(ld, "consctl");
+	fcons := sys->file2chan(ld, "cons");
+	fmouse := sys->file2chan(ld, "mouse");
+	fcursor := sys->file2chan(ld, "cursor");
+	if(!exportonly){
+		spawn run(sync := chan of string, w.ctl, ld, argv);
+		if((e := <-sync) != nil){
+			sys->fprint(sys->fildes(2), "9win: %s", e);
+			raise "fail:error";
+		}
+	}
+	spawn serveproc(w, rq, fwinname, fconsctl, fcons, fmouse, fcursor);
+	if(!exportonly){
+		# handle events synchronously so that we don't get a "killed" message
+		# from the shell.
+		handleevents(w, ptr, reshape);
+	}else{
+		spawn handleevents(w, ptr, reshape);
+		sys->bind(ld, "/dev", Sys->MBEFORE);
+		export(sys->fildes(0), w.ctl);
+	}
+}
+
+handleevents(w: ref Window, ptr: chan of ref Pointer, reshape: chan of int)
+{
+	for(;;)alt{
+	c := <-w.ctxt.ctl or
+	c = <-w.ctl =>
+		e := w.wmctl(c);
+		if(e != nil)
+			sys->fprint(sys->fildes(2), "9win: ctl error: %s\n", e);
+		if(e == nil && c != nil && c[0] == '!'){
+			alt{
+			reshape <-= 1 =>
+				;
+			* =>
+				;
+			}
+			winname = nil;
+		}
+	p := <-w.ctxt.ptr =>
+		if(w.pointer(*p) == 0){
+			# XXX would block here if client isn't reading mouse... but we do want to
+			# extert back-pressure, which conflicts.
+			alt{
+			ptr <-= p =>
+				;
+			* =>
+				; # sys->fprint(sys->fildes(2), "9win: discarding mouse event\n");
+			}
+		}
+	}
+}
+
+serveproc(w: ref Window, mouserq: chan of Sys->Rread, fwinname, fconsctl, fcons, fmouse, fcursor: ref Sys->FileIO)
+{
+	winid := 0;
+	krc: list of Sys->Rread;
+	ks: string;
+
+	for(;;)alt {
+	c := <-w.ctxt.kbd =>
+		ks[len ks] = inf2p9key(c);
+		if(krc != nil){
+			hd krc <-= (array of byte ks, nil);
+			ks = nil;
+			krc = tl krc;
+		}
+	(nil, d, nil, wc) := <-fcons.write =>
+		if(wc != nil){
+			sys->write(sys->fildes(1), d, len d);
+			wc <-= (len d, nil);
+		}
+	(nil, nil, nil, rc) := <-fcons.read =>
+		if(rc != nil){
+			if(ks != nil){
+				rc <-= (array of byte ks, nil);
+				ks = nil;
+			}else
+				krc = rc :: krc;
+		}
+	(offset, nil, nil, rc) := <-fwinname.read =>
+		if(rc != nil){
+			if(winname == nil){
+				winname = sys->sprint("noborder.9win.%d", winid++);
+				if(w.image.name(winname, 1) == -1){
+					sys->fprint(sys->fildes(2), "9win: namewin %q failed: %r", winname);
+					rc <-= (nil, "namewin failure");
+					break;
+				}
+			}
+			d := array of byte winname;
+			if(offset < len d)
+				d = d[offset:];
+			else
+				d = nil;
+			rc <-= (d, nil);
+		}
+	(nil, nil, nil, wc) := <-fwinname.write =>
+		if(wc != nil)
+			wc <-= (-1, "permission denied");
+	(nil, nil, nil, rc) := <-fconsctl.read =>
+		if(rc != nil)
+			rc <-= (nil, "permission denied");
+	(nil, d, nil, wc) := <-fconsctl.write =>
+		if(wc != nil){
+			if(string d != "rawon")
+				wc <-= (-1, "cannot change console mode");
+			else
+				wc <-= (len d, nil);
+		}
+	(nil, nil, nil, rc) := <-fmouse.read =>
+		if(rc != nil)
+			mouserq <-= rc;
+	(nil, d, nil, wc) := <-fmouse.write =>
+		if(wc != nil){
+			e := cursorset(w, string d);
+			if(e == nil)
+				wc <-= (len d, nil);
+			else
+				wc <-= (-1, e);
+		}
+	(nil, nil, nil, rc) := <-fcursor.read =>
+		if(rc != nil)
+			rc <-= (nil, "permission denied");
+	(nil, d, nil, wc) := <-fcursor.write =>
+		if(wc != nil){
+			e := cursorswitch(w, d);
+			if(e == nil)
+				wc <-= (len d, nil);
+			else
+				wc <-= (-1, e);
+		}
+	}
+}
+
+ptrproc(rq: chan of Sys->Rread, ptr: chan of ref Pointer, reshape: chan of int)
+{
+	rl: list of Sys->Rread;
+	c := ref Pointer(0, (0, 0), 0);
+	for(;;){
+		ch: int;
+		alt{
+		p := <-ptr =>
+			ch = 'm';
+			c = p;
+		<-reshape =>
+			ch = 'r';
+		rc := <-rq =>
+			rl  = rc :: rl;
+			continue;
+		}
+		if(rl == nil)
+			rl = <-rq :: rl;
+		hd rl <-= (sys->aprint("%c%11d %11d %11d %11d ", ch, c.xy.x, c.xy.y, c.buttons, c.msec), nil);
+		rl = tl rl;
+	}
+}
+
+cursorset(w: ref Window, m: string): string
+{
+	if(m == nil || m[0] != 'm')
+		return "invalid mouse message";
+	x := int m[1:];
+	for(i := 1; i < len m; i++)
+		if(m[i] == ' '){
+			while(m[i] == ' ')
+				i++;
+			break;
+		}
+	if(i == len m)
+		return "invalid mouse message";
+	y := int m[i:];
+	return w.wmctl(sys->sprint("ptr %d %d", x, y));
+}
+
+cursorswitch(w: ref Window, d: array of byte): string
+{
+	Hex: con "0123456789abcdef";
+	if(len d != 2*4+64)
+		return w.wmctl("cursor");
+	hot := Draw->Point(bglong(d, 0*4), bglong(d, 1*4));
+	s := sys->sprint("cursor %d %d 16 32 ", hot.x, hot.y);
+	for(i := 2*4; i < len d; i++){
+		c := int d[i];
+		s[len s] = Hex[c >> 4];
+		s[len s] = Hex[c & 16rf];
+	}
+	return w.wmctl(s);
+}
+
+run(sync, ctl: chan of string, ld: string, argv: list of string)
+{
+	Rcmeta: con "|<>&^*[]?();";
+	sys->pctl(Sys->FORKNS, nil);
+	if(sys->bind("#₪", "/srv", Sys->MCREATE) == -1){
+		sync <-= sys->sprint("cannot bind srv device: %r");
+		exit;
+	}
+	srvname := "/srv/9win."+string sys->pctl(0, nil);	# XXX do better.
+	fd := sys->create(srvname, Sys->ORDWR, 8r600);
+	if(fd == nil){
+		sync <-= sys->sprint("cannot create %s: %r", srvname);
+		exit;
+	}
+	sync <-= nil;
+	spawn export(fd, ctl);
+	sh->run(nil, "os" ::
+		"rc" :: "-c" ::
+			"mount "+srvname+" /mnt/term;"+
+			"rm "+srvname+";"+
+			"bind -b /mnt/term"+ld+" /dev;"+
+			"bind /mnt/term/dev/draw /dev/draw ||"+
+				"bind -a /mnt/term/dev /dev;"+
+			quotedc("cd"::"/mnt/term"+cwd()::nil, Rcmeta)+";"+
+			quotedc(argv, Rcmeta)+";"::
+			nil
+		);
+}
+
+export(fd: ref Sys->FD, ctl: chan of string)
+{
+	sys->export(fd, "/", Sys->EXPWAIT);
+	ctl <-= "exit";
+}
+
+inf2p9key(c: int): int
+{
+	KF: import Keyboard;
+
+	P9KF: con	16rF000;
+	Spec: con	16rF800;
+	Khome: con	P9KF|16r0D;
+	Kup: con	P9KF|16r0E;
+	Kpgup: con	P9KF|16r0F;
+	Kprint: con	P9KF|16r10;
+	Kleft: con	P9KF|16r11;
+	Kright: con	P9KF|16r12;
+	Kdown: con	Spec|16r00;
+	Kview: con	Spec|16r00;
+	Kpgdown: con	P9KF|16r13;
+	Kins: con	P9KF|16r14;
+	Kend: con	P9KF|16r18;
+	Kalt: con		P9KF|16r15;
+	Kshift: con	P9KF|16r16;
+	Kctl: con		P9KF|16r17;
+
+	case c {
+	Keyboard->LShift =>
+		return Kshift;
+	Keyboard->LCtrl =>
+		return Kctl;
+	Keyboard->LAlt =>
+		return Kalt;
+	Keyboard->Home =>
+		return Khome;
+	Keyboard->End =>
+		return Kend;
+	Keyboard->Up =>
+		return Kup;
+	Keyboard->Down =>
+		return Kdown;
+	Keyboard->Left =>
+		return Kleft;
+	Keyboard->Right =>
+		return Kright;
+	Keyboard->Pgup =>
+		return Kpgup;
+	Keyboard->Pgdown =>
+		return Kpgdown;
+	Keyboard->Ins =>
+		return Kins;
+
+	# function keys
+	KF|1 or
+	KF|2 or
+	KF|3 or
+	KF|4 or
+	KF|5 or
+	KF|6 or
+	KF|7 or
+	KF|8 or
+	KF|9 or
+	KF|10 or
+	KF|11 or
+	KF|12 =>
+		return (c - KF) + P9KF;
+	}
+	return c;
+}
+
+cwd(): string
+{
+	return sys->fd2path(sys->open(".", Sys->OREAD));
+}
+
+# from string.b, waiting for declaration to be uncommented.
+quotedc(argv: list of string, cl: string): string
+{
+	s := "";
+	while (argv != nil) {
+		arg := hd argv;
+		for (i := 0; i < len arg; i++) {
+			c := arg[i];
+			if (c == ' ' || c == '\t' || c == '\n' || c == '\'' || in(c, cl))
+				break;
+		}
+		if (i < len arg || arg == nil) {
+			s += "'" + arg[0:i];
+			for (; i < len arg; i++) {
+				if (arg[i] == '\'')
+					s[len s] = '\'';
+				s[len s] = arg[i];
+			}
+			s[len s] = '\'';
+		} else
+			s += arg;
+		if (tl argv != nil)
+			s[len s] = ' ';
+		argv = tl argv;
+	}
+	return s;
+}
+
+in(c: int, s: string): int
+{
+	n := len s;
+	if(n == 0)
+		return 0;
+	ans := 0;
+	negate := 0;
+	if(s[0] == '^') {
+		negate = 1;
+		s = s[1:];
+		n--;
+	}
+	for(i := 0; i < n; i++) {
+		if(s[i] == '-' && i > 0 && i < n-1)  {
+			if(c >= s[i-1] && c <= s[i+1]) {
+				ans = 1;
+				break;
+			}
+			i++;
+		}
+		else
+		if(c == s[i]) {
+			ans = 1;
+			break;
+		}
+	}
+	if(negate)
+		ans = !ans;
+
+	# just to showcase labels
+skip:
+	return ans;
+}
+
+bglong(d: array of byte, i: int): int
+{
+	return int d[i] | (int d[i+1]<<8) | (int d[i+2]<<16) | (int d[i+3]<<24);
+}
diff --git a/tests/examplefiles/objc_example.m b/tests/examplefiles/objc_example.m
index 67b33022..f4f27170 100644
--- a/tests/examplefiles/objc_example.m
+++ b/tests/examplefiles/objc_example.m
@@ -30,3 +30,6 @@ NSDictionary *d = @{ @"key": @"value" };
 
 NSNumber *n1 = @( 1 );
 NSNumber *n2 = @( [a length] );
+
++ (void)f1:(NSString *)s1;
++ (void)f2:(NSString *) s2;
diff --git a/tests/examplefiles/rql-queries.rql b/tests/examplefiles/rql-queries.rql
new file mode 100644
index 00000000..1d86df3c
--- /dev/null
+++ b/tests/examplefiles/rql-queries.rql
@@ -0,0 +1,34 @@
+Any N, N2 where N is Note, N2 is Note, N a_faire_par P1, P1 nom 'john', N2 a_faire_par P2, P2 nom 'jane' ;
+DISTINCT Any N, D, C, T, A ORDERBY D DESC LIMIT 40 where N is Note, N diem D, W is Workcase, W concerned_by N, N cost C, N text T, N author A, N diem <= today
+Bookmark B WHERE B owned_by G, G eid 5;
+Any X WHERE E eid 22762, NOT E is_in X, X modification_date D ORDERBY D DESC LIMIT 41;
+Any A, R, SUB ORDERBY R WHERE A is "Workcase", S is Division, S concerned_by A, A subject SUB, S eid 85, A ref R;
+Any D, T, L WHERE D is Document, A concerned_by D,A eid 14533, D title T, D location L;
+Any N,A,B,C,D ORDERBY A DESC WHERE N is Note, W concerned_by N, W eid 14533, N diem A,N author B,N text C,N cost D;
+Any X ORDERBY D DESC LIMIT 41 WHERE E eid 18134, NOT E concerned_by X, X modification_date D
+DISTINCT Any N, D, C, T, A ORDERBY D ASC LIMIT 40 WHERE N is Note, N diem D, P is Person, N to_be_contacted_by G, N cost C, N text T, N author A, G login "john";
+INSERT Person X: X surname "Doe", X firstname "John";
+Workcase W where W ref "ABCD12";
+Workcase W where W ref LIKE "AB%";
+Any X WHERE X X eid 53
+Any X WHERE X Document X occurence_of F, F class C, C name 'Comics' X owned_by U, U login 'syt' X available true
+Person P WHERE P work_for P, S name 'Acme', P interested_by T, T name 'training'
+Note N WHERE N written_on D, D day> (today -10), N written_by P, P name 'joe' or P name 'jack'
+Person P WHERE (P interested_by T, T name 'training') or (P city 'Paris')
+Any N, P WHERE X is Person, X name N, X first_name P
+String N, P WHERE X is Person, X name N, X first_name P
+INSERT Person X: X name 'widget'
+INSERT Person X, Person Y: X name 'foo', Y name 'nice', X friend Y
+INSERT Person X: X name 'foo', X friend  Y WHERE name 'nice'
+SET X name 'bar', X first_name 'original' where X is Person X name 'foo'
+SET X know Y  WHERE X friend Y
+DELETE Person X WHERE X name 'foo'
+DELETE X friend Y WHERE X is Person, X name 'foo'
+Any X WHERE X name LIKE '%lt'
+Any X WHERE X name IN ( 'joe', 'jack', 'william', 'averell')
+Any X, V WHERE X concerns P, P eid 42, X corrected_in V?
+Any C, P WHERE C is Card, P? documented_by C
+Point P where P abs X, P ord Y, P value X+Y
+Document X where X class C, C name 'Cartoon', X owned_by U, U login 'joe', X available true
+(Any X WHERE X is Document) UNION (Any X WHERE X is File)
+Any A,B WHERE A creation_date B WITH A BEING (Any X WHERE X is Document) UNION (Any X WHERE X is File)
diff --git a/tests/examplefiles/sparql.rq b/tests/examplefiles/sparql.rq
new file mode 100644
index 00000000..caedfd14
--- /dev/null
+++ b/tests/examplefiles/sparql.rq
@@ -0,0 +1,23 @@
+# This is a test SPARQL query
+
+PREFIX foaf: <http://xmlns.com/foaf/0.1/>
+PREFIX ex: <http://example.org/>
+PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+PREFIX dcterms: <http://purl.org/dc/terms/>
+
+SELECT ?person (COUNT(?nick) AS ?nickCount) {
+    ?person foaf:nick ?nick ;
+      foaf:lastName "Smith" ;
+      foaf:age "21"^^xsd:int ;
+      ex:title 'Mr' ; # single-quoted string
+      ex:height 1.80 ; # float
+      ex:distanceToSun +1.4e8 ; # float with exponent
+      ex:ownsACat true ;
+      dcterms:description "Someone with a cat called \"cat\"."@en .
+    OPTIONAL { ?person foaf:isPrimaryTopicOf ?page }
+    OPTIONAL { ?person foaf:name ?name
+               { ?person foaf:depiction ?img }
+               UNION
+               { ?person foaf:firstName ?firstN } }
+    FILTER ( bound(?page) || bound(?img) || bound(?firstN) )
+} GROUP BY ?person ORDER BY ?img 
diff --git a/tests/examplefiles/test.R b/tests/examplefiles/test.R
index 54325339..1dd8f64b 100644
--- a/tests/examplefiles/test.R
+++ b/tests/examplefiles/test.R
@@ -33,10 +33,11 @@ NA_foo_ <- NULL
 123456.78901
 123e3
 123E3
-1.23e-3
-1.23e3
-1.23e-3
-## integer constants
+6.02e23	
+1.6e-35	
+1.E12
+.1234
+## integers
 123L
 1.23L
 ## imaginary numbers
@@ -80,7 +81,7 @@ repeat {1+1}
 ## Switch
 x <- 3
 switch(x, 2+2, mean(1:10), rnorm(5))
-## Function, dot-dot-dot, return
+## Function, dot-dot-dot, return, sum
 foo <- function(...) {
     return(sum(...))
 }
@@ -151,3 +152,34 @@ world!'
 
 ## Backtick strings
 `foo123 +!"bar'baz` <- 2 + 2
+
+## Builtin funcitons
+file.create()
+gamma()
+grep()
+paste()
+rbind()
+rownames()
+R.Version()
+R.version.string()
+sample()
+sapply()
+save.image()
+seq()
+setwd()
+sin()
+
+## Data structures
+servo <- matrix(1:25, nrow = 5)
+numeric()
+vector(servo)
+data.frame()
+list1 <- list(time = 1:40)
+# multidimensional array 
+array(c(c(c(2,300,4),c(8,9,0)),c(c(5,60,0),c(66,7,847))), dim=c(3,2,2))
+
+## Namespace
+library(ggplot2)
+require(plyr)
+attach(cars)
+source("test.R")
diff --git a/tests/examplefiles/test.apl b/tests/examplefiles/test.apl
new file mode 100644
index 00000000..26ecf971
--- /dev/null
+++ b/tests/examplefiles/test.apl
@@ -0,0 +1,26 @@
+∇ R←M COMBIN N;D;E;F;G;P
+  ⍝ Returns a matrix of every possible
+  ⍝ combination of M elements from the
+  ⍝ vector ⍳N.  That is, returns a
+  ⍝ matrix with M!N rows and N columns.
+  ⍝
+  E←(⍳P←N-R←M-1)-⎕IO
+  D←R+⍳P
+  R←(P,1)⍴D
+  P←P⍴1
+ L1:→(⎕IO>1↑D←D-1)⍴0
+  P←+\P
+  G←+\¯1↓0,F←⌽P
+  E←F/E-G
+  R←(F/D),R[E+⍳⍴E;]
+  E←G
+  →L1
+∇
+
+∇ R←M QUICKEXP N
+  ⍝ Matrix exponentiation
+  B ← ⌊ 1 + 2 ⍟ N
+  V ← (B ⍴ 2) ⊤ N
+  L ← ⊂ M
+  R ← ⊃ +.× / V / L ⊣ { L ← (⊂ A +.× A ← ↑L) , L }¨ ⍳ B-1
+∇
diff --git a/tests/examplefiles/test.php b/tests/examplefiles/test.php
index 97e21f73..218892fe 100644
--- a/tests/examplefiles/test.php
+++ b/tests/examplefiles/test.php
@@ -1,5 +1,7 @@
 <?php
 
+$disapproval_ಠ_ಠ_of_php = 'unicode var';
+
 $test = function($a) { $lambda = 1; }
 
 /**
@@ -16,7 +18,7 @@ if(!defined('UNLOCK') || !UNLOCK)
 // Load the parent archive class
 require_once(ROOT_PATH.'/classes/archive.class.php');
 
-class Zip\Zipp {
+class Zip\Zippಠ_ಠ_ {
 
 }
 
diff --git a/tests/examplefiles/test.pig b/tests/examplefiles/test.pig
new file mode 100644
index 00000000..f67b0268
--- /dev/null
+++ b/tests/examplefiles/test.pig
@@ -0,0 +1,148 @@
+/**
+ *  This script is an example recommender (using made up data) showing how you might modify item-item links
+ *  by defining similar relations between items in a dataset and customizing the change in weighting.
+ *  This example creates metadata by using the genre field as the metadata_field.  The items with
+ *  the same genre have it's weight cut in half in order to boost the signals of movies that do not have the same genre.
+ *  This technique requires a customization of the standard GetItemItemRecommendations macro
+ */
+import 'recommenders.pig';
+
+
+
+%default INPUT_PATH_PURCHASES '../data/retail/purchases.json'
+%default INPUT_PATH_WISHLIST '../data/retail/wishlists.json'
+%default INPUT_PATH_INVENTORY '../data/retail/inventory.json'
+%default OUTPUT_PATH '../data/retail/out/modify_item_item'
+
+
+/******** Custom GetItemItemRecommnedations *********/
+define recsys__GetItemItemRecommendations_ModifyCustom(user_item_signals, metadata) returns item_item_recs {
+
+    -- Convert user_item_signals to an item_item_graph
+    ii_links_raw, item_weights   =   recsys__BuildItemItemGraph(
+                                       $user_item_signals,
+                                       $LOGISTIC_PARAM,
+                                       $MIN_LINK_WEIGHT,
+                                       $MAX_LINKS_PER_USER
+                                     );
+    -- NOTE this function is added in order to combine metadata with item-item links
+        -- See macro for more detailed explination
+    ii_links_metadata           =   recsys__AddMetadataToItemItemLinks(
+                                        ii_links_raw,
+                                        $metadata
+                                    );
+
+    /********* Custom Code starts here ********/
+
+    --The code here should adjust the weights based on an item-item link and the equality of metadata.
+    -- In this case, if the metadata is the same, the weight is reduced.  Otherwise the weight is left alone.
+    ii_links_adjusted           =  foreach ii_links_metadata generate item_A, item_B,
+                                        -- the amount of weight adjusted is dependant on the domain of data and what is expected
+                                        -- It is always best to adjust the weight by multiplying it by a factor rather than addition with a constant
+                                        (metadata_B == metadata_A ? (weight * 0.5): weight) as weight;
+
+
+    /******** Custom Code stops here *********/
+
+    -- remove negative numbers just incase
+    ii_links_adjusted_filt = foreach ii_links_adjusted generate item_A, item_B,
+                                      (weight <= 0 ? 0: weight) as weight;
+    -- Adjust the weights of the graph to improve recommendations.
+    ii_links                    =   recsys__AdjustItemItemGraphWeight(
+                                        ii_links_adjusted_filt,
+                                        item_weights,
+                                        $BAYESIAN_PRIOR
+                                    );
+
+    -- Use the item-item graph to create item-item recommendations.
+    $item_item_recs =  recsys__BuildItemItemRecommendationsFromGraph(
+                           ii_links,
+                           $NUM_RECS_PER_ITEM,
+                           $NUM_RECS_PER_ITEM
+                       );
+};
+
+
+/******* Load Data **********/
+
+--Get purchase signals
+purchase_input = load '$INPUT_PATH_PURCHASES' using org.apache.pig.piggybank.storage.JsonLoader(
+                    'row_id: int,
+                     movie_id: chararray,
+                     movie_name: chararray,
+                     user_id: chararray,
+                     purchase_price: int');
+
+--Get wishlist signals
+wishlist_input =  load '$INPUT_PATH_WISHLIST' using org.apache.pig.piggybank.storage.JsonLoader(
+                     'row_id: int,
+                      movie_id: chararray,
+                      movie_name: chararray,
+                      user_id: chararray');
+
+
+/******* Convert Data to Signals **********/
+
+-- Start with choosing 1 as max weight for a signal.
+purchase_signals = foreach purchase_input generate
+                        user_id    as user,
+                        movie_name as item,
+                        1.0        as weight;
+
+
+-- Start with choosing 0.5 as weight for wishlist items because that is a weaker signal than
+-- purchasing an item.
+wishlist_signals = foreach wishlist_input generate
+                        user_id    as user,
+                        movie_name as item,
+                        0.5        as weight;
+
+user_signals = union purchase_signals, wishlist_signals;
+
+
+/******** Changes for Modifying item-item links ******/
+inventory_input = load '$INPUT_PATH_INVENTORY' using org.apache.pig.piggybank.storage.JsonLoader(
+                     'movie_title: chararray,
+                      genres: bag{tuple(content:chararray)}');
+
+
+metadata = foreach inventory_input generate
+              FLATTEN(genres) as metadata_field,
+              movie_title as item;
+-- requires the macro to be written seperately
+  --NOTE this macro is defined within this file for clarity
+item_item_recs = recsys__GetItemItemRecommendations_ModifyCustom(user_signals, metadata);
+/******* No more changes ********/
+
+
+user_item_recs = recsys__GetUserItemRecommendations(user_signals, item_item_recs);
+
+--Completely unrelated code stuck in the middle
+data        =    LOAD 's3n://my-s3-bucket/path/to/responses'
+                 USING org.apache.pig.piggybank.storage.JsonLoader();
+responses   =    FOREACH data GENERATE object#'response' AS response: map[];
+out         =    FOREACH responses
+                 GENERATE response#'id' AS id: int, response#'thread' AS thread: chararray,
+                          response#'comments' AS comments: {t: (comment: chararray)};
+STORE out INTO 's3n://path/to/output' USING PigStorage('|');
+
+
+/******* Store recommendations **********/
+
+--  If your output folder exists already, hadoop will refuse to write data to it.
+
+rmf $OUTPUT_PATH/item_item_recs;
+rmf $OUTPUT_PATH/user_item_recs;
+
+store item_item_recs into '$OUTPUT_PATH/item_item_recs' using PigStorage();
+store user_item_recs into '$OUTPUT_PATH/user_item_recs' using PigStorage();
+
+-- STORE the item_item_recs into dynamo
+STORE item_item_recs
+ INTO '$OUTPUT_PATH/unused-ii-table-data'
+USING com.mortardata.pig.storage.DynamoDBStorage('$II_TABLE', '$AWS_ACCESS_KEY_ID', '$AWS_SECRET_ACCESS_KEY');
+
+-- STORE the user_item_recs into dynamo
+STORE user_item_recs
+ INTO '$OUTPUT_PATH/unused-ui-table-data'
+USING com.mortardata.pig.storage.DynamoDBStorage('$UI_TABLE', '$AWS_ACCESS_KEY_ID', '$AWS_SECRET_ACCESS_KEY');
diff --git a/tests/examplefiles/test.pwn b/tests/examplefiles/test.pwn
new file mode 100644
index 00000000..d6468617
--- /dev/null
+++ b/tests/examplefiles/test.pwn
@@ -0,0 +1,253 @@
+#include <core>
+
+// Single line comment
+/* Multi line
+   comment */
+
+/// documentation
+/**
+
+	documentation multi line
+	
+**/
+
+public OnGameModeInit() {
+    printf("Hello, World!");
+}
+
+enum info {
+	Float:ex;
+	exa,
+	exam[5],
+}
+new arr[5][info];
+
+stock Float:test_func()
+{
+	new a = 5, Float:b = 10.3;
+	if (a == b) {
+		
+	} else {
+		
+	}
+	
+	for (new i = 0; i < 10; i++) {
+		continue;
+	}
+	
+	do {
+		a--;
+	} while (a > 0);
+	
+	while (a < 5) {
+		a++;
+		break;
+	}
+	
+	switch (a) {
+		case 0: {
+		}
+		case 0..4: {
+		}
+		case 5, 6: {
+		}
+	}
+	
+	static x;
+	new xx = a > 5 ? 5 : 0;
+	new array[sizeof arr] = {0};
+	tagof a;
+	state a;
+	goto label;
+	new byte[2 char];
+	byte{0} = 'a';
+	
+	return (float(a) + b);
+}
+
+
+// float.inc
+/* Float arithmetic
+ *
+ * (c) Copyright 1999, Artran, Inc.
+ * Written by Greg Garner (gmg@artran.com)
+ * Modified in March 2001 to include user defined
+ * operators for the floating point functions.
+ *
+ * This file is provided as is (no warranties).
+ */
+#if defined _Float_included
+  #endinput
+#endif
+#define _Float_included
+#pragma library Float
+
+/* Different methods of rounding */
+enum floatround_method {
+  floatround_round,
+  floatround_floor,
+  floatround_ceil,
+  floatround_tozero,
+  floatround_unbiased
+}
+enum anglemode {
+  radian,
+  degrees,
+  grades
+}
+
+/**************************************************/
+/* Convert an integer into a floating point value */
+native Float:float(value);
+
+/**************************************************/
+/* Convert a string into a floating point value */
+native Float:floatstr(const string[]);
+
+/**************************************************/
+/* Multiple two floats together */
+native Float:floatmul(Float:oper1, Float:oper2);
+
+/**************************************************/
+/* Divide the dividend float by the divisor float */
+native Float:floatdiv(Float:dividend, Float:divisor);
+
+/**************************************************/
+/* Add two floats together */
+native Float:floatadd(Float:oper1, Float:oper2);
+
+/**************************************************/
+/* Subtract oper2 float from oper1 float */
+native Float:floatsub(Float:oper1, Float:oper2);
+
+/**************************************************/
+/* Return the fractional part of a float */
+native Float:floatfract(Float:value);
+
+/**************************************************/
+/* Round a float into a integer value */
+native floatround(Float:value, floatround_method:method=floatround_round);
+
+/**************************************************/
+/* Compare two integers. If the two elements are equal, return 0.
+   If the first argument is greater than the second argument, return 1,
+   If the first argument is less than the second argument, return -1. */
+native floatcmp(Float:oper1, Float:oper2);
+
+/**************************************************/
+/* Return the square root of the input value, same as floatpower(value, 0.5) */
+native Float:floatsqroot(Float:value);
+
+/**************************************************/
+/* Return the value raised to the power of the exponent */
+native Float:floatpower(Float:value, Float:exponent);
+
+/**************************************************/
+/* Return the logarithm */
+native Float:floatlog(Float:value, Float:base=10.0);
+
+/**************************************************/
+/* Return the sine, cosine or tangent. The input angle may be in radian,
+   degrees or grades. */
+native Float:floatsin(Float:value, anglemode:mode=radian);
+native Float:floatcos(Float:value, anglemode:mode=radian);
+native Float:floattan(Float:value, anglemode:mode=radian);
+
+/**************************************************/
+/* Return the absolute value */
+native Float:floatabs(Float:value);
+
+
+/**************************************************/
+#pragma rational Float
+
+/* user defined operators */
+native Float:operator*(Float:oper1, Float:oper2) = floatmul;
+native Float:operator/(Float:oper1, Float:oper2) = floatdiv;
+native Float:operator+(Float:oper1, Float:oper2) = floatadd;
+native Float:operator-(Float:oper1, Float:oper2) = floatsub;
+native Float:operator=(oper) = float;
+
+stock Float:operator++(Float:oper)
+    return oper+1.0;
+
+stock Float:operator--(Float:oper)
+    return oper-1.0;
+
+stock Float:operator-(Float:oper)
+    return oper^Float:cellmin;                  /* IEEE values are sign/magnitude */
+
+stock Float:operator*(Float:oper1, oper2)
+    return floatmul(oper1, float(oper2));       /* "*" is commutative */
+
+stock Float:operator/(Float:oper1, oper2)
+    return floatdiv(oper1, float(oper2));
+
+stock Float:operator/(oper1, Float:oper2)
+    return floatdiv(float(oper1), oper2);
+
+stock Float:operator+(Float:oper1, oper2)
+    return floatadd(oper1, float(oper2));       /* "+" is commutative */
+
+stock Float:operator-(Float:oper1, oper2)
+    return floatsub(oper1, float(oper2));
+
+stock Float:operator-(oper1, Float:oper2)
+    return floatsub(float(oper1), oper2);
+
+stock bool:operator==(Float:oper1, Float:oper2)
+    return floatcmp(oper1, oper2) == 0;
+
+stock bool:operator==(Float:oper1, oper2)
+    return floatcmp(oper1, float(oper2)) == 0;  /* "==" is commutative */
+
+stock bool:operator!=(Float:oper1, Float:oper2)
+    return floatcmp(oper1, oper2) != 0;
+
+stock bool:operator!=(Float:oper1, oper2)
+    return floatcmp(oper1, float(oper2)) != 0;  /* "!=" is commutative */
+
+stock bool:operator>(Float:oper1, Float:oper2)
+    return floatcmp(oper1, oper2) > 0;
+
+stock bool:operator>(Float:oper1, oper2)
+    return floatcmp(oper1, float(oper2)) > 0;
+
+stock bool:operator>(oper1, Float:oper2)
+    return floatcmp(float(oper1), oper2) > 0;
+
+stock bool:operator>=(Float:oper1, Float:oper2)
+    return floatcmp(oper1, oper2) >= 0;
+
+stock bool:operator>=(Float:oper1, oper2)
+    return floatcmp(oper1, float(oper2)) >= 0;
+
+stock bool:operator>=(oper1, Float:oper2)
+    return floatcmp(float(oper1), oper2) >= 0;
+
+stock bool:operator<(Float:oper1, Float:oper2)
+    return floatcmp(oper1, oper2) < 0;
+
+stock bool:operator<(Float:oper1, oper2)
+    return floatcmp(oper1, float(oper2)) < 0;
+
+stock bool:operator<(oper1, Float:oper2)
+    return floatcmp(float(oper1), oper2) < 0;
+
+stock bool:operator<=(Float:oper1, Float:oper2)
+    return floatcmp(oper1, oper2) <= 0;
+
+stock bool:operator<=(Float:oper1, oper2)
+    return floatcmp(oper1, float(oper2)) <= 0;
+
+stock bool:operator<=(oper1, Float:oper2)
+    return floatcmp(float(oper1), oper2) <= 0;
+
+stock bool:operator!(Float:oper)
+    return (_:oper & cellmax) == 0;
+
+/* forbidden operations */
+forward operator%(Float:oper1, Float:oper2);
+forward operator%(Float:oper1, oper2);
+forward operator%(oper1, Float:oper2);
+
diff --git a/tests/examplefiles/test.zep b/tests/examplefiles/test.zep
new file mode 100644
index 00000000..4724d4c4
--- /dev/null
+++ b/tests/examplefiles/test.zep
@@ -0,0 +1,33 @@
+namespace Test;
+
+use Test\Foo;
+
+class Bar
+{
+    protected a;
+    private b;
+    public c {set, get};
+
+    public function __construct(string str, boolean bool)
+    {
+        let this->c = str;
+        this->setC(bool);
+        let this->b = [];
+    }
+
+    public function sayHello(string name)
+    {
+        echo "Hello " . name;
+    }
+
+    protected function loops()
+    {
+        for a in b {
+            echo a;
+        }
+        loop {
+            return "boo!";
+        }
+    }
+
+}
\ No newline at end of file
diff --git a/tests/examplefiles/vctreestatus_hg b/tests/examplefiles/vctreestatus_hg
new file mode 100644
index 00000000..193ed803
--- /dev/null
+++ b/tests/examplefiles/vctreestatus_hg
@@ -0,0 +1,4 @@
+M LICENSE
+M setup.py
+! setup.cfg
+? vctreestatus_hg