summaryrefslogtreecommitdiff
path: root/tests
diff options
context:
space:
mode:
Diffstat (limited to 'tests')
-rw-r--r--tests/examplefiles/classes.dylan16
-rw-r--r--tests/examplefiles/example.moon629
-rw-r--r--tests/examplefiles/example.snobol15
-rw-r--r--tests/examplefiles/http_request_example14
-rw-r--r--tests/examplefiles/http_response_example27
-rw-r--r--tests/examplefiles/nemerle_sample.n6
-rw-r--r--tests/examplefiles/test.cs23
-rw-r--r--tests/test_basic_api.py2
-rw-r--r--tests/test_examplefiles.py5
-rw-r--r--tests/test_html_formatter.py32
-rw-r--r--tests/test_perllexer.py137
-rw-r--r--tests/test_util.py38
12 files changed, 935 insertions, 9 deletions
diff --git a/tests/examplefiles/classes.dylan b/tests/examplefiles/classes.dylan
index ff435b77..6dd55ff2 100644
--- a/tests/examplefiles/classes.dylan
+++ b/tests/examplefiles/classes.dylan
@@ -22,3 +22,19 @@ end function;
define constant $blue-car = make(<car>, model: "Viper");
define constant $black-car = make(<car>, model: "Town Car", sunroof?: #t);
define constant $red-car = make(<car>, model: "F40", sunroof?: #f);
+
+define method foo() => _ :: <boolean>
+ #t
+end method;
+
+define method foo() => _ :: <boolean>;
+ #t
+end method;
+
+define method \+()
+end;
+
+define constant $symbol = #"hello";
+define variable *vector* = #[3.5, 5]
+define constant $list = #(1, 2);
+define constant $pair = #(1 . "foo")
diff --git a/tests/examplefiles/example.moon b/tests/examplefiles/example.moon
new file mode 100644
index 00000000..d4415e32
--- /dev/null
+++ b/tests/examplefiles/example.moon
@@ -0,0 +1,629 @@
+-- transform.moon
+-- Leaf Corcoran (leafot@gmail.com) 2011
+--
+-- This is part of the MoonScript compiler. See <http://moonscript.org>
+-- MoonScript is licensed under the MIT License
+--
+
+module "moonscript.transform", package.seeall
+
+types = require "moonscript.types"
+util = require "moonscript.util"
+data = require "moonscript.data"
+
+import reversed from util
+import ntype, build, smart_node, is_slice from types
+import insert from table
+
+export Statement, Value, NameProxy, LocalName, Run
+
+-- always declares as local
+class LocalName
+ new: (@name) => self[1] = "temp_name"
+ get_name: => @name
+
+class NameProxy
+ new: (@prefix) =>
+ self[1] = "temp_name"
+
+ get_name: (scope) =>
+ if not @name
+ @name = scope\free_name @prefix, true
+ @name
+
+ chain: (...) =>
+ items = {...} -- todo: fix ... propagation
+ items = for i in *items
+ if type(i) == "string"
+ {"dot", i}
+ else
+ i
+
+ build.chain {
+ base: self
+ unpack items
+ }
+
+ index: (key) =>
+ build.chain {
+ base: self, {"index", key}
+ }
+
+ __tostring: =>
+ if @name
+ ("name<%s>")\format @name
+ else
+ ("name<prefix(%s)>")\format @prefix
+
+class Run
+ new: (@fn) =>
+ self[1] = "run"
+
+ call: (state) =>
+ self.fn state
+
+-- transform the last stm is a list of stms
+-- will puke on group
+apply_to_last = (stms, fn) ->
+ -- find last (real) exp
+ last_exp_id = 0
+ for i = #stms, 1, -1
+ stm = stms[i]
+ if stm and util.moon.type(stm) != Run
+ last_exp_id = i
+ break
+
+ return for i, stm in ipairs stms
+ if i == last_exp_id
+ fn stm
+ else
+ stm
+
+-- is a body a sindle expression/statement
+is_singular = (body) ->
+ return false if #body != 1
+ if "group" == ntype body
+ is_singular body[2]
+ else
+ true
+
+constructor_name = "new"
+
+class Transformer
+ new: (@transformers, @scope) =>
+ @seen_nodes = {}
+
+ transform: (scope, node, ...) =>
+ -- print scope, node, ...
+ return node if @seen_nodes[node]
+ @seen_nodes[node] = true
+ while true
+ transformer = @transformers[ntype node]
+ res = if transformer
+ transformer(scope, node, ...) or node
+ else
+ node
+ return node if res == node
+ node = res
+
+ __call: (node, ...) =>
+ @transform @scope, node, ...
+
+ instance: (scope) =>
+ Transformer @transformers, scope
+
+ can_transform: (node) =>
+ @transformers[ntype node] != nil
+
+construct_comprehension = (inner, clauses) ->
+ current_stms = inner
+ for _, clause in reversed clauses
+ t = clause[1]
+ current_stms = if t == "for"
+ _, names, iter = unpack clause
+ {"foreach", names, iter, current_stms}
+ elseif t == "when"
+ _, cond = unpack clause
+ {"if", cond, current_stms}
+ else
+ error "Unknown comprehension clause: "..t
+ current_stms = {current_stms}
+
+ current_stms[1]
+
+Statement = Transformer {
+ assign: (node) =>
+ _, names, values = unpack node
+ -- bubble cascading assigns
+ if #values == 1 and types.cascading[ntype values[1]]
+ values[1] = @transform.statement values[1], (stm) ->
+ t = ntype stm
+ if types.is_value stm
+ {"assign", names, {stm}}
+ else
+ stm
+
+ build.group {
+ {"declare", names}
+ values[1]
+ }
+ else
+ node
+
+ export: (node) =>
+ -- assign values if they are included
+ if #node > 2
+ if node[2] == "class"
+ cls = smart_node node[3]
+ build.group {
+ {"export", {cls.name}}
+ cls
+ }
+ else
+ build.group {
+ node
+ build.assign {
+ names: node[2]
+ values: node[3]
+ }
+ }
+ else
+ nil
+
+ update: (node) =>
+ _, name, op, exp = unpack node
+ op_final = op\match "^(.+)=$"
+ error "Unknown op: "..op if not op_final
+ build.assign_one name, {"exp", name, op_final, exp}
+
+ import: (node) =>
+ _, names, source = unpack node
+
+ stubs = for name in *names
+ if type(name) == "table"
+ name
+ else
+ {"dot", name}
+
+ real_names = for name in *names
+ type(name) == "table" and name[2] or name
+
+ if type(source) == "string"
+ build.assign {
+ names: real_names
+ values: [build.chain { base: source, stub} for stub in *stubs]
+ }
+ else
+ source_name = NameProxy "table"
+ build.group {
+ {"declare", real_names}
+ build["do"] {
+ build.assign_one source_name, source
+ build.assign {
+ names: real_names
+ values: [build.chain { base: source_name, stub} for stub in *stubs]
+ }
+ }
+ }
+
+ comprehension: (node, action) =>
+ _, exp, clauses = unpack node
+
+ action = action or (exp) -> {exp}
+ construct_comprehension action(exp), clauses
+
+ -- handle cascading return decorator
+ if: (node, ret) =>
+ if ret
+ smart_node node
+ -- mutate all the bodies
+ node['then'] = apply_to_last node['then'], ret
+ for i = 4, #node
+ case = node[i]
+ body_idx = #node[i]
+ case[body_idx] = apply_to_last case[body_idx], ret
+ node
+
+ with: (node, ret) =>
+ _, exp, block = unpack node
+ scope_name = NameProxy "with"
+ build["do"] {
+ build.assign_one scope_name, exp
+ Run => @set "scope_var", scope_name
+ build.group block
+ if ret
+ ret scope_name
+ }
+
+ foreach: (node) =>
+ smart_node node
+ if ntype(node.iter) == "unpack"
+ list = node.iter[2]
+
+ index_name = NameProxy "index"
+ list_name = NameProxy "list"
+
+ slice_var = nil
+ bounds = if is_slice list
+ slice = list[#list]
+ table.remove list
+ table.remove slice, 1
+
+ slice[2] = if slice[2] and slice[2] != ""
+ max_tmp_name = NameProxy "max"
+ slice_var = build.assign_one max_tmp_name, slice[2]
+ {"exp", max_tmp_name, "<", 0
+ "and", {"length", list_name}, "+", max_tmp_name
+ "or", max_tmp_name }
+ else
+ {"length", list_name}
+
+ slice
+ else
+ {1, {"length", list_name}}
+
+ build.group {
+ build.assign_one list_name, list
+ slice_var
+ build["for"] {
+ name: index_name
+ bounds: bounds
+ body: {
+ {"assign", node.names, {list_name\index index_name}}
+ build.group node.body
+ }
+ }
+ }
+
+ switch: (node, ret) =>
+ _, exp, conds = unpack node
+ exp_name = NameProxy "exp"
+
+ -- convert switch conds into if statment conds
+ convert_cond = (cond) ->
+ t, case_exp, body = unpack cond
+ out = {}
+ insert out, t == "case" and "elseif" or "else"
+ if t != "else"
+ insert out, {"exp", case_exp, "==", exp_name} if t != "else"
+ else
+ body = case_exp
+
+ if ret
+ body = apply_to_last body, ret
+
+ insert out, body
+
+ out
+
+ first = true
+ if_stm = {"if"}
+ for cond in *conds
+ if_cond = convert_cond cond
+ if first
+ first = false
+ insert if_stm, if_cond[2]
+ insert if_stm, if_cond[3]
+ else
+ insert if_stm, if_cond
+
+ build.group {
+ build.assign_one exp_name, exp
+ if_stm
+ }
+
+ class: (node) =>
+ _, name, parent_val, body = unpack node
+
+ -- split apart properties and statements
+ statements = {}
+ properties = {}
+ for item in *body
+ switch item[1]
+ when "stm"
+ insert statements, item[2]
+ when "props"
+ for tuple in *item[2,]
+ insert properties, tuple
+
+ -- find constructor
+ constructor = nil
+ properties = for tuple in *properties
+ if tuple[1] == constructor_name
+ constructor = tuple[2]
+ nil
+ else
+ tuple
+
+ parent_cls_name = NameProxy "parent"
+ base_name = NameProxy "base"
+ self_name = NameProxy "self"
+ cls_name = NameProxy "class"
+
+ if not constructor
+ constructor = build.fndef {
+ args: {{"..."}}
+ arrow: "fat"
+ body: {
+ build["if"] {
+ cond: parent_cls_name
+ then: {
+ build.chain { base: "super", {"call", {"..."}} }
+ }
+ }
+ }
+ }
+ else
+ smart_node constructor
+ constructor.arrow = "fat"
+
+ cls = build.table {
+ {"__init", constructor}
+ {"__base", base_name}
+ {"__name", {"string", '"', name}} -- "quote the string"
+ {"__parent", parent_cls_name}
+ }
+
+ -- look up a name in the class object
+ class_lookup = build["if"] {
+ cond: {"exp", "val", "==", "nil", "and", parent_cls_name}
+ then: {
+ parent_cls_name\index"name"
+ }
+ }
+ insert class_lookup, {"else", {"val"}}
+
+ cls_mt = build.table {
+ {"__index", build.fndef {
+ args: {{"cls"}, {"name"}}
+ body: {
+ build.assign_one LocalName"val", build.chain {
+ base: "rawget", {"call", {base_name, "name"}}
+ }
+ class_lookup
+ }
+ }}
+ {"__call", build.fndef {
+ args: {{"cls"}, {"..."}}
+ body: {
+ build.assign_one self_name, build.chain {
+ base: "setmetatable"
+ {"call", {"{}", base_name}}
+ }
+ build.chain {
+ base: "cls.__init"
+ {"call", {self_name, "..."}}
+ }
+ self_name
+ }
+ }}
+ }
+
+ cls = build.chain {
+ base: "setmetatable"
+ {"call", {cls, cls_mt}}
+ }
+
+ value = nil
+ with build
+ value = .block_exp {
+ Run =>
+ @set "super", (block, chain) ->
+ if chain
+ slice = [item for item in *chain[3,]]
+ new_chain = {"chain", parent_cls_name}
+
+ head = slice[1]
+
+ if head == nil
+ return parent_cls_name
+
+ switch head[1]
+ -- calling super, inject calling name and self into chain
+ when "call"
+ calling_name = block\get"current_block"
+ slice[1] = {"call", {"self", unpack head[2]}}
+ act = if ntype(calling_name) != "value" then "index" else "dot"
+ insert new_chain, {act, calling_name}
+
+ -- colon call on super, replace class with self as first arg
+ when "colon"
+ call = head[3]
+ insert new_chain, {"dot", head[2]}
+ slice[1] = { "call", { "self", unpack call[2] } }
+
+ insert new_chain, item for item in *slice
+
+ new_chain
+ else
+ parent_cls_name
+
+ .assign_one parent_cls_name, parent_val == "" and "nil" or parent_val
+ .assign_one base_name, {"table", properties}
+ .assign_one base_name\chain"__index", base_name
+
+ build["if"] {
+ cond: parent_cls_name
+ then: {
+ .chain {
+ base: "setmetatable"
+ {"call", {
+ base_name,
+ .chain { base: parent_cls_name, {"dot", "__base"}}
+ }}
+ }
+ }
+ }
+
+ .assign_one cls_name, cls
+ .assign_one base_name\chain"__class", cls_name
+
+ .group if #statements > 0 {
+ .assign_one LocalName"self", cls_name
+ .group statements
+ } else {}
+
+ cls_name
+ }
+
+ value = .group {
+ .declare names: {name}
+ .assign {
+ names: {name}
+ values: {value}
+ }
+ }
+
+ value
+}
+
+class Accumulator
+ body_idx: { for: 4, while: 3, foreach: 4 }
+
+ new: =>
+ @accum_name = NameProxy "accum"
+ @value_name = NameProxy "value"
+ @len_name = NameProxy "len"
+
+ -- wraps node and mutates body
+ convert: (node) =>
+ index = @body_idx[ntype node]
+ node[index] = @mutate_body node[index]
+ @wrap node
+
+ -- wrap the node into a block_exp
+ wrap: (node) =>
+ build.block_exp {
+ build.assign_one @accum_name, build.table!
+ build.assign_one @len_name, 0
+ node
+ @accum_name
+ }
+
+ -- mutates the body of a loop construct to save last value into accumulator
+ -- can optionally skip nil results
+ mutate_body: (body, skip_nil=true) =>
+ val = if not skip_nil and is_singular body
+ with body[1]
+ body = {}
+ else
+ body = apply_to_last body, (n) ->
+ build.assign_one @value_name, n
+ @value_name
+
+ update = {
+ {"update", @len_name, "+=", 1}
+ build.assign_one @accum_name\index(@len_name), val
+ }
+
+ if skip_nil
+ table.insert body, build["if"] {
+ cond: {"exp", @value_name, "!=", "nil"}
+ then: update
+ }
+ else
+ table.insert body, build.group update
+
+ body
+
+default_accumulator = (node) =>
+ Accumulator!\convert node
+
+
+implicitly_return = (scope) ->
+ fn = (stm) ->
+ t = ntype stm
+ if types.manual_return[t] or not types.is_value stm
+ stm
+ elseif types.cascading[t]
+ scope.transform.statement stm, fn
+ else
+ if t == "comprehension" and not types.comprehension_has_value stm
+ stm
+ else
+ {"return", stm}
+
+ fn
+
+Value = Transformer {
+ for: default_accumulator
+ while: default_accumulator
+ foreach: default_accumulator
+
+ comprehension: (node) =>
+ a = Accumulator!
+ node = @transform.statement node, (exp) ->
+ a\mutate_body {exp}, false
+ a\wrap node
+
+ tblcomprehension: (node) =>
+ _, key_exp, value_exp, clauses = unpack node
+
+ accum = NameProxy "tbl"
+ dest = build.chain { base: accum, {"index", key_exp} }
+ inner = build.assign_one dest, value_exp
+
+ build.block_exp {
+ build.assign_one accum, build.table!
+ construct_comprehension {inner}, clauses
+ accum
+ }
+
+ fndef: (node) =>
+ smart_node node
+ node.body = apply_to_last node.body, implicitly_return self
+ node
+
+ if: (node) => build.block_exp { node }
+ with: (node) => build.block_exp { node }
+ switch: (node) =>
+ build.block_exp { node }
+
+ -- pull out colon chain
+ chain: (node) =>
+ stub = node[#node]
+ if type(stub) == "table" and stub[1] == "colon_stub"
+ table.remove node, #node
+
+ base_name = NameProxy "base"
+ fn_name = NameProxy "fn"
+
+ is_super = node[2] == "super"
+ @transform.value build.block_exp {
+ build.assign {
+ names: {base_name}
+ values: {node}
+ }
+
+ build.assign {
+ names: {fn_name}
+ values: {
+ build.chain { base: base_name, {"dot", stub[2]} }
+ }
+ }
+
+ build.fndef {
+ args: {{"..."}}
+ body: {
+ build.chain {
+ base: fn_name, {"call", {is_super and "self" or base_name, "..."}}
+ }
+ }
+ }
+ }
+
+ block_exp: (node) =>
+ _, body = unpack node
+
+ fn = nil
+ arg_list = {}
+
+ insert body, Run =>
+ if @has_varargs
+ insert arg_list, "..."
+ insert fn.args, {"..."}
+
+ fn = smart_node build.fndef body: body
+ build.chain { base: {"parens", fn}, {"call", arg_list} }
+}
+
diff --git a/tests/examplefiles/example.snobol b/tests/examplefiles/example.snobol
new file mode 100644
index 00000000..26ca5cf4
--- /dev/null
+++ b/tests/examplefiles/example.snobol
@@ -0,0 +1,15 @@
+-SOME RANDOM DIRECTIVE WOULD GO HERE
+*
+* SNOBOL4 example file for lexer
+*
+ SOME.THING_OR_OTHER32 = 1 + 1.0 - 1E3 * 1E-3 ** 2.718284590E0
++ :F(END)S(IN_LOOP)
+ PATTERN = LEN(3) ("GAR" | "BAR")
+IN_LOOP THING = INPUT :F(END)
+ THING LEN(3) ("GAR" | "BAR") :S(OK)
+ OUTPUT = THING " : Failure!" :(IN_LOOP)
+OK OUTPUT = THING ' : "Success"!' :(IN_LOOP)
+END
+FOOBAR
+FOOGAR
+THiNIg
diff --git a/tests/examplefiles/http_request_example b/tests/examplefiles/http_request_example
new file mode 100644
index 00000000..5d2a1d52
--- /dev/null
+++ b/tests/examplefiles/http_request_example
@@ -0,0 +1,14 @@
+POST /demo/submit/ HTTP/1.1
+Host: pygments.org
+Connection: keep-alivk
+Cache-Control: max-age=0
+Origin: http://pygments.org
+User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.63 Safari/535.7
+Content-Type: application/x-www-form-urlencoded
+Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
+Referer: http://pygments.org/
+Accept-Encoding: gzip,deflate,sdch
+Accept-Language: en-US,en;q=0.8
+Accept-Charset: windows-949,utf-8;q=0.7,*;q=0.3
+
+name=test&lang=text&code=asdf&user=
diff --git a/tests/examplefiles/http_response_example b/tests/examplefiles/http_response_example
new file mode 100644
index 00000000..bf53d61d
--- /dev/null
+++ b/tests/examplefiles/http_response_example
@@ -0,0 +1,27 @@
+HTTP/1.1 200 OK
+Date: Tue, 13 Dec 2011 00:11:44 GMT
+Status: 200 OK
+X-Transaction: 50b85fff78dab4a3
+X-RateLimit-Limit: 150
+ETag: "b31143be48ebfe7512b65fe64fe092f3"
+X-Frame-Options: SAMEORIGIN
+Last-Modified: Tue, 13 Dec 2011 00:11:44 GMT
+X-RateLimit-Remaining: 145
+X-Runtime: 0.01190
+X-Transaction-Mask: a6183ffa5f8ca943ff1b53b5644ef1145f6f285d
+Content-Type: application/json; charset=utf-8
+Content-Length: 2389
+Pragma: no-cache
+X-RateLimit-Class: api
+X-Revision: DEV
+Expires: Tue, 31 Mar 1981 05:00:00 GMT
+Cache-Control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0
+X-MID: a55f21733bc52bb11d1fc58f9b51b4974fbb8f83
+X-RateLimit-Reset: 1323738416
+Set-Cookie: k=10.34.234.116.1323735104238974; path=/; expires=Tue, 20-Dec-11 00:11:44 GMT; domain=.twitter.com
+Set-Cookie: guest_id=v1%3A13237351042425496; domain=.twitter.com; path=/; expires=Thu, 12-Dec-2013 12:11:44 GMT
+Set-Cookie: _twitter_sess=BAh7CDoPY3JlYXRlZF9hdGwrCPS6wjQ0AToHaWQiJTFiMTlhY2E1ZjczYThk%250ANDUwMWQxNjMwZGU2YTQ1ODBhIgpmbGFzaElDOidBY3Rpb25Db250cm9sbGVy%250AOjpGbGFzaDo6Rmxhc2hIYXNoewAGOgpAdXNlZHsA--6b502f30a083e8a41a64f10930e142ea362b1561; domain=.twitter.com; path=/; HttpOnly
+Vary: Accept-Encoding
+Server: tfe
+
+[{"contributors_enabled":false,"profile_background_tile":true,"followers_count":644,"protected":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/69064242\/gb_normal.jpg","screen_name":"birkenfeld","default_profile_image":false,"following":null,"friends_count":88,"profile_sidebar_fill_color":"7AC3EE","url":"http:\/\/pythonic.pocoo.org\/","name":"Georg Brandl","default_profile":false,"is_translator":false,"utc_offset":3600,"profile_sidebar_border_color":"65B0DA","description":"","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme10\/bg.gif","favourites_count":0,"profile_use_background_image":true,"created_at":"Tue Dec 30 22:25:11 +0000 2008","status":{"retweet_count":10,"favorited":false,"geo":null,"possibly_sensitive":false,"coordinates":null,"in_reply_to_screen_name":null,"in_reply_to_status_id_str":null,"retweeted":false,"in_reply_to_status_id":null,"in_reply_to_user_id_str":null,"created_at":"Sat Jul 09 13:42:35 +0000 2011","truncated":false,"id_str":"89690914515206144","contributors":null,"place":null,"source":"web","in_reply_to_user_id":null,"id":89690914515206144,"retweeted_status":{"retweet_count":10,"favorited":false,"geo":null,"possibly_sensitive":false,"coordinates":null,"in_reply_to_screen_name":null,"in_reply_to_status_id_str":null,"retweeted":false,"in_reply_to_status_id":null,"in_reply_to_user_id_str":null,"created_at":"Sat Jul 09 13:07:04 +0000 2011","truncated":false,"id_str":"89681976755372032","contributors":null,"place":null,"source":"web","in_reply_to_user_id":null,"id":89681976755372032,"text":"Excellent Python posts from @mitsuhiko - http:\/\/t.co\/k1wt6e4 and @ncoghlan_dev - http:\/\/t.co\/eTxacgZ (links fixed)"},"text":"RT @jessenoller: Excellent Python posts from @mitsuhiko - http:\/\/t.co\/k1wt6e4 and @ncoghlan_dev - http:\/\/t.co\/eTxacgZ (links fixed)"},"follow_request_sent":null,"statuses_count":553,"geo_enabled":false,"notifications":null,"profile_text_color":"3D1957","id_str":"18490730","lang":"en","profile_background_image_url":"http:\/\/a1.twimg.com\/images\/themes\/theme10\/bg.gif","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/69064242\/gb_normal.jpg","show_all_inline_media":true,"listed_count":65,"profile_link_color":"FF0000","verified":false,"id":18490730,"time_zone":"Berlin","profile_background_color":"642D8B","location":"Bavaria, Germany"}]
diff --git a/tests/examplefiles/nemerle_sample.n b/tests/examplefiles/nemerle_sample.n
index 2c05033a..5236857d 100644
--- a/tests/examplefiles/nemerle_sample.n
+++ b/tests/examplefiles/nemerle_sample.n
@@ -13,13 +13,15 @@ namespace Demo.Ns
public virtual someMethod(str : string) : list[double]
{
def x = "simple string";
- def x = $"simple $splice string $(spliceMethod())";
+ def x = $"simple $splice string $(spliceMethod() + 1)";
def x = <#
recursive <# string #> sample
#>;
def x = $<#
recursive $splice <# string #> sample
+ ..$(lst; "; "; x => $"x * 2 = $(x * 2)") str
#>;
+ def x = @"somestring \";
def localFunc(arg)
{
@@ -80,6 +82,6 @@ namespace Demo.Ns
macro sampleMacro(expr)
syntax ("write", expr)
{
- <[ WriteLine($expr) ]>
+ <[ WriteLine($(expr : dyn)) ]>
}
}
diff --git a/tests/examplefiles/test.cs b/tests/examplefiles/test.cs
index ffa9bfea..faab7e42 100644
--- a/tests/examplefiles/test.cs
+++ b/tests/examplefiles/test.cs
@@ -153,6 +153,29 @@ namespace Diva.Core {
public OpenerTask (string fileName)
{
this.fileName = fileName;
+ var verbatimString = @"c:\test\";
+
+ var verbatimStringWithNewline = @"test \\ \n \t \r
+a
+b
+c";
+ var verbatimStringWithEscapedQuotes = @"He said
+""she says \"" is not an escaped character in verbatimstrings""
+";
+
+ int[] numbers = { 5,6,4,2,4,6,8,9,7,0 };
+ var linqExample = from n in numbers
+ where n > 5
+ select n;
+
+ var anotherlinqExample = from n in numbers
+ orderby n descending
+ select n;
+
+ int[] someMoreNumbers = { 8,2,17,34,8,9,9,5,3,4,2,1,5 };
+ var moreLinq = from n in numbers
+ join mn in moreNumbers on n equals mn + 2
+ select new {n, mn};
}
public override void Reset ()
diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py
index 02261d24..680a6085 100644
--- a/tests/test_basic_api.py
+++ b/tests/test_basic_api.py
@@ -57,7 +57,7 @@ def test_lexer_classes():
assert 'root' in cls._tokens, \
'%s has no root state' % cls
- if cls.name == 'XQuery': # XXX temporary
+ if cls.name in ['XQuery', 'Opa']: # XXX temporary
return
tokens = list(inst.get_tokens(test_content))
diff --git a/tests/test_examplefiles.py b/tests/test_examplefiles.py
index e5dbcf4c..9326660c 100644
--- a/tests/test_examplefiles.py
+++ b/tests/test_examplefiles.py
@@ -60,8 +60,9 @@ def check_lexer(lx, absfn, outfn):
tokens = []
for type, val in lx.get_tokens(text):
ntext.append(val)
- assert type != Error, 'lexer %s generated error token for %s' % \
- (lx, absfn)
+ assert type != Error, \
+ 'lexer %s generated error token for %s: %r at position %d' % \
+ (lx, absfn, val, len(u''.join(ntext)))
tokens.append((type, val))
if u''.join(ntext) != text:
print '\n'.join(difflib.unified_diff(u''.join(ntext).splitlines(),
diff --git a/tests/test_html_formatter.py b/tests/test_html_formatter.py
index 5a506755..58a50c74 100644
--- a/tests/test_html_formatter.py
+++ b/tests/test_html_formatter.py
@@ -75,6 +75,38 @@ class HtmlFormatterTest(unittest.TestCase):
fmt = HtmlFormatter(**optdict)
fmt.format(tokensource, outfile)
+ def test_linenos(self):
+ optdict = dict(linenos=True)
+ outfile = StringIO.StringIO()
+ fmt = HtmlFormatter(**optdict)
+ fmt.format(tokensource, outfile)
+ html = outfile.getvalue()
+ self.assert_(re.search("<pre>\s+1\s+2\s+3", html))
+
+ def test_linenos_with_startnum(self):
+ optdict = dict(linenos=True, linenostart=5)
+ outfile = StringIO.StringIO()
+ fmt = HtmlFormatter(**optdict)
+ fmt.format(tokensource, outfile)
+ html = outfile.getvalue()
+ self.assert_(re.search("<pre>\s+5\s+6\s+7", html))
+
+ def test_lineanchors(self):
+ optdict = dict(lineanchors="foo")
+ outfile = StringIO.StringIO()
+ fmt = HtmlFormatter(**optdict)
+ fmt.format(tokensource, outfile)
+ html = outfile.getvalue()
+ self.assert_(re.search("<pre><a name=\"foo-1\">", html))
+
+ def test_lineanchors_with_startnum(self):
+ optdict = dict(lineanchors="foo", linenostart=5)
+ outfile = StringIO.StringIO()
+ fmt = HtmlFormatter(**optdict)
+ fmt.format(tokensource, outfile)
+ html = outfile.getvalue()
+ self.assert_(re.search("<pre><a name=\"foo-5\">", html))
+
def test_valid_output(self):
# test all available wrappers
fmt = HtmlFormatter(full=True, linenos=True, noclasses=True,
diff --git a/tests/test_perllexer.py b/tests/test_perllexer.py
new file mode 100644
index 00000000..4f99af6b
--- /dev/null
+++ b/tests/test_perllexer.py
@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+"""
+ Pygments regex lexer tests
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ :copyright: Copyright 2011 by the Pygments team, see AUTHORS.
+ :license: BSD, see LICENSE for details.
+"""
+
+import time
+import unittest
+
+from pygments.token import String
+from pygments.lexers.agile import PerlLexer
+
+
+class RunawayRegexTest(unittest.TestCase):
+ # A previous version of the Perl lexer would spend a great deal of
+ # time backtracking when given particular strings. These tests show that
+ # the runaway backtracking doesn't happen any more (at least for the given
+ # cases).
+
+ lexer = PerlLexer()
+
+ ### Test helpers.
+
+ def assert_single_token(self, s, token):
+ """Show that a given string generates only one token."""
+ tokens = list(self.lexer.get_tokens_unprocessed(s))
+ self.assertEqual(len(tokens), 1, tokens)
+ self.assertEqual(s, tokens[0][2])
+ self.assertEqual(token, tokens[0][1])
+
+ def assert_tokens(self, strings, expected_tokens):
+ """Show that a given string generates the expected tokens."""
+ tokens = list(self.lexer.get_tokens_unprocessed(''.join(strings)))
+ self.assertEqual(len(tokens), len(expected_tokens), tokens)
+ for index, s in enumerate(strings):
+ self.assertEqual(s, tokens[index][2])
+ self.assertEqual(expected_tokens[index], tokens[index][1])
+
+ def assert_fast_tokenization(self, s):
+ """Show that a given string is tokenized quickly."""
+ start = time.time()
+ tokens = list(self.lexer.get_tokens_unprocessed(s))
+ end = time.time()
+ # Isn't 10 seconds kind of a long time? Yes, but we don't want false
+ # positives when the tests are starved for CPU time.
+ if end-start > 10:
+ self.fail('tokenization took too long')
+ return tokens
+
+ ### Strings.
+
+ def test_single_quote_strings(self):
+ self.assert_single_token(r"'foo\tbar\\\'baz'", String)
+ self.assert_fast_tokenization("'" + '\\'*999)
+
+ def test_double_quote_strings(self):
+ self.assert_single_token(r'"foo\tbar\\\"baz"', String)
+ self.assert_fast_tokenization('"' + '\\'*999)
+
+ def test_backtick_strings(self):
+ self.assert_single_token(r'`foo\tbar\\\`baz`', String.Backtick)
+ self.assert_fast_tokenization('`' + '\\'*999)
+
+ ### Regex matches with various delimiters.
+
+ def test_match(self):
+ self.assert_single_token(r'/aa\tbb/', String.Regex)
+ self.assert_fast_tokenization('/' + '\\'*999)
+
+ def test_match_with_slash(self):
+ self.assert_tokens(['m', '/\n\\t\\\\/'], [String.Regex, String.Regex])
+ self.assert_fast_tokenization('m/xxx\n' + '\\'*999)
+
+ def test_match_with_bang(self):
+ self.assert_tokens(['m', r'!aa\t\!bb!'], [String.Regex, String.Regex])
+ self.assert_fast_tokenization('m!' + '\\'*999)
+
+ def test_match_with_brace(self):
+ self.assert_tokens(['m', r'{aa\t\}bb}'], [String.Regex, String.Regex])
+ self.assert_fast_tokenization('m{' + '\\'*999)
+
+ def test_match_with_angle_brackets(self):
+ self.assert_tokens(['m', r'<aa\t\>bb>'], [String.Regex, String.Regex])
+ self.assert_fast_tokenization('m<' + '\\'*999)
+
+ def test_match_with_parenthesis(self):
+ self.assert_tokens(['m', r'(aa\t\)bb)'], [String.Regex, String.Regex])
+ self.assert_fast_tokenization('m(' + '\\'*999)
+
+ def test_match_with_at_sign(self):
+ self.assert_tokens(['m', r'@aa\t\@bb@'], [String.Regex, String.Regex])
+ self.assert_fast_tokenization('m@' + '\\'*999)
+
+ def test_match_with_percent_sign(self):
+ self.assert_tokens(['m', r'%aa\t\%bb%'], [String.Regex, String.Regex])
+ self.assert_fast_tokenization('m%' + '\\'*999)
+
+ def test_match_with_dollar_sign(self):
+ self.assert_tokens(['m', r'$aa\t\$bb$'], [String.Regex, String.Regex])
+ self.assert_fast_tokenization('m$' + '\\'*999)
+
+ ### Regex substitutions with various delimeters.
+
+ def test_substitution_with_slash(self):
+ self.assert_single_token('s/aaa/bbb/g', String.Regex)
+ self.assert_fast_tokenization('s/foo/' + '\\'*999)
+
+ def test_substitution_with_at_sign(self):
+ self.assert_single_token(r's@aaa@bbb@g', String.Regex)
+ self.assert_fast_tokenization('s@foo@' + '\\'*999)
+
+ def test_substitution_with_percent_sign(self):
+ self.assert_single_token(r's%aaa%bbb%g', String.Regex)
+ self.assert_fast_tokenization('s%foo%' + '\\'*999)
+
+ def test_substitution_with_brace(self):
+ self.assert_single_token(r's{aaa}', String.Regex)
+ self.assert_fast_tokenization('s{' + '\\'*999)
+
+ def test_substitution_with_angle_bracket(self):
+ self.assert_single_token(r's<aaa>', String.Regex)
+ self.assert_fast_tokenization('s<' + '\\'*999)
+
+ def test_substitution_with_angle_bracket(self):
+ self.assert_single_token(r's<aaa>', String.Regex)
+ self.assert_fast_tokenization('s<' + '\\'*999)
+
+ def test_substitution_with_square_bracket(self):
+ self.assert_single_token(r's[aaa]', String.Regex)
+ self.assert_fast_tokenization('s[' + '\\'*999)
+
+ def test_substitution_with_parenthesis(self):
+ self.assert_single_token(r's(aaa)', String.Regex)
+ self.assert_fast_tokenization('s(' + '\\'*999)
diff --git a/tests/test_util.py b/tests/test_util.py
index 0876cf70..af1f4e44 100644
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -13,6 +13,12 @@ import os
from pygments import util
+class FakeLexer(object):
+ def analyse(text):
+ return float(text)
+ analyse = util.make_analysator(analyse)
+
+
class UtilTest(unittest.TestCase):
def test_getoptions(self):
@@ -53,12 +59,36 @@ class UtilTest(unittest.TestCase):
self.assertEquals(util.docstring_headline(f1), "docstring headline")
self.assertEquals(util.docstring_headline(f2), "docstring headline")
- def test_analysator(self):
- class X(object):
+ def test_analysator_returns_float(self):
+ # If an analysator wrapped by make_analysator returns a floating point
+ # number, then that number will be returned by the wrapper.
+ self.assertEquals(FakeLexer.analyse('0.5'), 0.5)
+
+ def test_analysator_returns_boolean(self):
+ # If an analysator wrapped by make_analysator returns a boolean value,
+ # then the wrapper will return 1.0 if the boolean was True or 0.0 if
+ # it was False.
+ self.assertEquals(FakeLexer.analyse(True), 1.0)
+ self.assertEquals(FakeLexer.analyse(False), 0.0)
+
+ def test_analysator_raises_exception(self):
+ # If an analysator wrapped by make_analysator raises an exception,
+ # then the wrapper will return 0.0.
+ class ErrorLexer(object):
def analyse(text):
- return 0.5
+ raise RuntimeError('something bad happened')
analyse = util.make_analysator(analyse)
- self.assertEquals(X.analyse(''), 0.5)
+ self.assertEquals(ErrorLexer.analyse(''), 0.0)
+
+ def test_analysator_value_error(self):
+ # When converting the analysator's return value to a float a
+ # ValueError may occur. If that happens 0.0 is returned instead.
+ self.assertEquals(FakeLexer.analyse('bad input'), 0.0)
+
+ def test_analysator_type_error(self):
+ # When converting the analysator's return value to a float a
+ # TypeError may occur. If that happens 0.0 is returned instead.
+ self.assertEquals(FakeLexer.analyse(None), 0.0)
def test_shebang_matches(self):
self.assert_(util.shebang_matches('#!/usr/bin/env python', r'python(2\.\d)?'))
View Lorry Controller Status