1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
|
module CodeRay
module Scanners
class RuleBasedScanner4 < Scanner
class << self
attr_accessor :states
def state *names, &block
@@code ||= ""
@@code << "when #{names.map(&:inspect).join(', ')}\n"
@@first = true
instance_eval(&block)
@@code << " else\n"
# @@code << " raise 'no match for #{names.map(&:inspect).join(', ')}'\n"
@@code << " encoder.text_token getch, :error\n"
@@code << " end\n"
@@code << " \n"
end
def token pattern, *actions
@@code << " #{'els' unless @@first}if match = scan(#{pattern.inspect})\n"
for action in actions
case action
when Symbol
@@code << " p 'text_token %p %p' % [match, #{action.inspect}]\n" if $DEBUG
@@code << " encoder.text_token match, #{action.inspect}\n"
when Array
case action.first
when :push
@@code << " p 'push %p' % [#{action.last.inspect}]\n" if $DEBUG
@@code << " state = #{action.last.inspect}\n"
@@code << " states << state\n"
@@code << " encoder.begin_group state\n"
when :pop
@@code << " p 'pop %p' % [states.last]\n" if $DEBUG
@@code << " encoder.end_group states.pop\n"
@@code << " state = states.last\n"
end
end
end
@@first = false
end
def push state
[:push, state]
end
def pop
[:pop]
end
end
end
# Scanner for JSON (JavaScript Object Notation).
class JSON4 < RuleBasedScanner4
register_for :json4
file_extension 'json'
KINDS_NOT_LOC = [
:float, :char, :content, :delimiter,
:error, :integer, :operator, :value,
] # :nodoc:
ESCAPE = / [bfnrt\\"\/] /x # :nodoc:
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x # :nodoc:
KEY = / (?> (?: [^\\"]+ | \\. )* ) " \s* : /mx
state :initial do
token %r/ \s+ /x, :space
token %r/ [:,\[{\]}] /x, :operator
token %r/ " (?=#{KEY}) /x, push(:key), :delimiter
token %r/ " /x, push(:string), :delimiter
token %r/ true | false | null /x, :value
token %r/ -? (?: 0 | [1-9]\d* ) (?: \.\d+ (?: e[-+]? \d+ )? | e[-+]? \d+ ) /ix, :float
token %r/ -? (?: 0 | [1-9]\d* ) (?: e[+-] \d+ )? /ix, :integer
end
state :key, :string do
token %r/ [^\\"]+ /x, :content
token %r/ " /x, :delimiter, pop
token %r/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /x, :char
token %r/ \\. /mx, :content
token %r/ \\ /x, :error, pop
end
protected
def setup
@state = :initial
end
# See http://json.org/ for a definition of the JSON lexic/grammar.
scan_tokens_code = <<-"RUBY"
def scan_tokens encoder, options
state = options[:state] || @state
if [:string, :key].include? state
encoder.begin_group state
end
states = [state]
until eos?
case state
#{ @@code.chomp.gsub(/^/, ' ') }
else
raise_inspect 'Unknown state: %p' % [state], encoder
end
end
if options[:keep_state]
@state = state
end
if [:string, :key].include? state
encoder.end_group state
end
encoder
end
RUBY
# puts scan_tokens_code
class_eval scan_tokens_code
end
end
end
|
