1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
|
module CodeRay
module Scanners
class RuleBasedScanner2 < Scanner
class << self
attr_accessor :states
def state *names, &block
@@states ||= {}
@@rules = []
instance_eval(&block)
for name in names
@@states[name] = @@rules
end
@@rules = nil
end
def token pattern, *actions
@@rules << [pattern, *actions]
end
def push_group name
[:begin_group, name]
end
def pop_group
[:end_group]
end
end
end
# Scanner for JSON (JavaScript Object Notation).
class JSON2 < RuleBasedScanner2
register_for :json2
file_extension 'json'
KINDS_NOT_LOC = [
:float, :char, :content, :delimiter,
:error, :integer, :operator, :value,
] # :nodoc:
ESCAPE = / [bfnrt\\"\/] /x # :nodoc:
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x # :nodoc:
KEY = / (?> (?: [^\\"]+ | \\. )* ) " \s* : /mx
state :initial do
token %r/ \s+ /x, :space
token %r/ " (?=#{KEY}) /x, push_group(:key), :delimiter
token %r/ " /x, push_group(:string), :delimiter
token %r/ [:,\[{\]}] /x, :operator
token %r/ true | false | null /x, :value
token %r/ -? (?: 0 | [1-9]\d* ) (?: \.\d+ (?: [eE][-+]? \d+ )? | [eE][-+]? \d+ ) /x, :float
token %r/ -? (?: 0 | [1-9]\d* ) /x, :integer
end
state :string, :key do
token %r/ [^\\"]+ /x, :content
token %r/ " /x, :delimiter, pop_group
token %r/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /x, :char
token %r/ \\. /mx, :content
token %r/ \\ /x, pop_group, :error
# token %r/$/, end_group
end
protected
def setup
@state = :initial
end
# See http://json.org/ for a definition of the JSON lexic/grammar.
def scan_tokens encoder, options
state = options[:state] || @state
if [:string, :key].include? state
encoder.begin_group state
end
states = [state]
until eos?
for pattern, *actions in @@states[state]
if match = scan(pattern)
for action in actions
case action
when Symbol
encoder.text_token match, action
when Array
case action.first
when :begin_group
encoder.begin_group action.last
state = action.last
states << state
when :end_group
encoder.end_group states.pop
state = states.last
end
end
end
break
end
end && encoder.text_token(getch, :error)
end
if options[:keep_state]
@state = state
end
if [:string, :key].include? state
encoder.end_group state
end
encoder
end
end
end
end
|
