1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
|
# frozen_string_literal: true
require 'rubygems/request_set/lockfile/parser'
class Gem::RequestSet::Lockfile::Tokenizer
Token = Struct.new :type, :value, :column, :line
EOF = Token.new :EOF
def self.from_file(file)
new File.read(file), file
end
def initialize(input, filename = nil, line = 0, pos = 0)
@line = line
@line_pos = pos
@tokens = []
@filename = filename
tokenize input
end
def make_parser(set, platforms)
Gem::RequestSet::Lockfile::Parser.new self, set, platforms, @filename
end
def to_a
@tokens.map {|token| [token.type, token.value, token.column, token.line] }
end
def skip(type)
@tokens.shift while not @tokens.empty? and peek.type == type
end
##
# Calculates the column (by byte) and the line of the current token based on
# +byte_offset+.
def token_pos(byte_offset) # :nodoc:
[byte_offset - @line_pos, @line]
end
def empty?
@tokens.empty?
end
def unshift(token)
@tokens.unshift token
end
def next_token
@tokens.shift
end
alias :shift :next_token
def peek
@tokens.first || EOF
end
private
def tokenize(input)
require 'strscan'
s = StringScanner.new input
until s.eos? do
pos = s.pos
pos = s.pos if leading_whitespace = s.scan(/ +/)
if s.scan(/[<|=>]{7}/)
message = "your #{@filename} contains merge conflict markers"
column, line = token_pos pos
raise Gem::RequestSet::Lockfile::ParseError.new message, column, line, @filename
end
@tokens <<
case
when s.scan(/\r?\n/) then
token = Token.new(:newline, nil, *token_pos(pos))
@line_pos = s.pos
@line += 1
token
when s.scan(/[A-Z]+/) then
if leading_whitespace
text = s.matched
text += s.scan(/[^\s)]*/).to_s # in case of no match
Token.new(:text, text, *token_pos(pos))
else
Token.new(:section, s.matched, *token_pos(pos))
end
when s.scan(/([a-z]+):\s/) then
s.pos -= 1 # rewind for possible newline
Token.new(:entry, s[1], *token_pos(pos))
when s.scan(/\(/) then
Token.new(:l_paren, nil, *token_pos(pos))
when s.scan(/\)/) then
Token.new(:r_paren, nil, *token_pos(pos))
when s.scan(/<=|>=|=|~>|<|>|!=/) then
Token.new(:requirement, s.matched, *token_pos(pos))
when s.scan(/,/) then
Token.new(:comma, nil, *token_pos(pos))
when s.scan(/!/) then
Token.new(:bang, nil, *token_pos(pos))
when s.scan(/[^\s),!]*/) then
Token.new(:text, s.matched, *token_pos(pos))
else
raise "BUG: can't create token for: #{s.string[s.pos..-1].inspect}"
end
end
@tokens
end
end
|
