1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
|
require "strscan"
module Liquid
class Lexer
SPECIALS = {
'|'.freeze => :pipe,
'.'.freeze => :dot,
':'.freeze => :colon,
','.freeze => :comma,
'['.freeze => :open_square,
']'.freeze => :close_square,
'('.freeze => :open_round,
')'.freeze => :close_round,
'?'.freeze => :question,
'-'.freeze => :dash
}.freeze
IDENTIFIER = /[a-zA-Z_][\w-]*\??/
SINGLE_STRING_LITERAL = /'[^\']*'/
DOUBLE_STRING_LITERAL = /"[^\"]*"/
NUMBER_LITERAL = /-?\d+(\.\d+)?/
DOTDOT = /\.\./
COMPARISON_OPERATOR = /==|!=|<>|<=?|>=?|contains(?=\s)/
WHITESPACE_OR_NOTHING = /\s*/
def initialize(input)
@ss = StringScanner.new(input)
end
def tokenize
@output = []
until @ss.eos?
@ss.skip(WHITESPACE_OR_NOTHING)
break if @ss.eos?
tok = case
when t = @ss.scan(COMPARISON_OPERATOR) then [:comparison, t]
when t = @ss.scan(SINGLE_STRING_LITERAL) then [:string, t]
when t = @ss.scan(DOUBLE_STRING_LITERAL) then [:string, t]
when t = @ss.scan(NUMBER_LITERAL) then [:number, t]
when t = @ss.scan(IDENTIFIER) then [:id, t]
when t = @ss.scan(DOTDOT) then [:dotdot, t]
else
c = @ss.getch
if s = SPECIALS[c]
[s, c]
else
raise SyntaxError, "Unexpected character #{c}"
end
end
@output << tok
end
@output << [:end_of_string]
end
end
end
|
