1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
|
# coding: US-ASCII
# frozen_string_literal: false
require "stringio"
require "strscan"
require_relative 'encoding'
module REXML
if StringScanner::Version < "1.0.0"
module StringScannerCheckScanString
refine StringScanner do
def check(pattern)
pattern = /#{Regexp.escape(pattern)}/ if pattern.is_a?(String)
super(pattern)
end
def scan(pattern)
pattern = /#{Regexp.escape(pattern)}/ if pattern.is_a?(String)
super(pattern)
end
def match?(pattern)
pattern = /#{Regexp.escape(pattern)}/ if pattern.is_a?(String)
super(pattern)
end
def skip(pattern)
pattern = /#{Regexp.escape(pattern)}/ if pattern.is_a?(String)
super(pattern)
end
end
end
using StringScannerCheckScanString
end
# Generates Source-s. USE THIS CLASS.
class SourceFactory
# Generates a Source object
# @param arg Either a String, or an IO
# @return a Source, or nil if a bad argument was given
def SourceFactory::create_from(arg)
if arg.respond_to? :read and
arg.respond_to? :readline and
arg.respond_to? :nil? and
arg.respond_to? :eof?
IOSource.new(arg)
elsif arg.respond_to? :to_str
IOSource.new(StringIO.new(arg))
elsif arg.kind_of? Source
arg
else
raise "#{arg.class} is not a valid input stream. It must walk \n"+
"like either a String, an IO, or a Source."
end
end
end
# A Source can be searched for patterns, and wraps buffers and other
# objects and provides consumption of text
class Source
include Encoding
# The line number of the last consumed text
attr_reader :line
attr_reader :encoding
module Private
SPACES_PATTERN = /\s+/um
SCANNER_RESET_SIZE = 100000
PRE_DEFINED_TERM_PATTERNS = {}
pre_defined_terms = ["'", '"', "<", "]]>", "?>"]
if StringScanner::Version < "3.1.1"
pre_defined_terms.each do |term|
PRE_DEFINED_TERM_PATTERNS[term] = /#{Regexp.escape(term)}/
end
else
pre_defined_terms.each do |term|
PRE_DEFINED_TERM_PATTERNS[term] = term
end
end
end
private_constant :Private
# Constructor
# @param arg must be a String, and should be a valid XML document
# @param encoding if non-null, sets the encoding of the source to this
# value, overriding all encoding detection
def initialize(arg, encoding=nil)
@orig = arg
@scanner = StringScanner.new(@orig)
if encoding
self.encoding = encoding
else
detect_encoding
end
@line = 0
@encoded_terms = {}
end
# The current buffer (what we're going to read next)
def buffer
@scanner.rest
end
def drop_parsed_content
if @scanner.pos > Private::SCANNER_RESET_SIZE
@scanner.string = @scanner.rest
end
end
def buffer_encoding=(encoding)
@scanner.string.force_encoding(encoding)
end
# Inherited from Encoding
# Overridden to support optimized en/decoding
def encoding=(enc)
return unless super
encoding_updated
end
def read(term = nil)
end
def read_until(term)
pattern = Private::PRE_DEFINED_TERM_PATTERNS[term] || /#{Regexp.escape(term)}/
data = @scanner.scan_until(pattern)
unless data
data = @scanner.rest
@scanner.pos = @scanner.string.bytesize
end
data
end
def ensure_buffer
end
def match(pattern, cons=false)
if cons
@scanner.scan(pattern).nil? ? nil : @scanner
else
@scanner.check(pattern).nil? ? nil : @scanner
end
end
def match?(pattern, cons=false)
if cons
!@scanner.skip(pattern).nil?
else
!@scanner.match?(pattern).nil?
end
end
def skip_spaces
@scanner.skip(Private::SPACES_PATTERN) ? true : false
end
def position
@scanner.pos
end
def position=(pos)
@scanner.pos = pos
end
def peek_byte
@scanner.peek_byte
end
def scan_byte
@scanner.scan_byte
end
# @return true if the Source is exhausted
def empty?
@scanner.eos?
end
# @return the current line in the source
def current_line
lines = @orig.split
res = lines.grep @scanner.rest[0..30]
res = res[-1] if res.kind_of? Array
lines.index( res ) if res
end
private
def detect_encoding
scanner_encoding = @scanner.rest.encoding
detected_encoding = "UTF-8"
begin
@scanner.string.force_encoding("ASCII-8BIT")
if @scanner.scan(/\xfe\xff/n)
detected_encoding = "UTF-16BE"
elsif @scanner.scan(/\xff\xfe/n)
detected_encoding = "UTF-16LE"
elsif @scanner.scan(/\xef\xbb\xbf/n)
detected_encoding = "UTF-8"
end
ensure
@scanner.string.force_encoding(scanner_encoding)
end
self.encoding = detected_encoding
end
def encoding_updated
if @encoding != 'UTF-8'
@scanner.string = decode(@scanner.rest)
@to_utf = true
else
@to_utf = false
@scanner.string.force_encoding(::Encoding::UTF_8)
end
end
end
# A Source that wraps an IO. See the Source class for method
# documentation
class IOSource < Source
#attr_reader :block_size
# block_size has been deprecated
def initialize(arg, block_size=500, encoding=nil)
@er_source = @source = arg
@to_utf = false
@pending_buffer = nil
if encoding
super("", encoding)
else
super(@source.read(3) || "")
end
if !@to_utf and
@orig.respond_to?(:force_encoding) and
@source.respond_to?(:external_encoding) and
@source.external_encoding != ::Encoding::UTF_8
@force_utf8 = true
else
@force_utf8 = false
end
end
def read(term = nil, min_bytes = 1)
term = encode(term) if term
begin
str = readline(term)
@scanner << str
read_bytes = str.bytesize
begin
while read_bytes < min_bytes
str = readline(term)
@scanner << str
read_bytes += str.bytesize
end
rescue IOError
end
true
rescue Exception, NameError
@source = nil
false
end
end
def read_until(term)
pattern = Private::PRE_DEFINED_TERM_PATTERNS[term] || /#{Regexp.escape(term)}/
term = @encoded_terms[term] ||= encode(term)
until str = @scanner.scan_until(pattern)
break if @source.nil?
break if @source.eof?
@scanner << readline(term)
end
if str
read if @scanner.eos? and @source and !@source.eof?
str
else
rest = @scanner.rest
@scanner.pos = @scanner.string.bytesize
rest
end
end
def ensure_buffer
read if @scanner.eos? && @source
end
def match( pattern, cons=false )
# To avoid performance issue, we need to increase bytes to read per scan
min_bytes = 1
while true
if cons
md = @scanner.scan(pattern)
else
md = @scanner.check(pattern)
end
break if md
return nil if pattern.is_a?(String)
return nil if @source.nil?
return nil unless read(nil, min_bytes)
min_bytes *= 2
end
md.nil? ? nil : @scanner
end
def match?( pattern, cons=false )
# To avoid performance issue, we need to increase bytes to read per scan
min_bytes = 1
while true
if cons
n_matched_bytes = @scanner.skip(pattern)
else
n_matched_bytes = @scanner.match?(pattern)
end
return true if n_matched_bytes
return false if pattern.is_a?(String)
return false if @source.nil?
return false unless read(nil, min_bytes)
min_bytes *= 2
end
end
def empty?
super and ( @source.nil? || @source.eof? )
end
# @return the current line in the source
def current_line
begin
pos = @er_source.pos # The byte position in the source
lineno = @er_source.lineno # The XML < position in the source
@er_source.rewind
line = 0 # The \r\n position in the source
begin
while @er_source.pos < pos
@er_source.readline
line += 1
end
rescue
end
@er_source.seek(pos)
rescue IOError, SystemCallError
pos = -1
line = -1
end
[pos, lineno, line]
end
private
def readline(term = nil)
if @pending_buffer
begin
str = @source.readline(term || @line_break)
rescue IOError
end
if str.nil?
str = @pending_buffer
else
str = @pending_buffer + str
end
@pending_buffer = nil
else
str = @source.readline(term || @line_break)
end
return nil if str.nil?
if @to_utf
decode(str)
else
str.force_encoding(::Encoding::UTF_8) if @force_utf8
str
end
end
def encoding_updated
case @encoding
when "UTF-16BE", "UTF-16LE"
@source.binmode
@source.set_encoding(@encoding, @encoding)
end
@line_break = encode(">")
@pending_buffer, @scanner.string = @scanner.rest, ""
@pending_buffer.force_encoding(@encoding)
super
end
end
end
|