class VueCompiler::Tokenizer
Constants
- ATTRIBUTE_NAME_RE
- ATTRIBUTE_VALUE_RE
- QUOTE_RE
- TAG_NAME_RE
- TAG_RE
- VOID_TAG
- WHITESPACE_RE
Public Class Methods
new(code)
click to toggle source
@param code {String}
# File lib/vue_component_compiler/tokenizer.rb, line 13 def initialize(code) @code = code.delete_prefix("\uFEFF") @tokens = [] end
Public Instance Methods
eat(str)
click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 39 def eat(str) @code.delete_prefix!(str) end
eat_whitespaces()
click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 43 def eat_whitespaces @code.sub!(WHITESPACE_RE, '') end
expect(str)
click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 47 def expect(str) if @code.start_with?(str) self.eat(str) elsif raise Exception.new("Found character `#{@code[0]}`, expecting `#{str}`") end end
scan_attribute()
click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 145 def scan_attribute name = @code.slice(ATTRIBUTE_NAME_RE) value = nil @tokens << Token::AttributeName.new(name) self.eat(name) self.eat_whitespaces return { name: name, value: value } unless @code.start_with?('=') self.expect('=') self.eat_whitespaces if (@code =~ ATTRIBUTE_VALUE_RE) == 0 self.scan_attribute_value value = self.eat_whitespaces end { name: name, value: value } end
scan_attribute_value()
click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 164 def scan_attribute_value quoted = @code.start_with?("'") || @code.start_with?('"') matched = @code.match(ATTRIBUTE_VALUE_RE) value = matched[1] || matched[2] || matched[3] @tokens << Token::AttributeValue.new(value) @code = @code[((quoted ? 2 : 0) + value.size)..-1] value end
scan_comment()
click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 208 def scan_comment @tokens << Token::CommentStart.new end_pos = @code.index('-->') if end_pos @tokens << Token::Comment.new(@code[4...end_pos]) @tokens << Token::CommentEnd.new @code = @code[end_pos + 3..-1] end end
scan_custom_block()
click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 299 def scan_custom_block self.eat('<') type = @code.slice(TAG_NAME_RE) content = '' self.eat(type) self.eat_whitespaces # TODO: support attributes on custom block. self.eat('>') until @code.start_with?("</#{type}") char = @code[0] content << char self.eat(char) end self.eat("</#{type}") self.eat_whitespaces self.eat('>') @tokens << Token::CustomBlock.new(type, content) end
scan_interpolation()
click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 186 def scan_interpolation expr = '' is_string = false @tokens << Token::InterpolationStart.new self.eat('{{') while is_string || (!is_string && !@code.start_with?('}}')) char = @code[0] expr << char self.eat(char) if char == '"' || char == "'" || char == '`' is_string = !is_string end end expr.strip! @tokens << Token::Interpolation.new(expr) @tokens << Token::InterpolationEnd.new self.eat('}}') end
scan_script_block()
click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 218 def scan_script_block self.eat('<script') self.eat_whitespaces lang = if @code.start_with?('lang') self.eat('lang') self.eat_whitespaces self.expect('=') self.eat_whitespaces quoted = @code.start_with?("'") || @code.start_with?('"') matched = @code.match(ATTRIBUTE_VALUE_RE) lang = matched[1] || matched[2] || matched[3] @code = @code[((quoted ? 2 : 0) + lang.size)..-1] self.eat_whitespaces self.expect('>') lang elsif @code.start_with?('>') self.eat('>') nil end content = '' is_string = false while is_string || (!is_string && !@code.start_with?('</script')) char = @code[0] if char == '"' || char == "'" || char == '`' is_string = !is_string end content << char self.eat(char) end self.eat('</script') self.eat_whitespaces self.expect('>') @tokens << Token::ScriptBlock.new(content, lang) end
scan_style_block()
click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 257 def scan_style_block content = '' scoped = false lang = 'css' self.eat('<style') self.eat_whitespaces until @code.start_with?('>') if @code.start_with?('scoped') scoped = true self.eat('scoped') elsif @code.start_with?('lang') self.eat('lang') self.eat_whitespaces self.expect('=') self.eat_whitespaces quoted = @code.start_with?("'") || @code.start_with?('"') matched = @code.match(ATTRIBUTE_VALUE_RE) lang = matched[1] || matched[2] || matched[3] @code.sub!(QUOTE_RE, '') self.eat(lang) @code.sub!(QUOTE_RE, '') end self.eat_whitespaces end self.eat('>') until @code.start_with?('</style') char = @code[0] content << char self.eat(char) end self.eat('</style') self.eat_whitespaces self.expect('>') @tokens << Token::StyleBlock.new(content, lang, scoped) end
scan_tag()
click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 86 def scan_tag tag_name = self.scan_tag_name is_v_pre = false self.eat_whitespaces while (@code =~ ATTRIBUTE_NAME_RE) == 0 attr = self.scan_attribute if attr[:name] == 'v-pre' is_v_pre = true end self.eat_whitespaces end if @code.start_with?('/>') @tokens << Token::SelfClosingTag.new self.eat('/>') return elsif @code.start_with?('>') if VOID_TAG.include?(tag_name) @tokens << Token::SelfClosingTag.new self.eat('>') return else @tokens << Token::TagEnd.new self.eat('>') end end until @code.start_with?('</') if @code.start_with?('<!--') self.scan_comment elsif @code.start_with?('<') self.scan_tag elsif @code.start_with?('{{') self.scan_interpolation else self.scan_text(is_v_pre) end end self.expect('</') @tokens << Token::ClosingTagStart.new self.expect(tag_name) @tokens << Token::ClosingTagName.new(tag_name) self.eat_whitespaces self.expect('>') @tokens << Token::TagEnd.new end
scan_tag_name()
click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 135 def scan_tag_name self.expect('<') @tokens << Token::OpenTagStart.new tag_name = @code.slice(TAG_NAME_RE) @tokens << Token::OpenTagName.new(tag_name) self.eat(tag_name) tag_name end
scan_text(ignore_interpolation = false)
click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 175 def scan_text(ignore_interpolation = false) text = '' until @code.start_with?('<') || (!ignore_interpolation && @code.start_with?('{{')) char = @code[0] self.eat(char) next if char == ' ' && text.end_with?(' ') text << char end @tokens << Token::Text.new(text) end
scan_top_level_template()
click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 55 def scan_top_level_template @tokens << Token::OpenTagStart.new @tokens << Token::OpenTagName.new('template') @tokens << Token::TagEnd.new self.eat('<template') self.eat_whitespaces self.expect('>') self.eat_whitespaces root_scanned = false until @code.start_with?('</') if @code.start_with?('<!--') self.scan_comment else unless root_scanned self.scan_tag root_scanned = true end end self.eat_whitespaces end self.eat('</') @tokens << Token::ClosingTagStart.new self.expect('template') @tokens << Token::ClosingTagName.new('template') self.eat_whitespaces self.expect('>') @tokens << Token::TagEnd.new end
tokenize()
click to toggle source
# File lib/vue_component_compiler/tokenizer.rb, line 18 def tokenize while @code && !@code.empty? if @code.start_with?('<template') self.scan_top_level_template elsif @code =~ WHITESPACE_RE self.eat_whitespaces elsif @code.start_with?('<script') self.scan_script_block elsif @code.start_with?('<style') self.scan_style_block elsif @code.start_with?('<!--') self.scan_comment elsif @code =~ TAG_RE self.scan_custom_block end end @tokens << Token::EndOfSource.new @tokens end