class Markie::Tokenizer
Public Class Methods
tokenize(markdown)
click to toggle source
# File lib/markie/tokenizer.rb, line 7 def tokenize(markdown) scan(markdown, []) end
Private Class Methods
eof_token()
click to toggle source
# File lib/markie/tokenizer.rb, line 34 def eof_token Token.new(type: :eof) end
rules()
click to toggle source
# File lib/markie/tokenizer.rb, line 46 def rules [ Rule.new( regex: Regexp.new("\_"), tokenize_rule: ->(markdown, regex) { Token.new(type: :underscore, value: "_") } ), Rule.new( regex: Regexp.new("\\*"), tokenize_rule: ->(markdown, regex) { Token.new(type: :asterisk, value: "*") } ), Rule.new( regex: Regexp.new("\\["), tokenize_rule: ->(markdown, regex) { Token.new(type: :open_square_bracket, value: "[") } ), Rule.new( regex: Regexp.new("\\]"), tokenize_rule: ->(markdown, regex) { Token.new(type: :close_square_bracket, value: "]") } ), Rule.new( regex: Regexp.new("\\("), tokenize_rule: ->(markdown, regex) { Token.new(type: :open_parenthesis, value: "(") } ), Rule.new( regex: Regexp.new("\\)"), tokenize_rule: ->(markdown, regex) { Token.new(type: :close_parenthesis, value: ")") } ), Rule.new( regex: Regexp.new("\n"), tokenize_rule: ->(markdown, regex) { Token.new(type: :newline, value: "\n") } ), ] end
scan(markdown, tokens)
click to toggle source
# File lib/markie/tokenizer.rb, line 13 def scan(markdown, tokens) if markdown.length == 0 return tokens.append(eof_token) end next_token = nil rules.each do |rule| if rule.applies_to?(markdown) next_token = rule.tokenize(markdown) break end end if next_token.nil? next_token = text_token(markdown) end scan(markdown[next_token.length..-1], tokens.append(next_token)) end
text_token(markdown)
click to toggle source
# File lib/markie/tokenizer.rb, line 38 def text_token(markdown) text = markdown.each_char.take_while { |char| !char.match?(Regexp.union(rules.map(&:regex))) }.join("") Token.new(type: :text, value: text) end