class TreeParser
Parse tokens into AST structure.
Constants
- BLOCK_TOKEN_TYPES
Public Class Methods
new(tokens)
click to toggle source
# File lib/rosetta/services/tree_parser.rb, line 30 def initialize(tokens) @tokens = tokens @counter = 0 end
Public Instance Methods
call()
click to toggle source
TODO: Investigate if this can be done without declaring and mutating the array.
# File lib/rosetta/services/tree_parser.rb, line 36 def call parsed_tree = [] parsed_tree << consume_next_block until end_of_file? parsed_tree.compact end
Private Instance Methods
consume_current_token()
click to toggle source
# File lib/rosetta/services/tree_parser.rb, line 117 def consume_current_token frozen_current_token = current_token @counter += 1 frozen_current_token end
consume_next_block()
click to toggle source
# File lib/rosetta/services/tree_parser.rb, line 45 def consume_next_block block_types = BLOCK_TOKEN_TYPES.keys + [:CODE_BLOCK_DELIMITER] if block_types.include?(current_token.type) handle_generic_block_types(current_token.type) elsif current_token.type == :LINE_BREAK match_break elsif current_token.inline? match_paragraph else consume_current_token end end
current_token()
click to toggle source
# File lib/rosetta/services/tree_parser.rb, line 59 def current_token @tokens[@counter] end
end_of_file?()
click to toggle source
# File lib/rosetta/services/tree_parser.rb, line 123 def end_of_file? @counter > @tokens.length - 1 end
group_siblings_into_block(block_type)
click to toggle source
# File lib/rosetta/services/tree_parser.rb, line 85 def group_siblings_into_block(block_type) block_class = BLOCK_TOKEN_TYPES[block_type] child_tokens = [] child_tokens << consume_current_token while !end_of_file? && current_token.type == block_type block_class.new(child_tokens) end
handle_generic_block_types(token_type)
click to toggle source
# File lib/rosetta/services/tree_parser.rb, line 79 def handle_generic_block_types(token_type) return match_code_block if token_type == :CODE_BLOCK_DELIMITER group_siblings_into_block(token_type) end
match_break()
click to toggle source
# File lib/rosetta/services/tree_parser.rb, line 67 def match_break if next_token.type == :LINE_BREAK @counter += 2 Break.new else @counter += 1 # We don't want to insert newlines all the time, the desired # insertion is handled in paragraph grouping. nil end end
match_code_block()
click to toggle source
# File lib/rosetta/services/tree_parser.rb, line 104 def match_code_block @counter += 1 child_tokens = [] while !end_of_file? && current_token.type != :CODE_BLOCK_DELIMITER child_tokens << current_token @counter += 1 end @counter += 1 unless end_of_file? CodeBlock.new(child_tokens) end
match_paragraph()
click to toggle source
# File lib/rosetta/services/tree_parser.rb, line 94 def match_paragraph child_tokens = [] while !end_of_file? && (current_token.inline? || current_token.type == :NEW_LINE) child_tokens << consume_current_token end Paragraph.new(child_tokens) end
next_token()
click to toggle source
# File lib/rosetta/services/tree_parser.rb, line 63 def next_token @tokens[@counter + 1] end