class Astrapi::Lexer
Constants
- ARROW
.….….….operators.….….….….….…..
- ATTR
- CLASS
- COMMENT
- END_
- FLOAT
- IDENT
.….….….….….….….….….….….…
- IDENTIFIER
.….….….literals.….….….….….….
- INT
- LBRACK
- LPAREN
.….….….punctuation.….….….….….…
- LT
- MODULE
.….….….….….….….….….….….…
- NEWLINE
.….….….….….….….….….….….…
- RBRACK
- RPAREN
- SPACE
- STRING
Attributes
suppress_comment[RW]
Public Class Methods
new(str='')
click to toggle source
# File lib/lexer.rb, line 61 def initialize str='' init(str) @suppress_space=true @suppress_comment=false end
Public Instance Methods
init(str)
click to toggle source
# File lib/lexer.rb, line 67 def init str @ss=StringScanner.new(str) @line=0 end
next_token()
click to toggle source
next token can detect spaces length
# File lib/lexer.rb, line 82 def next_token if @ss.bol? @line+=1 @old_pos=@ss.pos end position=[@line,@ss.pos-@old_pos+1] return :eos if @ss.eos? case when text = @ss.scan(NEWLINE) next_token() when text = @ss.scan(SPACE) next_token() when text = @ss.scan(COMMENT) next_token() when text = @ss.scan(ARROW) return Token.new [:arrow,text,position] when text = @ss.scan(LT) return Token.new [:lt,text,position] when text = @ss.scan(LBRACK) return Token.new [:lbrack,text,position] when text = @ss.scan(RBRACK) return Token.new [:rbrack,text,position] when text = @ss.scan(IDENTIFIER) case when value = text.match(IDENT) return Token.new [:IDENT,text,position] when value = text.match(FLOAT) return Token.new [:FLOAT,text,position] when value = text.match(INT) return Token.new [:INT,text,position] when value = text.match(STRING) return Token.new [:STRING,text,position] when value = text.match(MODULE) return Token.new [:module,text,position] when value = text.match(CLASS) return Token.new [:class,text,position] when value = text.match(END_) return Token.new [:end,text,position] when value = text.match(ATTR) return Token.new [:attr,text,position] when value = text.match(LPAREN) return Token.new [:lparen,text,position] when value = text.match(RPAREN) return Token.new [:rparen,text,position] else return Token.new [:identifier,text,position] end else x = @ss.getch return Token.new [x, x,position] end end
tokenize(str)
click to toggle source
# File lib/lexer.rb, line 72 def tokenize str @tokens=[] init(str) until @ss.eos? @tokens << next_token() end return @tokens[0..-2] end