class Astrapi::Lexer

Constants

ARROW

.….….….operators.….….….….….…..

ATTR
CLASS
COMMENT
END_
FLOAT
IDENT

.….….….….….….….….….….….…

IDENTIFIER

.….….….literals.….….….….….….

INT
LBRACK
LPAREN

.….….….punctuation.….….….….….…

LT
MODULE

.….….….….….….….….….….….…

NEWLINE

.….….….….….….….….….….….…

RBRACK
RPAREN
SPACE
STRING

Attributes

suppress_comment[RW]

Public Class Methods

new(str='') click to toggle source
# File lib/lexer.rb, line 61
def initialize str=''
  init(str)
  @suppress_space=true
  @suppress_comment=false
end

Public Instance Methods

init(str) click to toggle source
# File lib/lexer.rb, line 67
def init str
  @ss=StringScanner.new(str)
  @line=0
end
next_token() click to toggle source

next token can detect spaces length

# File lib/lexer.rb, line 82
def next_token

  if @ss.bol?
    @line+=1
    @old_pos=@ss.pos
  end

  position=[@line,@ss.pos-@old_pos+1]

  return :eos if @ss.eos?

  case
  when text = @ss.scan(NEWLINE)
    next_token()
  when text = @ss.scan(SPACE)
    next_token()
  when text = @ss.scan(COMMENT)
    next_token()
  when text = @ss.scan(ARROW)
    return Token.new [:arrow,text,position]
  when text = @ss.scan(LT)
    return Token.new [:lt,text,position]
  when text = @ss.scan(LBRACK)
    return Token.new [:lbrack,text,position]
  when text = @ss.scan(RBRACK)
    return Token.new [:rbrack,text,position]
  when text = @ss.scan(IDENTIFIER)
    case
    when value = text.match(IDENT)
      return Token.new [:IDENT,text,position]
    when value = text.match(FLOAT)
      return Token.new [:FLOAT,text,position]
    when value = text.match(INT)
      return Token.new [:INT,text,position]
    when value = text.match(STRING)
      return Token.new [:STRING,text,position]
    when value = text.match(MODULE)
      return Token.new [:module,text,position]
    when value = text.match(CLASS)
      return Token.new [:class,text,position]
    when value = text.match(END_)
      return Token.new [:end,text,position]
    when value = text.match(ATTR)
      return Token.new [:attr,text,position]
    when value = text.match(LPAREN)
      return Token.new [:lparen,text,position]
    when value = text.match(RPAREN)
      return Token.new [:rparen,text,position]
    else
      return Token.new [:identifier,text,position]
    end
  else
    x = @ss.getch
    return Token.new [x, x,position]
  end
end
tokenize(str) click to toggle source
# File lib/lexer.rb, line 72
def tokenize str
  @tokens=[]
  init(str)
  until @ss.eos?
    @tokens << next_token()
  end
  return @tokens[0..-2]
end