class IDL::Scanner
Constants
- ALPHA_LC
- ALPHA_UC
- ANNOTATION
- ANNOTATION_STR
- BOOL_LITERALS
- BREAKCHARS
- DIGITS
- DOT
- ESCTBL
- FULL_IDCHARS
- HEXCHARS
- IDCHARS
- KEYWORDS
- LFCR
- LITERALS
- OCTALS
- Position
- SHIFTCHARS
- SIGNS
- SPACES
- WHITESPACE
Public Class Methods
new(src, directiver, params = {})
click to toggle source
# File lib/ridl/scanner.rb, line 225 def initialize(src, directiver, params = {}) @includepaths = params[:includepaths] || [] @xincludepaths = params[:xincludepaths] || [] @stack = [] @expansions = [] @prefix = nil @directiver = directiver @directiver.instance_variable_get('@d').instance_variable_set('@scanner', self) @defined = TokenRegistry.new # initialize with predefined macros if params[:macros] params[:macros].each do |(name, value)| @defined[name] = value end end @ifdef = [] @ifskip = false @ifnest = 0 i = nil nm = '' case src when String i = StrIStream.new(src) nm = '<string>' when File i = src nm = src.path when IO, StringIO i = src nm = '<io>' else parse_error "illegal type for input source: #{src.class} " end @in = In.new(i, nm) @scan_comment = false # true if parsing commented annotation @in_annotation = false # true if parsing annotation end
Public Instance Methods
check_include(path, fname)
click to toggle source
# File lib/ridl/scanner.rb, line 284 def check_include(path, fname) fp = path + fname File.file?(fp) && File.readable?(fp) end
do_parse?()
click to toggle source
# File lib/ridl/scanner.rb, line 348 def do_parse? @ifdef.empty? || @ifdef.last end
enter_expansion(src, define)
click to toggle source
# File lib/ridl/scanner.rb, line 312 def enter_expansion(src, define) IDL.log(2, "** RIDL - enter_expansion > #{define} = #{src}") @stack << [:define, nil, nil, @in, nil] @expansions << define @in = In.new(StrIStream.new(src), @in.position.name, @in.position.line, @in.position.column) end
enter_include(src, all = true)
click to toggle source
# File lib/ridl/scanner.rb, line 293 def enter_include(src, all = true) if @directiver.is_included?(src) @directiver.declare_include(src) else fpath = find_include(src, all) if fpath.nil? parse_error "Cannot open include file '#{src}'" end @stack << [:include, @prefix, @ifdef, @in, @ifskip] # record file dir as new searchpath @xincludepaths << (File.dirname(fpath) + '/') @prefix = nil @ifdef = [] @in = In.new(File.open(fpath, 'r'), fpath) @directiver.enter_include(src, fpath) @directiver.pragma_prefix(nil) end end
eval_directive(s)
click to toggle source
# File lib/ridl/scanner.rb, line 805 def eval_directive(s) IDL.log(3, "** RIDL - eval_directive(#{s})") rc = eval(s) case rc when FalseClass, TrueClass rc when Numeric rc != 0 else parse_error 'invalid preprocessor expression.' end end
extract_annotation()
click to toggle source
# File lib/ridl/scanner.rb, line 453 def extract_annotation annotation_body = nil # next token should be '(' in case of normal/single value annotation # or anything else in case of marker annotation skip_spaces # skip till next non-space or eol if peek_next == '(' token = next_token # parse '(' begin # identifier or value (in case of single value annotation) expected token = next_token if token.first == ')' # marker annotation; leave body empty annotation_body = { } else parse_error 'annotation member expected!' unless token.first == :identifier || is_literal?(token.first) s1 = token.last token = next_token # ')' (in case of single value annotation) or '=' if token.first == ')' parse_error 'invalid annotation member' if annotation_body annotation_body = { value: s1 } else parse_error 'invalid annotation member' unless token.first == '=' token, annotation_value = extract_annotation_value parse_error 'invalid annotation body' unless token.first == ',' || token.first == ')' (annotation_body ||= {})[s1.to_s] = annotation_value end end end until token.first == ')' token = next_token_before_eol else token = next_token_before_eol # marker annotation or symbolic value; leave body nil end [token, annotation_body] end
extract_annotation_value()
click to toggle source
# File lib/ridl/scanner.rb, line 422 def extract_annotation_value token = next_token # needs '{' (array) or literal or identifier (which means nested annotation object or enum value) if token.first == '{' # extract array of values (literals or identifiers) separated by ',' annotation_value = [] begin token, ann_value = extract_annotation_value parse_error 'invalid annotation value array' unless token.first == ',' || token.first == '}' annotation_value << ann_value end until token.first == '}' token = next_token elsif token.first == :identifier member_annotation_id = token.last.to_s # get nested body token, member_annotation_body = extract_annotation # determin vaue type; if it has a body it is an annotation instance annotation_value = if member_annotation_body { member_annotation_id => member_annotation_body } else # otherwise it is a symbolic value member_annotation_id.to_sym end # get next token if needed token = next_token unless token else parse_error 'invalid annotation member' unless is_literal?(token.first) annotation_value = token.last token = next_token end [token, annotation_value] end
find_include(fname, all = true)
click to toggle source
# File lib/ridl/scanner.rb, line 263 def find_include(fname, all = true) if File.file?(fname) && File.readable?(fname) File.expand_path(fname) else # search transient include paths if allowed (quoted includes) fp = if all then @xincludepaths.find do |p| check_include(p, fname) end else nil end # search system include paths if still needed fp = @includepaths.find do |p| check_include(p, fname) end unless fp fp += fname if fp fp end end
getline()
click to toggle source
# File lib/ridl/scanner.rb, line 708 def getline s = '' while true ch = @in.lookc break if ch.nil? case when (ch == "\"") # " s << @in.getc # opening quote while true if @in.lookc == "\\" # escape sequence s << @in.getc _, escstr = next_escape_str(true) s << escstr elsif @in.lookc == "\"" # " break elsif @in.lookc # normal character s << @in.getc else parse_error 'unterminated string literal' end end s << @in.getc # closing quote when (ch == "\'") # ' # quoted character s << @in.getc # opening quote if @in.lookc == "\\" # escape sequence s << @in.getc _, escstr = next_escape_str(true) s << escstr elsif @in.lookc && @in.lookc != "\'" # ' # normal character s << @in.getc end if @in.lookc != "\'" # ' parse_error "character literal must be single character enclosed in \"'\"" end s << @in.getc # closing quote when LFCR.include?(ch) @in.skipwhile { |ch_| LFCR.include? ch_ } break when ch == '/' @in.skipc if @in.lookc == '/' # //-style comment; skip till eol @in.gets break elsif @in.lookc == '*' # /*...*/ style comment; skip comment ch1 = nil @in.skipuntil { |ch_| ch0 = ch1; ch1 = ch_ ch0 == '*' and ch1 == '/' # } if @in.lookc.nil? parse_error "cannot find comment closing brace (\'*/\'). " end @in.skipc else s << ch end when ch == "\\" @in.skipc if LFCR.include?(@in.lookc) # line continuation @in.skipwhile { |ch_| LFCR.include? ch_ } if @in.lookc.nil? parse_error "line continuation character ('\\') not allowed as last character in file." end else s << ch end else @in.skipc s << ch end end s end
in_expansion?()
click to toggle source
# File lib/ridl/scanner.rb, line 327 def in_expansion? more_source? and @stack.last[0] == :define end
is_expanded?(define)
click to toggle source
# File lib/ridl/scanner.rb, line 319 def is_expanded?(define) @expansions.include?(define) end
is_literal?(o)
click to toggle source
# File lib/ridl/scanner.rb, line 418 def is_literal?(o) LITERALS.include?(o) end
leave_source()
click to toggle source
# File lib/ridl/scanner.rb, line 331 def leave_source # make sure to close the input source @in.close # check if we have any previous source still stacked up unless @stack.empty? if @stack.last[0] == :include @xincludepaths.pop # remove directory of finished include @directiver.leave_include _, @prefix, @ifdef, @in, @ifskip = @stack.pop @directiver.pragma_prefix(@prefix) else _, _, _, @in, _ = @stack.pop @expansions.pop end end end
more_source?()
click to toggle source
# File lib/ridl/scanner.rb, line 323 def more_source? !@stack.empty? end
next_escape()
click to toggle source
# File lib/ridl/scanner.rb, line 575 def next_escape ret = 0 case (ch = @in.getc) when nil parse_error 'illegal escape sequence' when '0'..'7' ret = '' ret << ch 1.upto(2) { ch = @in.lookc if ('0'..'7').include? ch ret << ch else break end @in.skipc } ret = ret.oct when 'x' # i'm not sure '\x' should be 0 or 'x'. currently returns 0. ret = '' 1.upto(2) { ch = @in.lookc if HEXCHARS.include? ch ret << ch else break end @in.skipc } ret = ret.hex when 'u' ret = '' 1.upto(4) { ch = @in.lookc if HEXCHARS.include? ch ret << ch else break end @in.skipc } ret = ret.hex when 'n', 't', 'v', 'b', 'r', 'f', 'a' ret = ESCTBL[ch] else ret = ('' << ch).unpack('C').first end ret end
next_escape_str(keep_type_ch = false)
click to toggle source
# File lib/ridl/scanner.rb, line 625 def next_escape_str(keep_type_ch = false) ret = 0 case (ch = @in.getc) when nil parse_error 'illegal escape sequence' when '0'..'7' ret = '' ret << ch 1.upto(2) { ch = @in.lookc if ('0'..'7').include? ch ret << ch else break end @in.skipc } ret = [ :oct, ret ] when 'x' # i'm not sure '\x' should be 0 or 'x'. currently returns 0. ret = '' ret << ch if keep_type_ch 1.upto(2) { ch = @in.lookc if HEXCHARS.include? ch ret << ch else break end @in.skipc } ret = [ :hex2, ret ] when 'u' ret = '' ret << ch if keep_type_ch 1.upto(4) { ch = @in.lookc if HEXCHARS.include? ch ret << ch else break end @in.skipc } ret = [ :hex4, ret ] when 'n', 't', 'v', 'b', 'r', 'f', 'a' ret = '' ret << ch ret = [:esc, ret] else ret = '' ret << ch ret = [:esc_ch, ch] end ret end
next_identifier(first = nil)
click to toggle source
# File lib/ridl/scanner.rb, line 529 def next_identifier(first = nil) @in.mark(first) while true if FULL_IDCHARS.include?(@in.lookc) @in.skipc else break end end s0 = @in.getregion # raw IDL id s1 = s0.downcase.to_sym # downcased symbolized s2 = s0.dup # (to be) unescaped id # simple check if s2.empty? parse_error 'identifier expected!' else if s2[0] == '_' s2.slice!(0) ## if starts with CORBA IDL escape => remove end parse_error "identifier must begin with alphabet character: #{s2}" unless ALPHA_LC.include?(s2[0]) || ALPHA_UC.include?(s2[0]) end # preprocessor check if @defined.has_key?(s2) and !is_expanded?(s2) # enter expansion as new source enter_expansion(@defined[s2], s2) # call next_token to parse expanded source next_token # keyword check elsif @in_annotation if BOOL_LITERALS.has_key?(s1) [:boolean_literal, BOOL_LITERALS[s1]] else [:identifier, Identifier.new(s2, s2, s0)] end elsif (a = KEYWORDS.assoc(s1)).nil? # check for language mapping keyword [:identifier, Identifier.new(s2, chk_identifier(s2), s0)] elsif s0 == a[1] [a[1], nil] else parse_error "'#{s0}' collides with IDL keyword '#{a[1]}'" end end
next_token()
click to toggle source
# File lib/ridl/scanner.rb, line 969 def next_token sign = nil str = '' # initialize empty string while true ch = @in.getc if ch.nil? if !@ifdef.empty? and !in_expansion? parse_error 'mismatched #if/#endif' end if more_source? leave_source next else return [false, nil] end end if WHITESPACE.include? ch @in.skipwhile { |c| WHITESPACE.include?(c) } next end if str.empty? && ch == "\#" parse_directive next end unless do_parse? skipline next end str << ch case when BREAKCHARS.include?(ch) if SHIFTCHARS.include?(ch) && @in.lookc == ch # '<<' or '>>' str << @in.getc end return [str, str] when ch == ANNOTATION if @in_annotation return [str, str] else # return token returned by parse_annotation or parse next token recursively return parse_annotation || next_token end when ch == ':' # if @in.lookc == ':' # @in.skipc return %w(:: ::) else return %w(: :) end when ch == 'L' _nxtc = @in.lookc if _nxtc == "\'" # ' #single quote, for a character literal. @in.skipc # skip 'L' _nxtc = @in.lookc ret = if _nxtc == "\\" @in.skipc next_escape_str elsif _nxtc == "\'" # ' [ nil, nil ] else [:char, '' << @in.getc] end if @in.lookc != "\'" # ' parse_error "wide character literal must be single wide character enclosed in \"'\"" end @in.skipc return [:wide_character_literal, ret] elsif _nxtc == "\"" # " #double quote, for a string literal. ret = [] chs = '' @in.skipc # skip 'L' while true _nxtc = @in.lookc if _nxtc == "\\" @in.skipc ret << [:char, chs] unless chs.empty? chs = '' ret << next_escape_str elsif _nxtc == "\"" # " @in.skipc ret << [:char, chs] unless chs.empty? return [:wide_string_literal, ret] else chs << @in.getc end end else return next_identifier(ch) end when IDCHARS.include?(ch) return next_identifier(ch) when ch == '/' # _nxtc = @in.lookc if _nxtc == '*' # skip comment like a `/* ... */' @in.skipc # forward stream beyond `/*' ch1 = nil @in.skipuntil { |ch_| ch0 = ch1; ch1 = ch_ ch0 == '*' and ch1 == '/' # } if @in.lookc.nil? parse_error "cannot find comment closing brace (\'*/\'). " end @in.skipc str = '' # reset next elsif _nxtc == '/' # skip comment like a `// ...\n' @in.skipc unless @scan_comment # scan_comment will be true when parsing commented annotations _nxtc = @in.lookc if _nxtc == ANNOTATION @in.skipc # return token returned by parse_annotation or parse next token recursively return parse_annotation(true) || next_token else @in.skipuntil { |c| LFCR.include?(c) } end end str = '' # reset next else return %w(/ /) end when SIGNS.include?(ch) _nxtc = @in.lookc if DIGITS.include? _nxtc sign = ch str = '' # reset next else return [str, str] end when ('1'..'9').include?(ch) @in.mark(sign, ch) @in.skipwhile { |c| DIGITS.include?(c) } num_type = (['.', 'e', 'E', 'd', 'D'].include?(@in.lookc)) ? skipfloat_or_fixed : :integer_literal r = @in.getregion if num_type == :floating_pt_literal return [:floating_pt_literal, r.to_f] elsif num_type == :fixed_pt_literal return [:fixed_pt_literal, r] else return [:integer_literal, r.to_i] end when ch == DOT # @in.mark(ch) @in.skipwhile { |c| DIGITS.include?(c) } num_type = (DOT != @in.lookc) ? skipfloat_or_fixed : nil s = @in.getregion if s == '.' parse_error 'token consisting of single dot (.) is invalid.' end if num_type == :floating_pt_literal return [:floating_pt_literal, s.to_f] elsif num_type == :fixed_pt_literal return [:fixed_pt_literal, s] else parse_error 'invalid floating point constant.' end when ch == '0' @in.mark(sign, ch) _nxtc = @in.lookc if _nxtc == 'x' || _nxtc == 'X' @in.skipc @in.skipwhile { |ch_| HEXCHARS.include? ch_ } s = @in.getregion return [:integer_literal, s.hex] else dec = false @in.skipwhile { |c| OCTALS.include?(c) } if ('8'..'9').include? @in.lookc dec = TRUE @in.skipwhile { |c| DIGITS.include?(c) } end num_type = (['.', 'e', 'E', 'd', 'D'].include?(@in.lookc)) ? skipfloat_or_fixed : :integer_literal s = @in.getregion ret = if num_type == :floating_pt_literal [:floating_pt_literal, s.to_f] elsif num_type == :fixed_pt_literal [:fixed_pt_literal, s] elsif dec parse_error "decimal literal starting with '0' should be octal ('0'..'7' only): #{s}" else [:integer_literal, s.oct] end return ret end when ch == "\'" # ' #single quote, for a character literal. _nxtc = @in.lookc ret = if _nxtc == "\\" @in.skipc next_escape elsif _nxtc == "\'" # ' 0 elsif _nxtc ('' << @in.getc).unpack('C').first end if @in.lookc != "\'" # ' parse_error "character literal must be single character enclosed in \"'\"" end @in.skipc return [:character_literal, ret] when ch == "\"" # " #double quote, for a string literal. ret = '' while true _nxtc = @in.lookc if _nxtc == "\\" @in.skipc ret << next_escape elsif _nxtc == "\"" # " @in.skipc return [:string_literal, ret] elsif _nxtc ret << @in.getc else parse_error 'unterminated string literal' end end else parse_error 'illegal character [' << ch << ']' end # of case end # of while parse_error 'unexcepted error' end
next_token_before_eol()
click to toggle source
# File lib/ridl/scanner.rb, line 964 def next_token_before_eol @in.skipwhile { |c| SPACES.include?(c) } LFCR.include?(@in.lookc) ? nil : next_token end
parse_annotation(in_comment = false)
click to toggle source
# File lib/ridl/scanner.rb, line 488 def parse_annotation(in_comment = false) @in_annotation = true @scan_comment = in_comment begin # parse (possibly multiple) annotation(s) begin annotation_position = self.position.dup # next token should be identifier (must be on same line following '@') token = next_token parse_error 'annotation identifier expected!' unless token.first == :identifier annotation_id = token.last.to_s token, annotation_body = extract_annotation # pass annotation to directiver for processing @directiver.define_annotation(annotation_id, annotation_position, in_comment, annotation_body || {}) end until token.nil? || token.first != ANNOTATION_STR ensure @in_annotation = false @scan_comment = false end # check identifier for keywords if token&.first == :identifier # keyword check if (a = KEYWORDS.assoc(token.last.to_s)).nil? token = [:identifier, Identifier.new(token.last.to_s, chk_identifier(token.last.to_s), token.last.unescaped_name)] elsif token.last == a[1] token = [a[1], nil] else parse_error "'#{token.last}' collides with a keyword '#{a[1]}'" end end token end
parse_directive()
click to toggle source
# File lib/ridl/scanner.rb, line 818 def parse_directive @in.skipwhile { |c| SPACES.include?(c) } s = getline /^(\w*)\s*/ === s s1 = $1 s2 = $' # ' if /(else|endif|elif)/ === s1 if @ifdef.empty? parse_error '#else/#elif/#endif must not appear without preceding #if' end case s1 when 'else' if @ifnest.zero? if @ifskip # true branch has already been parsed @ifdef[@ifdef.size - 1] = false else @ifdef[@ifdef.size - 1] ^= true @ifskip = @ifdef.last end end when 'endif' if @ifnest.zero? @ifdef.pop @ifskip = @ifdef.last else @ifnest -= 1 end else if @ifnest.zero? if @ifskip || @ifdef[@ifdef.size - 1] # true branch has already been parsed so skip from now on @ifdef[@ifdef.size - 1] = false @ifskip = true else while s2 =~ /(^|[\W])defined\s*\(\s*(\w+)\s*\)/ def_id = $2 s2.gsub!(/(^|[\W])(defined\s*\(\s*\w+\s*\))/, '\1' + "#{@defined.has_key?(def_id).to_s}") end s2.gsub!(/(^|[\W])([A-Za-z_][\w]*)/) do |_| "#{$1}#{resolve_define($2)}" end begin @ifdef[@ifdef.size - 1] = eval_directive(s2) @ifskip = @ifdef[@ifdef.size - 1] rescue IDL::ParseError raise rescue => e p e puts e.backtrace.join("\n") parse_error 'error evaluating #elif' end end end end elsif /(if|ifn?def)/ === s1 if /ifn?def/ === s1 if do_parse? parse_error 'no #if(n)def target.' unless /^(\w+)/ === s2 @ifdef.push(@defined[$1].nil? ^ (s1 == 'ifdef')) @ifskip = @ifdef.last else @ifnest += 1 end else # when 'if' if do_parse? # match 'defined(Foo)' or 'defined Foo' while s2 =~ /(^|[\W])defined(\s*\(\s*(\w+)\s*\)|\s+(\w+))/ IDL.log(3, "** RIDL - parse_directive : resolving 'defined(#{$3 || $4})'") def_id = $3 || $4 # resolve 'defined' expression to 'true' or 'false' according to actual macro definition s2.gsub!(/(^|[\W])(defined\s*[\s\(]\s*#{def_id}(\s*\))?)/, '\1' + "#{@defined.has_key?(def_id).to_s}") end # match and resolve any macro variables listed in conditional expression s2.gsub!(/(^|[\W])([A-Za-z_][\w]*)/) do |_| "#{$1}#{resolve_define($2)}" end begin @ifdef.push(eval_directive(s2)) @ifskip = @ifdef.last rescue IDL::ParseError raise rescue => e p e puts e.backtrace.join("\n") parse_error 'error evaluating #if' end else @ifnest += 1 end end elsif do_parse? case s1 when 'pragma' parse_pragma(s2) when 'error' parse_error(s2) when 'define' defid = s2.split.first parse_error 'no #define target.' unless defid parse_error "#{defid} is already #define-d." if @defined[defid] defval = s2.sub(/^\s*#{defid}/, '').strip defval = true if defval.empty? @defined[defid] = defval when 'undef' @defined.delete(s2) when 'include' if s2[0, 1] == '"' || s2[0, 1] == '<' quoted_inc = (s2[0, 1] == '"') if s2.size > 2 s2.strip! s2 = s2.slice(1..(s2.size - 2)) else s2 = '' end end enter_include(s2, quoted_inc) when /[0-9]+/ # ignore line directive else parse_error "unknown directive: #{s}." end end end
parse_error(msg, ex = nil)
click to toggle source
# File lib/ridl/scanner.rb, line 356 def parse_error(msg, ex = nil) e = IDL::ParseError.new(msg, positions) e.set_backtrace(ex.backtrace) unless ex.nil? raise e end
parse_pragma(s)
click to toggle source
# File lib/ridl/scanner.rb, line 950 def parse_pragma(s) case s when /^ID\s+(.*)\s+"(.*)"\s*$/ @directiver.pragma_id($1.strip, $2) when /^version\s+(.*)\s+([0-9]+)\.([0-9]+)\s*$/ @directiver.pragma_version($1.strip, $2, $3) when /^prefix\s+"(.*)"\s*$/ @prefix = $1 @directiver.pragma_prefix(@prefix) else @directiver.handle_pragma(s) end end
peek_next()
click to toggle source
# File lib/ridl/scanner.rb, line 521 def peek_next @in.lookc end
position()
click to toggle source
# File lib/ridl/scanner.rb, line 289 def position @in.position end
positions()
click to toggle source
# File lib/ridl/scanner.rb, line 352 def positions @stack.reverse.inject(@in.nil? ? [] : [@in.position]) { |pos_arr, (_, _, _, in_, _)| pos_arr << in_.position } end
resolve_define(id, stack = [])
click to toggle source
# File lib/ridl/scanner.rb, line 790 def resolve_define(id, stack = []) return id if %w(true false).include?(id) IDL.log(3, "*** RIDL - resolve_define(#{id})") if @defined.has_key?(id) define_ = @defined[id] stack << id parse_error("circular macro reference detected for [#{define_}]") if stack.include?(define_) # resolve any nested macro definitions define_.gsub(/(^|[\W])([A-Za-z_][\w]*)/) do |_| "#{$1}#{resolve_define($2, stack)}" end else '0' # unknown id end end
skip_spaces()
click to toggle source
# File lib/ridl/scanner.rb, line 525 def skip_spaces @in.skipwhile { |c| SPACES.include?(c) } end
skipfloat_or_fixed()
click to toggle source
# File lib/ridl/scanner.rb, line 681 def skipfloat_or_fixed if @in.lookc == DOT @in.skipc @in.skipwhile { |c| DIGITS.include?(c) } end if ['e', 'E'].include? @in.lookc @in.skipc @in.skipc if SIGNS.include? @in.lookc @in.skipwhile { |c| DIGITS.include?(c) } return :floating_pt_literal elsif ['d', 'D'].include? @in.lookc @in.skipc @in.skipc if SIGNS.include? @in.lookc @in.skipwhile { |c| DIGITS.include?(c) } return :fixed_pt_literal end :floating_pt_literal end
skipline()
click to toggle source
# File lib/ridl/scanner.rb, line 700 def skipline while true s = @in.gets until s.chomp!.nil?; end break unless s[s.length - 1] == "\\" end end