class Demo::RubyTokenizer

Constants

RESERVED_WORDS
RESERVED_WORDS_PATTERN

Public Instance Methods

tokenize(str, index = 0) { |*args| ... } click to toggle source
# File gtk3/sample/gtk-demo/main.rb, line 485
def tokenize(str, index = 0)
  until str.empty?
    tag = nil

    case str
    when /".+?"/, /'.+?'/
      tag = :string
    when /#.*$/
      tag = :comment
    when RESERVED_WORDS_PATTERN
      tag = :reserved
    when /[A-Z][A-Za-z0-9_]+/
      tag = :const
    end

    if tag
      tokenize($LAST_MATCH_INFO.pre_match, index) do |*args|
        yield(*args)
      end
      yield(tag, index + $LAST_MATCH_INFO.begin(0), index + $LAST_MATCH_INFO.end(0))
      index += (str.length - $LAST_MATCH_INFO.post_match.length)
      str = $LAST_MATCH_INFO.post_match
    else
      index += str.length
      str = ""
    end
  end
end