class RDoc::Parser::RipperStateLex

# File lib/rdoc/parser/ripper_state_lex.rb, line 33
def self.end?(token)
  (token[:state] & EXPR_END)
end

# File lib/rdoc/parser/ripper_state_lex.rb, line 38
def initialize(code)
  @buf = []
  @heredoc_queue = []
  # Use Prism.lex_compat for Ripper-compatible tokenization
  lex_result = Prism.lex_compat(code)
  prism_tokens = lex_result.value.map do |(pos, kind, text, state)|
    line_no, char_no = pos
    # Convert Ripper::Lexer::State to integer to avoid Ripper dependency
    state_int = state.respond_to?(:to_i) ? state.to_i : state
    Token.new(line_no, char_no, kind, text, state_int)
  end

  # Prism.lex_compat omits :on_sp tokens, so we need to insert them for proper
  # syntax highlighting and token stream reconstruction
  tokens_with_spaces = insert_space_tokens(prism_tokens, code)

  # Fix Prism incompatibility: Prism returns :on_ignored_nl after `def foo; end`
  # but parsers expect :on_nl for proper token collection in single-line methods
  @tokens = normalize_ignored_nl_for_single_line_methods(tokens_with_spaces)
end

# File lib/rdoc/parser/ripper_state_lex.rb, line 20
def self.parse(code)
  lex = self.new(code)
  tokens = []
  begin
    while tk = lex.get_squashed_tk
      tokens.push tk
    end
  rescue StopIteration
  end
  tokens
end

# File lib/rdoc/parser/ripper_state_lex.rb, line 59
def get_squashed_tk
  if @buf.empty?
    tk = @tokens.shift
  else
    tk = @buf.shift
  end
  return nil if tk.nil?
  case tk[:kind]
  when :on_symbeg
    tk = get_symbol_tk(tk)
  when :on_tstring_beg
    tk = get_string_tk(tk)
  when :on_backtick
    if (tk[:state] & (EXPR_FNAME | EXPR_ENDFN)) != 0
      tk[:kind] = :on_ident
      tk[:state] = EXPR_ARG
    else
      tk = get_string_tk(tk)
    end
  when :on_regexp_beg
    tk = get_regexp_tk(tk)
  when :on_embdoc_beg
    tk = get_embdoc_tk(tk)
  when :on_heredoc_beg
    @heredoc_queue << retrieve_heredoc_info(tk)
  when :on_nl, :on_ignored_nl, :on_comment, :on_heredoc_end
    if !@heredoc_queue.empty?
      get_heredoc_tk(*@heredoc_queue.shift)
    elsif tk[:text].nil? # :on_ignored_nl sometimes gives nil
      tk[:text] = ''
    end
  when :on_words_beg, :on_qwords_beg, :on_symbols_beg, :on_qsymbols_beg
    tk = get_words_tk(tk)
  when :on_op
    if '&.' == tk[:text]
      tk[:kind] = :on_period
    else
      tk = get_op_tk(tk)
    end
  end
  tk
end