class GroongaQueryLog::ServerVerifier

Constants

GRN_CANCEL

Attributes

n_executed_commands[R]

Public Class Methods

new(options) click to toggle source
# File lib/groonga-query-log/server-verifier.rb, line 33
def initialize(options)
  @options = options
  @queue = SizedQueue.new(@options.request_queue_size)
  @events = Queue.new
  @n_executed_commands = 0
  @request_id_counter = 0
end

Public Instance Methods

verify(input, &callback) click to toggle source
# File lib/groonga-query-log/server-verifier.rb, line 41
def verify(input, &callback)
  @same = true
  @slow = false
  @client_error_is_occurred = false
  producer = run_producer(input, &callback)
  reporter = run_reporter
  producer.join
  @events.push(nil)
  reporter.join
  success?
end

Private Instance Methods

failed?() click to toggle source
# File lib/groonga-query-log/server-verifier.rb, line 165
def failed?
  not success?
end
generate_request_id() click to toggle source
# File lib/groonga-query-log/server-verifier.rb, line 173
def generate_request_id
  (@request_id_counter += 1).to_s
end
log_client_error(error) { || ... } click to toggle source
# File lib/groonga-query-log/server-verifier.rb, line 328
def log_client_error(error)
  $stderr.puts(Time.now.iso8601(6))
  yield if block_given?
  if error.respond_to?(:raw_error)
    target_error = error.raw_error
  else
    target_error = error
  end
  $stderr.puts("#{target_error.class}: #{target_error.message}")
  $stderr.puts(target_error.backtrace)
end
report_different(output, command, response1, response2) click to toggle source
# File lib/groonga-query-log/server-verifier.rb, line 294
def report_different(output, command, response1, response2)
  command_source = command.original_source || command.to_uri_format
  output.puts("command: #{command_source}")
  output.puts("response1: #{response1.body.to_json}")
  output.puts("response2: #{response2.body.to_json}")
  output.flush
end
report_error(output, command, error) click to toggle source
# File lib/groonga-query-log/server-verifier.rb, line 318
def report_error(output, command, error)
  command_source = command.original_source || command.to_uri_format
  output.puts("command: #{command_source}")
  error.backtrace.reverse_each do |trace|
    output.puts("backtrace: #{trace}")
  end
  output.puts("error: #{error.class}: #{error.message}")
  output.flush
end
report_slow(output, command, old_elapsed_time, new_elapsed_time, old_elapsed_times, new_elapsed_times) click to toggle source
# File lib/groonga-query-log/server-verifier.rb, line 302
def report_slow(output,
                command,
                old_elapsed_time,
                new_elapsed_time,
                old_elapsed_times,
                new_elapsed_times)
  command_source = command.original_source || command.to_uri_format
  output.puts("command: #{command_source}")
  output.puts("elapsed_time_old: #{old_elapsed_time}")
  output.puts("elapsed_time_new: #{new_elapsed_time}")
  output.puts("elapsed_times_old: #{old_elapsed_times.join(' ')}")
  output.puts("elapsed_times_new: #{new_elapsed_times.join(' ')}")
  output.puts("elapsed_time_ratio: #{new_elapsed_time / old_elapsed_time}")
  output.flush
end
rewrite_filter(command, name) click to toggle source
# File lib/groonga-query-log/server-verifier.rb, line 277
def rewrite_filter(command, name)
  target = command[name]
  return if target.nil?
  return unless @options.need_filter_rewrite?

  rewriter = FilterRewriter.new(target, @options.to_filter_rewriter_options)
  rewritten_target = rewriter.rewrite
  return if target == rewritten_target

  if @options.debug_rewrite?
    $stderr.puts("Rewritten #{name}")
    $stderr.puts("  Before: #{target}")
    $stderr.puts("   After: #{rewritten_target}")
  end
  command[name] = rewritten_target
end
run_consumer() click to toggle source
# File lib/groonga-query-log/server-verifier.rb, line 96
def run_consumer
  @options.groonga1.create_client do |groonga1_client|
    @options.groonga2.create_client do |groonga2_client|
      loop do
        statistic = @queue.pop
        return true if statistic.nil?
        next if stop?

        original_source = statistic.command.original_source
        begin
          verify_command(groonga1_client, groonga2_client,
                         statistic.command)
        rescue => error
          log_client_error(error) do
            $stderr.puts(original_source)
          end
          @client_error_is_occurred = true
          @events.push([:error, statistic.command, error])
          return false
        end
        if @options.verify_cache?
          command = Groonga::Command::Status.new
          begin
            verify_command(groonga1_client, groonga2_client,
                           command)
          rescue => error
            log_client_error(error) do
              $stderr.puts("status after #{original_source}")
            end
            @client_error_is_occurred = true
            @events.push([:error, command, error])
            return false
          end
        end
      end
    end
  end
end
run_consumers() click to toggle source
# File lib/groonga-query-log/server-verifier.rb, line 86
def run_consumers
  @options.n_clients.times.collect do
    Thread.new do
      loop do
        break if run_consumer
      end
    end
  end
end
run_producer(input, &callback) click to toggle source
# File lib/groonga-query-log/server-verifier.rb, line 54
def run_producer(input, &callback)
  Thread.new do
    consumers = run_consumers

    parser = Parser.new
    callback_per_n_commands = 100
    parser.parse(input) do |statistic|
      break if stop?

      command = statistic.command
      next if command.nil?
      next unless target_command?(command)
      next if rand < @options.omit_rate
      @n_executed_commands += 1
      @queue.push(statistic)

      if callback and (@n_executed_commands % callback_per_n_commands).zero?
        @options.n_clients.times do
          @queue.push(nil)
        end
        consumers.each(&:join)
        callback.call
        consumers = run_consumers
      end
    end
    @options.n_clients.times do
      @queue.push(nil)
    end
    consumers.each(&:join)
  end
end
run_reporter() click to toggle source
# File lib/groonga-query-log/server-verifier.rb, line 135
def run_reporter
  Thread.new do
    @options.open_output do |output|
      loop do
        event = @events.pop
        break if event.nil?
        case event[0]
        when :different
          report_different(output, *event[1..-1])
        when :slow
          report_slow(output, *event[1..-1])
        when :error
          report_error(output, *event[1..-1])
        end
      end
    end
  end
end
stop?() click to toggle source
# File lib/groonga-query-log/server-verifier.rb, line 169
def stop?
  @options.stop_on_failure? and failed?
end
success?() click to toggle source
# File lib/groonga-query-log/server-verifier.rb, line 158
def success?
  return false unless @same
  return false if @slow
  return false if @client_error_is_occurred
  true
end
target_command?(command) click to toggle source
# File lib/groonga-query-log/server-verifier.rb, line 154
def target_command?(command)
  @options.target_command_name?(command.command_name)
end
verify_command(groonga1_client, groonga2_client, command) click to toggle source
# File lib/groonga-query-log/server-verifier.rb, line 177
def verify_command(groonga1_client, groonga2_client, command)
  command["cache"] = "no" if @options.disable_cache?
  command["cache"] = "no" if @options.verify_performance?
  command["request_id"] = generate_request_id if @options.verify_cancel?

  if @options.max_limit >= 0 and command["limit"]
    limit = command["limit"].to_i
    if limit >= 0
      command["limit"] = [limit, @options.max_limit].min.to_s
    else
      command["limit"] = @options.max_limit.to_s
    end
  end
  command["output_type"] = "json"
  rewrite_filter(command, "filter")
  rewrite_filter(command, "scorer")

  response1 = groonga1_client.execute(command)
  # groonga2 is new Groonga.
  response2 = nil
  if @options.verify_cancel?
    sleep_thread = nil
    request = groonga2_client.execute(command) do |response|
      response2 = response
      sleep_thread.kill
    end

    # Randomize timing of sending request_cancel command
    sleep_thread = Thread.new do
      sleep(rand(0..@options.cancel_max_wait))
    end
    sleep_thread.join
    unless response2
      @options.groonga2.create_client do |cancel_client|
        cancel_client.execute("request_cancel", id: command.request_id)
      end
    end
    request.wait

    return if response2.return_code == GRN_CANCEL
  else
    response2 = groonga2_client.execute(command)
  end

  compare_options = {
    :care_order => @options.care_order,
    :ignored_drilldown_keys => @options.ignored_drilldown_keys,
  }
  comparer = ResponseComparer.new(command, response1, response2,
                                  compare_options)
  unless comparer.same?
    @same = false
    @events.push([:different, command, response1, response2])
    return
  end

  if @options.verify_performance?
    verify_performance(command,
                       groonga1_client,
                       groonga2_client,
                       response1,
                       response2)
  end
end
verify_performance(command, groonga1_client, groonga2_client, response1, response2) click to toggle source
# File lib/groonga-query-log/server-verifier.rb, line 242
def verify_performance(command,
                       groonga1_client,
                       groonga2_client,
                       response1,
                       response2)
  responses1 = [response1]
  responses2 = [response2]
  verifier = PerformanceVerifier.new(command,
                                     responses1,
                                     responses2,
                                     @options.performance_verifier_options)
  return unless verifier.slow?

  n_tries = 4
  n_tries.times do
    return if stop?

    responses1 << groonga1_client.execute(command)
    responses2 << groonga2_client.execute(command)
    verifier = PerformanceVerifier.new(command,
                                       responses1,
                                       responses2,
                                       @options.performance_verifier_options)
    return unless verifier.slow?
  end

  @slow = true
  @events.push([:slow,
                command,
                verifier.old_elapsed_time,
                verifier.new_elapsed_time,
                verifier.old_elapsed_times,
                verifier.new_elapsed_times])
end