class GroongaQueryLog::Command::RunRegressionTest

Public Class Methods

new() click to toggle source
# File lib/groonga-query-log/command/run-regression-test.rb, line 35
def initialize
  @input_directory = Pathname.new(".")
  @working_directory = Pathname.new(".")

  @n_clients = 1

  @old_groonga = "groonga"
  @old_database = "db.old/db"
  @old_groonga_options = []
  @old_groonga_env = {}
  @old_groonga_warm_up_commands = []

  @new_groonga = "groonga"
  @new_database = "db.new/db"
  @new_groonga_options = []
  @new_groonga_env = {}
  @new_groonga_warm_up_commands = []

  @recreate_database = false
  @warm_up = true
  @load_data = true
  @run_queries = true
  @skip_finished_queries = false
  @output_query_log = false
  @stop_on_failure = false
  @rewrite_vector_equal = false
  @rewrite_vector_not_equal_empty_string = false
  @vector_accessors = []
  @rewrite_nullable_reference_number = false
  @nullable_reference_number_accessors = []
  @rewrite_not_or_regular_expression = false
  @rewrite_and_not_operator = false
  @debug_rewrite = false
  @omit_rate = 0.0
  @max_limit = -1
  @verify_cancel = false
  @cancel_max_wait = 5.0

  @care_order = true
  @ignored_drilldown_keys = []
  @target_command_names = ServerVerifier::Options.new.target_command_names

  @verify_performance = false
  @performance_verfifier_options = PerformanceVerifier::Options.new

  @read_timeout = Groonga::Client::Default::READ_TIMEOUT

  @notifier_options = {
    mail_subject_on_start: "Start",
    mail_subject_on_success: "Success",
    mail_subject_on_failure: "Failure",
    mail_from: "groonga-query-log@#{Socket.gethostname}",
    mail_to: nil,
    mail_only_on_failure: false,
    smtp_server: "localhost",
    smtp_auth_user: nil,
    smtp_auth_password: nil,
    smtp_starttls: false,
    smtp_port: 25,
  }
end

Public Instance Methods

run(command_line) click to toggle source
# File lib/groonga-query-log/command/run-regression-test.rb, line 97
def run(command_line)
  option_parser = create_option_parser
  begin
    option_parser.parse!(command_line)
  rescue OptionParser::ParseError => error
    $stderr.puts(error.message)
    return false
  end

  notifier = MailNotifier.new(@notifier_options)
  notifier.notify_started

  start_time = Time.now
  tester = Tester.new(old_groonga_server,
                      new_groonga_server,
                      tester_options)
  success = tester.run
  elapsed_time = Time.now - start_time
  n_leaked_objects = tester.new.n_leaked_objects

  report = format_report(success,
                         elapsed_time,
                         n_leaked_objects,
                         tester.n_executed_commands)
  notifier.notify_finished(success, report)
  puts(report)

  success and n_leaked_objects.zero?
end

Private Instance Methods

create_option_parser() click to toggle source
# File lib/groonga-query-log/command/run-regression-test.rb, line 135
def create_option_parser
  parser = OptionParser.new
  parser.version = VERSION

  parser.separator("")
  parser.separator("Path:")
  parser.on("--input-directory=DIRECTORY",
            "Load schema and data from DIRECTORY.",
            "(#{@input_directory})") do |directory|
    @input_directory = Pathname.new(normalize_path(directory))
  end
  parser.on("--working-directory=DIRECTORY",
            "Use DIRECTORY as working directory.",
            "(#{@working_directory})") do |directory|
    @working_directory = Pathname.new(normalize_path(directory))
  end

  parser.separator("")
  parser.separator("Throughput:")
  parser.on("--n-clients=N", Integer,
            "Use N clients concurrently.",
            "(#{@n_clients})") do |n|
    @n_clients = n
  end

  parser.separator("")
  parser.separator("Old Groonga:")
  parser.on("--old-groonga=GROONGA",
            "Old groonga command",
            "(#{@old_groonga})") do |groonga|
    @old_groonga = groonga
  end

  parser.on("--old-groonga-option=OPTION",
            "Add an additional old groonga option",
            "You can specify this option multiple times",
            "to specify multiple groonga options",
            "(no options)") do |groonga_option|
    @old_groonga_options << groonga_option
  end

  parser.on("--old-groonga-env=KEY=VALUE",
            "Use KEY=VALUE environment variable for old groonga",
            "You can specify this option multiple times",
            "to specify multiple environment variables",
            "(no environment variables)") do |env|
    key, value = env.split("=", 2)
    @old_groonga_env[key] = value
  end

  parser.on("--old-groonga-warm-up-commands=COMMAND",
            "Run COMMAND before running tests to warm old groonga up",
            "You can specify this option multiple times",
            "to specify multiple warm up commands",
            "(no additional warm up commands)") do |command|
    @old_groonga_warm_up_commands << command
  end

  parser.separator("")
  parser.separator("New Groonga:")
  parser.on("--new-groonga=GROONGA",
            "New groonga command",
            "(#{@new_groonga})") do |groonga|
    @new_groonga = groonga
  end

  parser.on("--new-groonga-option=OPTION",
            "Add an additional new groonga option",
            "You can specify this option multiple times",
            "to specify multiple groonga options",
            "(no options)") do |groonga_option|
    @new_groonga_options << groonga_option
  end

  parser.on("--new-groonga-env=KEY=VALUE",
            "Use KEY=VALUE environment variable for new groonga",
            "You can specify this option multiple times",
            "to specify multiple environment variables",
            "(no environment variables)") do |env|
    key, value = env.split("=", 2)
    @new_groonga_env[key] = value
  end

  parser.on("--new-groonga-warm-up-commands=COMMAND",
            "Run COMMAND before running tests to warm new groonga up",
            "You can specify this option multiple times",
            "to specify multiple warm up commands",
            "(no additional warm up commands)") do |command|
    @new_groonga_warm_up_commands << command
  end

  parser.separator("")
  parser.separator("Operations:")
  parser.on("--recreate-database",
            "Always recreate Groonga database") do
    @recreate_database = true
  end
  parser.on("--no-warm-up",
            "Don't warm up before test",
            "(#{@warm_up})") do |boolean|
    @warm_up = boolean
  end
  parser.on("--no-load-data",
            "Don't load data. Just loads schema to Groonga database",
           "(#{@load_data})") do |boolean|
    @load_data = boolean
  end
  parser.on("--no-run-queries",
            "Don't run queries. Just creates Groonga database",
           "(#{@run_queries})") do |boolean|
    @run_queries = boolean
  end
  parser.on("--skip-finished-queries",
            "Don't run finished query logs.") do
    @skip_finished_queries = true
  end
  parser.on("--output-query-log",
            "Output query log in verified target Groonga servers") do
    @output_query_log = true
  end
  parser.on("--[no-]stop-on-failure",
            "Stop execution on the first failure",
            "(#{@stop_on_failure})") do |boolean|
    @stop_on_failure = boolean
  end
  parser.on("--[no-]rewrite-vector-equal",
            "Rewrite 'vector == ...' with 'vector @ ...'",
            "(#{@rewrite_vector_equal})") do |boolean|
    @rewrite_vector_equal = boolean
  end
  parser.on("--[no-]rewrite-vector-not-equal-empty-string",
            "Rewrite 'vector != \"\"' and " +
            "'vector.column != \"\"' " +
            "with 'vector_size(vector) > 0'",
            "(#{@rewrite_vector_not_equal_empty_string})") do |boolean|
    @rewrite_vector_not_equal_empty_string = boolean
  end
  parser.on("--vector-accessor=ACCESSOR",
            "Mark ACCESSOR as rewrite vector targets",
            "You can specify multiple vector accessors by",
            "specifying this option multiple times") do |accessor|
    @vector_accessors << accessor
  end
  parser.on("--[no-]rewrite-nullable-reference-number",
            "Rewrite 'nullable_reference.number' with " +
            "with '(nullable_reference._key == null ? 0 : " +
            "nullable_reference.number)'",
            "(#{@rewrite_nullable_reference_number})") do |boolean|
    @rewrite_nullable_reference_number = boolean
  end
  parser.on("--nullable-reference-number-accessor=ACCESSOR",
            "Mark ACCESSOR as rewrite nullable reference number targets",
            "You can specify multiple accessors by",
            "specifying this option multiple times") do |accessor|
    @nullable_reference_number_accessors << accessor
  end
  parser.on("--[no-]rewrite-not-or-regular-expression",
            "Rewrite 'column1 @ \"keyword1\" && column2 @~ " +
            "\"^(?!.*keyword2|keyword3|...).+$\"' " +
            "with 'column1 @ \"keyword1\" &! column2 @ \"keyword2\" " +
            "&! column2 @ \"keyword3\" &! ...'",
            "(#{@rewrite_not_or_regular_expression})") do |boolean|
    @rewrite_not_or_regular_expression = boolean
  end
  parser.on("--[no-]rewrite-and-not-operator",
            "Rewrite '(column1 @ \"keyword1\") && !(column2 @ " +
            "\"keyword2\")' " +
            "with '(column1 @ \"keyword1\") &! (column2 @ " +
            "\"keyword2\")'",
            "(#{@rewrite_and_not_operator})") do |boolean|
    @rewrite_and_not_operator = boolean
  end
  parser.on("--[no-]debug-rewrite",
            "Output rewrite logs for debugging",
            "(#{@debug_rewrite})") do |boolean|
    @debug_rewrite = boolean
  end
  parser.on("--omit-rate=RATE", Float,
            "You can specify rate for omitting execution queries." +
            "For example, if you specify 0.9 in this option, " +
            "execute queries with the probability of 1/10.",
            "(#{@omit_rate})") do |rate|
    @omit_rate = rate
  end
  parser.on("--max-limit=LIMIT", Integer,
            "Use LIMIT as the max limit value",
            "Negative value doesn't rewrite the limit parameter",
            "(#{@max_limit})") do |limit|
    @max_limit = limit
  end

  parser.separator("")
  parser.separator("Comparisons:")
  parser.on("--no-care-order",
            "Don't care order of select response records") do
    @care_order = false
  end
  parser.on("--ignore-drilldown-key=KEY",
            "Don't compare drilldown result for KEY",
            "You can specify multiple drilldown keys by",
            "specifying this option multiple times") do |key|
    @ignored_drilldown_keys << key
  end
  target_command_names_label = @target_command_names.join(",")
  parser.on("--target-command-names=NAME1,NAME2,...", Array,
            "Test only NAME1,NAME2,... commands",
            "You can use glob to choose command name",
            "[#{target_command_names_label}]") do |names|
    @target_command_names = names
  end

  parser.separator("")
  parser.separator("Performance:")
  parser.on("--[no-]verify-performance",
            "Whether verify performance or not",
            "[#{@verify_performance}]") do |boolean|
    @verify_performance = boolean
  end
  available_choose_strategies =
    @performance_verfifier_options.available_choose_strategies
  default_choose_strategy =
    @performance_verfifier_options.choose_strategy
  parser.on("--performance-choose-strategy=STRATEGY",
            available_choose_strategies,
            "How to choose elapsed time",
            "(#{available_choose_strategies.join(", ")})",
            "[#{default_choose_strategy}]") do |strategy|
    @performance_verfifier_options.choose_strategy = strategy
  end

  parser.separator("")
  parser.separator("Network:")
  parser.on("--read-timeout=TIMEOUT", Integer,
            "Timeout on reading response from Groonga servers.",
            "You can disable timeout by specifying -1.",
            "[#{@read_timeout}]") do |timeout|
    @read_timeout = timeout
  end

  parser.separator("")
  parser.separator("Notifications:")
  parser.on("--smtp-server=SERVER",
            "Use SERVER as SMTP server",
            "(#{@notifier_options[:smtp_server]})") do |server|
    @notifier_options[:smtp_server] = server
  end
  parser.on("--smtp-auth-user=USER",
            "Use USER for SMTP AUTH",
            "(#{@notifier_options[:smtp_auth_user]})") do |user|
    @notifier_options[:smtp_auth_user] = user
  end
  parser.on("--smtp-auth-password=PASSWORD",
            "Use PASSWORD for SMTP AUTH",
            "(#{@notifier_options[:smtp_auth_password]})") do |password|
    @notifier_options[:smtp_auth_password] = password
  end
  parser.on("--[no-]smtp-starttls",
            "Whether use StartTLS in SMTP or not",
            "(#{@notifier_options[:smtp_starttls]})") do |boolean|
    @notifier_options[:smtp_starttls] = boolean
  end
  parser.on("--smtp-port=PORT", Integer,
            "Use PORT as SMTP server port",
            "(#{@notifier_options[:smtp_port]})") do |port|
    @notifier_options[:smtp_port] = port
  end
  parser.on("--mail-from=FROM",
            "Send a notification e-mail from FROM",
            "(#{@notifier_options[:mail_from]})") do |from|
    @notifier_options[:mail_from] = from
  end
  parser.on("--mail-to=TO",
            "Send a notification e-mail to TO",
            "(#{@notifier_options[:mail_to]})") do |to|
    @notifier_options[:mail_to] = to
  end
  parser.on("--mail-subject-on-start=SUBJECT",
            "Use SUBJECT as subject for notification e-mail on start",
            "(#{@notifier_options[:mail_subject_on_start]})") do |subject|
    @notifier_options[:mail_subject_on_start] = subject
  end
  parser.on("--mail-subject-on-success=SUBJECT",
            "Use SUBJECT as subject for notification e-mail on success",
            "(#{@notifier_options[:mail_subject_on_success]})") do |subject|
    @notifier_options[:mail_subject_on_success] = subject
  end
  parser.on("--mail-subject-on-failure=SUBJECT",
            "Use SUBJECT as subject for notification e-mail on failure",
            "(#{@notifier_options[:mail_subject_on_failure]})") do |subject|
    @notifier_options[:mail_subject_on_failure] = subject
  end
  parser.on("--[no-]mail-only-on-failure",
            "Send a notification e-mail only on failure",
            "(#{@notifier_options[:mail_only_on_failure]})") do |boolean|
    @notifier_options[:mail_only_on_failure] = boolean
  end
  parser.on("--[no-]verify-cancel",
            "Verify cancellation",
            "(#{@verify_cancel})") do |boolean|
    @verify_cancel = boolean
  end
  parser.on("--cancel-max-wait=SECONDS", Float,
            "Used with --verify_cancel. " +
            "You can specify the maximum number of seconds to wait " +
            "before sending request_cancel command. " +
            "For example, if you specify 5.0 in this option, " +
            "wait randomly between 0~5.0 seconds before sending request_cancel command.",
            "(#{@cancel_max_wait})") do |seconds|
    @cancel_max_wait = seconds
  end
  parser
end
directory_options() click to toggle source
# File lib/groonga-query-log/command/run-regression-test.rb, line 452
def directory_options
  {
    :input_directory   => @input_directory,
    :working_directory => @working_directory,
    :results_directory => results_directory,
  }
end
format_elapsed_time(elapsed_time) click to toggle source
# File lib/groonga-query-log/command/run-regression-test.rb, line 550
def format_elapsed_time(elapsed_time)
  elapsed_seconds = elapsed_time % 60
  elapsed_minutes = elapsed_time / 60 % 60
  elapsed_hours = elapsed_time / 60 / 60 % 24
  elapsed_days = elapsed_time / 60 / 60 / 24
  "Elapsed: %ddays %02d:%02d:%02d\n" % [
    elapsed_days,
    elapsed_hours,
    elapsed_minutes,
    elapsed_seconds
  ]
end
format_report(success, elapsed_time, n_leaked_objects, n_executed_commands) click to toggle source
# File lib/groonga-query-log/command/run-regression-test.rb, line 523
def format_report(success,
                  elapsed_time,
                  n_leaked_objects,
                  n_executed_commands)
  formatted = format_elapsed_time(elapsed_time)
  formatted << "The number of executed commands: #{n_executed_commands}\n"
  if success
    formatted << "Success"
    formatted << " but leaked" if n_leaked_objects > 0
  else
    formatted << "Failure"
    formatted << " and leaked" if n_leaked_objects > 0
  end
  formatted << "\n"
  unless n_leaked_objects.zero?
    formatted << "\nLeaked: #{n_leaked_objects}\n"
  end
  unless success
    output = StringIO.new
    formetter = FormatRegressionTestLogs.new(output: output)
    formetter.run([results_directory])
    formatted << "Report:\n"
    formatted << output.string
  end
  formatted
end
new_groonga_server() click to toggle source
# File lib/groonga-query-log/command/run-regression-test.rb, line 513
def new_groonga_server
  options = server_options
  options[:warm_up_commands] = @new_groonga_warm_up_commands
  GroongaServer.new(@new_groonga,
                    @new_groonga_options,
                    @new_groonga_env,
                    @new_database,
                    options)
end
normalize_path(path) click to toggle source
# File lib/groonga-query-log/command/run-regression-test.rb, line 128
def normalize_path(path)
  if File::ALT_SEPARATOR
    path = path.gsub(File::ALT_SEPARATOR, File::SEPARATOR)
  end
  path
end
old_groonga_server() click to toggle source
# File lib/groonga-query-log/command/run-regression-test.rb, line 503
def old_groonga_server
  options = server_options
  options[:warm_up_commands] = @old_groonga_warm_up_commands
  GroongaServer.new(@old_groonga,
                    @old_groonga_options,
                    @old_groonga_env,
                    @old_database,
                    options)
end
results_directory() click to toggle source
# File lib/groonga-query-log/command/run-regression-test.rb, line 448
def results_directory
  @working_directory + "results"
end
server_options() click to toggle source
# File lib/groonga-query-log/command/run-regression-test.rb, line 460
def server_options
  options = {
    :load_data             => @load_data,
    :output_query_log      => @output_query_log,
    :recreate_database     => @recreate_database,
    :run_queries           => @run_queries,
    :warm_up               => @warm_up,
  }
  directory_options.merge(options)
end
tester_options() click to toggle source
# File lib/groonga-query-log/command/run-regression-test.rb, line 471
def tester_options
  options = {
    :n_clients  => @n_clients,
    :care_order => @care_order,
    :skip_finished_queries => @skip_finished_queries,
    :ignored_drilldown_keys => @ignored_drilldown_keys,
    :stop_on_failure => @stop_on_failure,
    :rewrite_vector_equal => @rewrite_vector_equal,
    :rewrite_vector_not_equal_empty_string =>
      @rewrite_vector_not_equal_empty_string,
    :vector_accessors => @vector_accessors,
    :rewrite_nullable_reference_number =>
      @rewrite_nullable_reference_number,
    :nullable_reference_number_accessors =>
      @nullable_reference_number_accessors,
    :rewrite_not_or_regular_expression =>
      @rewrite_not_or_regular_expression,
    :rewrite_and_not_operator =>
      @rewrite_and_not_operator,
    :debug_rewrite => @debug_rewrite,
    :omit_rate => @omit_rate,
    :max_limit => @max_limit,
    :target_command_names => @target_command_names,
    :verify_performance => @verify_performance,
    :performance_verfifier_options => @performance_verfifier_options,
    :read_timeout => @read_timeout,
    :verify_cancel => @verify_cancel,
    :cancel_max_wait => @cancel_max_wait,
  }
  directory_options.merge(options)
end