module Benchmark::Experiment

Constants

MEASURED_TIMES
VERSION

Public Instance Methods

aggregate_and_rank(jsons) click to toggle source
# File lib/benchmark/lab.rb, line 72
def aggregate_and_rank(jsons)
  return if jsons.empty?
  all_stats = jsons.inject({}) { |elem, hsh| hsh.merge(elem) }
  rank(all_stats)
end
experiment(sample_size, &blk) click to toggle source
# File lib/benchmark/lab.rb, line 78
def experiment(sample_size, &blk)
  all_stats = JSON.parse(observe_and_summarize(sample_size, &blk))
  print_stats(all_stats)

  best, is_the_best_significative = rank(all_stats)

  puts "The best \"#{best['label']}\" is #{is_the_best_significative ? '' : 'not '}significantly (95%) better (total time)."

  all_stats
end
iterative_experiment() click to toggle source
# File lib/benchmark/lab.rb, line 105
def iterative_experiment
end
observe_and_summarize(sample_size) { |job| ... } click to toggle source
# File lib/benchmark/lab.rb, line 64
def observe_and_summarize(sample_size, &blk)
  job = Job.new(0)
  yield(job)
  job.observe_and_summarize(sample_size)
  all_stats = job.list.map{ |label, _, _, stats| [label, stats] }.to_h
  all_stats.to_json
end
rank(all_stats, alpha = 0.05) click to toggle source
# File lib/benchmark/lab.rb, line 89
def rank(all_stats, alpha = 0.05)
  ranked = all_stats.map do |label, stats|
    total = stats.select{ |stat| stat['name'] == 'total' }.first
    total['label'] = label
    total
  end.sort_by { |stat| stat['median'] }
  is_h0_rejected = true
  if all_stats.size > 1
    z = Benchmark::Experiment::MannWhitneyUTest::calculate_z(ranked.first['sample'], ranked[1]['sample'])
    p_value = Benchmark::Experiment::MannWhitneyUTest::calculate_probability_z(z)
    is_h0_rejected = Benchmark::Experiment::MannWhitneyUTest::is_null_hypothesis_rejected?(p_value, alpha)
  end

  return ranked.first, is_h0_rejected
end

Private Instance Methods

label_width(all_stats) click to toggle source
# File lib/benchmark/lab.rb, line 139
def label_width(all_stats)
  label_widths = all_stats.map { |label, _| label.to_s.length }
  label_widths.minmax.last
end
print_stats(all_stats) click to toggle source