class DerailedBenchmarks::StatsFromDir

A class used to read several benchmark files it will parse each file, then sort by average time of benchmarks. It can be used to find the fastest and slowest examples and give information about them such as what the percent difference is and if the results are statistically significant

Example:

branch_info = {}
branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
stats = DerailedBenchmarks::StatsFromDir.new(branch_info)

stats.newest.average  # => 10.5
stats.oldest.average  # => 11.0
stats.significant?    # => true
stats.x_faster        # => "1.0476"

Constants

FORMAT

Attributes

newest[R]
oldest[R]
stats[R]

Public Class Methods

new(hash) click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 29
def initialize(hash)
  @files = []

  hash.each do |branch, info_hash|
    file = info_hash.fetch(:file)
    desc = info_hash.fetch(:desc)
    time = info_hash.fetch(:time)
    @files << StatsForFile.new(file: file, desc: desc, time: time, name: branch)
  end
  @files.sort_by! { |f| f.time }
  @oldest = @files.first
  @newest = @files.last
end

Public Instance Methods

align() click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 99
def align
  " " * (percent_faster.to_s.index(".") - x_faster.to_s.index("."))
end
banner(io = Kernel) click to toggle source
call() click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 43
def call
  @files.each(&:call)

  stats_95 = statistical_test(confidence: 95)

  # If default check is good, see if we also pass a more rigorous test
  # if so, then use the more rigourous test
  if stats_95[:alternative]
    stats_99 = statistical_test(confidence: 99)
    @stats = stats_99 if stats_99[:alternative]
  end
  @stats ||= stats_95

  self
end
change_direction() click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 91
def change_direction
  if faster?
    "FASTER 🚀🚀🚀"
  else
    "SLOWER 🐢🐢🐢"
  end
end
d_critical() click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 75
def d_critical
  @stats[:d_critical].to_f
end
d_max() click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 71
def d_max
  @stats[:d_max].to_f
end
faster?() click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 83
def faster?
  newest.median < oldest.median
end
percent_faster() click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 87
def percent_faster
  (((oldest.median - newest.median) / oldest.median).to_f  * 100)
end
significant?() click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 67
def significant?
  @stats[:alternative]
end
statistical_test(series_1=oldest.values, series_2=newest.values, confidence: 95) click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 59
def statistical_test(series_1=oldest.values, series_2=newest.values, confidence: 95)
  StatisticalTest::KSTest.two_samples(
    group_one: series_1,
    group_two: series_2,
    alpha: (100 - confidence) / 100.0
  )
end
x_faster() click to toggle source
# File lib/derailed_benchmarks/stats_from_dir.rb, line 79
def x_faster
  (oldest.median/newest.median).to_f
end