class Benchmark::Sweet::Job
abstract notion of a job
Constants
- ALL_METRICS
- DATABASE_METRICS
metrics calculated by the database test suite
- HIGHER_BETTER
- IPS_METRICS
metrics calculated by the ips test suite
- MEMORY_METRICS
metrics calculated by the memory test suite
Attributes
@returns [Hash<String,Hash<String,Stat>>] entries[metric] = value
lambda used to group metrics that should be compared The lambda takes the label as an argument and returns a unique object per comparison group NOTE: This lambda takes a label hash as an argument
While other lambdas in this api take a comparison object
a symbol is assumed to refer to the label @return [Nil|Lambda] lambda for grouping
@returns [Array<Job::Item>] list of report items to run
@option options :quiet [Boolean] true to suppress the display of interim test calculations @option options :warmup [Number] For ips tests, the amount of time to warmup @option options :time [Number] For ips tests, the amount of time to the calculations @option options :metrics [String|Symbol,Array<String|Symbol] list of metrics to run TODO: :confidence
Public Class Methods
# File lib/benchmark/sweet/job.rb, line 38 def initialize(options = {}) @options = options @options[:metrics] ||= IPS_METRICS + %w() validate_metrics(@options[:metrics]) @items = [] @entries = {} @symbolize_keys = false # load / save @filename = nil # display @grouping = nil @report_options = {} @report_block = nil # current item metadata @meta = {} end
Public Instance Methods
report results
# File lib/benchmark/sweet/job.rb, line 131 def add_entry(label, metric, stat) (@entries[metric] ||= {})[label] = stat.respond_to?(:central_tendency) ? stat : create_stats(stat) end
&block - a lambda that accepts a label and a stats object returns a unique object for each set of metrics that should be compared with each other
unfortunatly, this currently has a different signature than all other lambdas at this time, there are no comparisons created yet. so it is hard to pass one in example:
x.compare_by { |label, value| label[:data] } x.compare_by :data
# File lib/benchmark/sweet/job.rb, line 104 def compare_by(*symbol, &block) @grouping = symbol.empty? ? block : Proc.new { |label, value| symbol.map { |s| label[s] } } end
of note, this groups with @grouping (defined by group_by) but then all data continues to the next step this allows you to make comparisons across rows / columns / grouping
# File lib/benchmark/sweet/job.rb, line 212 def comparison_values relevant_entries.flat_map do |metric_name, metric_entries| # TODO: map these to Comparison(metric_name, label, stats) So we only have 1 type of lambda partitioned_metrics = grouping ? metric_entries.group_by(&grouping) : {nil => metric_entries} partitioned_metrics.flat_map do |grouping_name, grouped_metrics| sorted = grouped_metrics.sort_by { |n, e| e.central_tendency } sorted.reverse! if HIGHER_BETTER.include?(metric_name) _best_label, best_stats = sorted.first total = sorted.count # TODO: fix ranking. i / total doesn't work as well when there is only 1 entry or some entries are the same sorted.each_with_index.map { |(label, stats), i| Comparison.new(metric_name, label, stats, i, total, best_stats) } end end end
# File lib/benchmark/sweet/job.rb, line 55 def configure(options) @options.merge!(options) end
@returns [Boolean] true to run database queries tests
# File lib/benchmark/sweet/job.rb, line 64 def database? ; !(relevant_metric_names & DATABASE_METRICS).empty? ; end
# File lib/benchmark/sweet/job.rb, line 201 def display_report(comparisons) if !@report_block || @report_block.arity == 2 Benchmark::Sweet.table(comparisons, **@report_options, &@report_block) else @report_block.call(comparisons) end end
# File lib/benchmark/sweet/job.rb, line 135 def entry_stat(label, metric) @entries.dig(metric, label) end
@returns [Boolean] true to run tests for data that has already been processed
# File lib/benchmark/sweet/job.rb, line 70 def force? ; options[:force] ; end
@returns [Boolean] true to run iterations per second tests
# File lib/benchmark/sweet/job.rb, line 60 def ips? ; !(relevant_metric_names & IPS_METRICS).empty? ; end
items to run (typical benchmark/benchmark-ips use case)
# File lib/benchmark/sweet/job.rb, line 76 def item(label, action = nil, &block) # could use Benchmark::IPS::Job::Entry current_meta = label.kind_of?(Hash) ? @meta.merge(label) : @meta.merge(method: label) @items << Item.new(current_meta, action || block) end
if we are using symbols as keys for our labels
# File lib/benchmark/sweet/job.rb, line 127 def labels_have_symbols! end
serialization
# File lib/benchmark/sweet/job.rb, line 144 def load_entries(filename = @filename) # ? have ips save / load their own data? return unless filename && File.exist?(filename) require "json" JSON.load(IO.read(filename)).each do |v| n = v["name"] n.symbolize_keys! add_entry n, v["metric"], v["samples"] end end
@returns [Boolean] true to run memory tests
# File lib/benchmark/sweet/job.rb, line 62 def memory? ; !(relevant_metric_names & MEMORY_METRICS).empty? ; end
# File lib/benchmark/sweet/job.rb, line 83 def metadata(options) @old_meta = @meta @meta = @meta.merge(options) return unless block_given? yield @meta = @old_meta end
@returns [Boolean] true to suppress the display of interim test calculations
# File lib/benchmark/sweet/job.rb, line 67 def quiet? ; options[:quiet] ; end
# File lib/benchmark/sweet/job.rb, line 139 def relevant_entries relevant_metric_names.map { |n| [n, @entries[n] ] } end
@returns [Array<String>] List of metrics to compare
# File lib/benchmark/sweet/job.rb, line 73 def relevant_metric_names ; options[:metrics] ; end
Setup the testing framework TODO: would be easier to debug if these were part of run_report
@keyword :grouping [Symbol|lambda|nil] proc with parameters label, stat that generates grouping names
defaults to the compare_by value
@keyword :sort [Boolean] true to sort the rows (default false). NOTE: grouping names ARE sorted @keyword :row [Symbol|lambda] a lambda (default - display the full label) @keyword :column [Symbol|lambda] (default :metric) @keyword :value (default :comp_short - the value and delta)
for color, consider passing `value: ->(m){ m.comp_short("\033[#{m.color}m#{m[field]}\e[0m") }`
# File lib/benchmark/sweet/job.rb, line 117 def report_with(args = {}, &block) @report_options = args @report_block = block # Assume the display grouping is the same as comparison grouping unless an explicit value was provided if !args.key?(:grouping) && @grouping args[:grouping] = @grouping.respond_to?(:call) ? -> v { @grouping.call(v.label, v.stats) } : @grouping end end
# File lib/benchmark/sweet/job.rb, line 181 def run # run metrics if they are requested and haven't run yet # only run the suites that provide the data the user needs. # if the first node has the data, assumes all do # # TODO: may want to override these values run_ips if ips? && (force? || !@entries.dig(IPS_METRICS.first, items.first.label)) run_memory if memory? && (force? || !@entries.dig(MEMORY_METRICS.first, items.first.label)) run_queries if database? && (force? || !@entries.dig(DATABASE_METRICS.first, items.first.label)) end
? metric => label(:version, :method) => stats ? label(:metric, :version, :method) => stats @returns [Hash<String,Hash<String,Comparison>>] Same as entries, but contains comparisons not Stats
# File lib/benchmark/sweet/job.rb, line 195 def run_report comparison_values.tap do |results| display_report(results) end end
# File lib/benchmark/sweet/job.rb, line 157 def save_entries(filename = @filename) return unless filename require "json" # sanity checking symbol_value = false data = @entries.flat_map do |metric_name, metric_values| metric_values.map do |label, stat| # warnings symbol_values ||= label.kind_of?(Hash) && label.values.detect { |v| v.nil? || v.kind_of?(Symbol) } { 'name' => label, 'metric' => metric_name, 'samples' => stat.samples, # extra data like measured_us, iter, and others? } end end puts "", "Warning: Please use strings or numbers for label hash values (not nils or symbols). Symbols are not JSON friendly." if symbol_value IO.write(filename, JSON.pretty_generate(data) << "\n") end
# File lib/benchmark/sweet/job.rb, line 91 def save_file(filename) @filename = filename end
Private Instance Methods
# File lib/benchmark/sweet/job.rb, line 239 def create_stats(samples) Benchmark::IPS::Stats::SD.new(Array(samples)) end
# File lib/benchmark/sweet/job.rb, line 231 def validate_metrics(metric_options) if !(invalid = metric_options - ALL_METRICS).empty? $stderr.puts "unknown metrics: #{invalid.join(", ")}" $stderr.puts "choose: #{(ALL_METRICS).join(", ")}" raise IllegalArgument, "unknown metric: #{invalid.join(", ")}" end end