module Evoc::Evaluate
Public Class Methods
ap(rec:,exp: nil)
click to toggle source
# File lib/evoc/evaluate.rb, line 246 def self.ap(rec:,exp: nil) # AP is 0 for the empty list if rec.is_a?(Array) && rec.empty? # array and empty return nil end self.validateInput(rec) i = 0 correct_i = 0 ap = 0 rec.each do |cluster| cluster.each do |item| i = i + 1 correct_i = correct_i + item precision_i = correct_i/i ap = ap + (precision_i*item) end end if exp.nil? exp = correct_i else if correct_i > exp raise ArgumentError, "Found more relevant items than the provided number of relevant items" end end return (exp == 0 ? 0 : (ap/exp).to_f) end
applicable(rec:)
click to toggle source
# File lib/evoc/evaluate.rb, line 52 def self.applicable(rec:) if rec.is_a?(Array) (rec <=> []).abs else raise Evoc::Exceptions::FormatError.new "Wrong format given to #{__method__}, expected an array, input was: #{input}" end end
average_precision(recommendation,expected_outcome)
click to toggle source
calculate the average precision of the result based on an expected outcome @param [Array] recommendation a sorted array @param [Array] expected_outcome an array of items @return [Float] the average precision
# File lib/evoc/evaluate.rb, line 281 def self.average_precision(recommendation,expected_outcome) raise Error.new "#average_precision has been deprecated, use #ap instead" if !expected_outcome.is_a?(Array) then expected_outcome = [expected_outcome] end if (expected_outcome.size > 0) & !recommendation.empty? average_precision = 0 correct_items = [] total_items_considered = [] # sort rules by weight # we first group rules with equal weights # and then sort the groups by weight recommendation.each do |items| if !items.is_a?(Array) then items = [items] end if items.first.class != expected_outcome.first.class raise ArgumentError, "Expected outcome was of type #{expected_outcome.first.class}, while the item in the recommendation was of type #{items.first.class}" end # skip already considered items if (new_items = items - total_items_considered).size > 0 new_items.each {|item| total_items_considered << item} if correct_in_rule = (items & expected_outcome) if correct_in_rule.size > 0 # make sure that the new items havent already been added earlier new_correct = (correct_in_rule - correct_items) # add new items new_correct.each {|item| correct_items << item} change_in_recall = new_correct.size.to_r/expected_outcome.size precision_at_k = correct_items.size.to_r/total_items_considered.size average_precision += (precision_at_k * change_in_recall) end end end end average_precision.to_f else nil end end
discernibility(rec:)
click to toggle source
# File lib/evoc/evaluate.rb, line 32 def self.discernibility(rec:) # AP is 0 for the empty list if rec.is_a?(Array) && rec.empty? # array and empty return nil end self.validateInput(rec) rec_size = 0 rec_clusters = 0 rec.each do |c| rec_clusters = rec_clusters + 1 c.each do |e| rec_size = rec_size + 1 end end return (rec_clusters/rec_size).to_f end
f1(rec:,exp:)
click to toggle source
@return the f1 score (preision/recall harmonic mean)
# File lib/evoc/evaluate.rb, line 62 def self.f1(rec:,exp:) # AP is 0 for the empty list if rec.is_a?(Array) && rec.empty? # array and empty return nil end self.validateInput(rec) rec_size = 0 rec_correct = 0 rec.each do |c| c.each do |e| rec_size = rec_size + 1 rec_correct = rec_correct + e end end return (2*rec_correct/(rec_size + exp)).to_f end
first_relevant(rec:)
click to toggle source
@return the rank of the first relevant itemjk
# File lib/evoc/evaluate.rb, line 105 def self.first_relevant(rec:) # AP is 0 for the empty list if rec.is_a?(Array) && rec.empty? # array and empty return nil end self.validateInput(rec) last_checked = 1 rec.each do |c| c.each do |e| if e == 1 return last_checked end last_checked = last_checked + 1 end end return nil end
last_relevant(rec:)
click to toggle source
@return the rank of the last relevant itemjk
# File lib/evoc/evaluate.rb, line 126 def self.last_relevant(rec:) # AP is 0 for the empty list if rec.is_a?(Array) && rec.empty? # array and empty return nil end self.validateInput(rec) size = rec.inject(0) {|sum,c| sum + c.size} last_checked = size rec.reverse_each do |c| c.reverse_each do |e| if e == 1 return last_checked end last_checked = last_checked - 1 end end return nil end
mean_confidence(rules:)
click to toggle source
# File lib/evoc/evaluate.rb, line 23 def self.mean_confidence(rules:) if rules.empty? then return nil end return (rules.inject(0) {|sum,r| sum + r.m_confidence.value}/rules.size).to_f end
mean_confidence10(rules:)
click to toggle source
# File lib/evoc/evaluate.rb, line 28 def self.mean_confidence10(rules:) return self.mean_confidence(rules: Evoc::RuleStore.sort_on(rules: rules,measures: ['m_confidence']).take(10).flatten.take(10)) end
mean_support(rules:)
click to toggle source
# File lib/evoc/evaluate.rb, line 14 def self.mean_support(rules:) if rules.empty? then return nil end return (rules.inject(0) {|sum,r| sum + r.m_support.value}/rules.size).to_f end
mean_support10(rules:)
click to toggle source
# File lib/evoc/evaluate.rb, line 19 def self.mean_support10(rules:) return self.mean_support(rules: Evoc::RuleStore.sort_on(rules: rules,measures: ['m_support']).take(10).flatten.take(10)) end
precision(rec:,exp: nil)
click to toggle source
# File lib/evoc/evaluate.rb, line 164 def self.precision(rec:,exp: nil) # AP is 0 for the empty list if rec.is_a?(Array) && rec.empty? # array and empty return nil end self.validateInput(rec) size_rec = rec.inject(0) {|sum,c| sum + c.size} num_correct_in_rec = rec.inject(0) {|sum,c| sum + c.inject(&:+)} return (num_correct_in_rec/size_rec).to_f end
precision10(rec:,exp: nil)
click to toggle source
# File lib/evoc/evaluate.rb, line 155 def self.precision10(rec:,exp: nil) # AP is 0 for the empty list if rec.is_a?(Array) && rec.empty? # array and empty return nil end self.validateInput(rec) return self.precision(rec: [rec.take(10).flatten.take(10)]) end
recall(rec:,exp: nil)
click to toggle source
# File lib/evoc/evaluate.rb, line 177 def self.recall(rec:,exp: nil) # AP is 0 for the empty list if rec.is_a?(Array) && rec.empty? # array and empty return nil end self.validateInput(rec) num_correct_in_rec = rec.inject(0) {|sum,c| sum + c.inject(&:+)} if exp.nil? return num_correct_in_rec else if num_correct_in_rec > exp raise ArgumentError, "Found more relevant items than the provided number of relevant items" end return (num_correct_in_rec/exp).to_f end end
recall10(rec:,exp: nil)
click to toggle source
# File lib/evoc/evaluate.rb, line 146 def self.recall10(rec:,exp: nil) # AP is 0 for the empty list if rec.is_a?(Array) && rec.empty? # array and empty return nil end self.validateInput(rec) return self.recall(rec: [rec.take(10).flatten.take(10)],exp: exp) end
relevant_ranks(rec:)
click to toggle source
@return an array containg the rank of each consequtive expected outcome
# File lib/evoc/evaluate.rb, line 83 def self.relevant_ranks(rec:) # AP is 0 for the empty list if rec.is_a?(Array) && rec.empty? # array and empty return [] end self.validateInput(rec) ranks = [] last_checked = 1 rec.each do |c| c.each do |e| if e == 1 ranks << last_checked end last_checked = last_checked + 1 end end return ranks end
t_ap(rec:,exp: nil)
click to toggle source
r_p : relevant items in previous groups i_p : index previous group r_g : relevant items in group n_g : items in group i : index of current item
# File lib/evoc/evaluate.rb, line 205 def self.t_ap(rec:,exp: nil) # AP is 0 for the empty list if rec.is_a?(Array) && rec.empty? # array and empty return nil end self.validateInput(rec) ap = 0 r_p = 0 i_p = 0 rec.each do |cluster| r_g = cluster.inject(&:+).to_r n_g = cluster.size.to_r cluster.each_with_index do |_,i| i = i_p + i + 1 chance_relevant = r_g/n_g avg_previous_rel = if (n_g == 1) (r_p + 1) * (1/i) else (r_p + (i - i_p - 1)*((r_g-1)/(n_g-1)) + 1) * (1/i) end item_ap_contribution = chance_relevant * avg_previous_rel ap = ap + item_ap_contribution end r_p = r_p + r_g i_p = i_p + n_g end # if the number of relevant documents is not supplied # assume that the recommendation contains all relevant documents if exp.nil? exp = r_p else if r_p > exp raise ArgumentError, "Found more relevant items than the provided number of relevant items" end end return (r_p == 0 ? 0 : (ap/exp).to_f) end
validateInput(input)
click to toggle source
# File lib/evoc/evaluate.rb, line 5 def self.validateInput(input) # verify format if !input.is_a?(Array) || # not an array !input.first.is_a?(Array) || # not containg an array ![0,1].include?(input.first.first) # items are not 0s and 1s raise Evoc::Exceptions::FormatError.new "Wrong format given to #{__method__}, expected list of list of 0s and 1s, input was: #{input}" end end