module Evoc::Evaluate

Public Class Methods

ap(rec:,exp: nil) click to toggle source
# File lib/evoc/evaluate.rb, line 246
def self.ap(rec:,exp: nil)
  # AP is 0 for the empty list
  if rec.is_a?(Array) && rec.empty? # array and empty
    return nil
  end
  self.validateInput(rec)

  i = 0
  correct_i = 0
  ap = 0

  rec.each do |cluster|
    cluster.each do |item|
      i = i + 1
      correct_i = correct_i + item
      precision_i = correct_i/i
      ap = ap + (precision_i*item) 
    end
  end

  if exp.nil?
    exp = correct_i
  else
    if correct_i > exp
      raise ArgumentError, "Found more relevant items than the provided number of relevant items"
    end
  end
  return (exp == 0 ? 0 : (ap/exp).to_f)

end
applicable(rec:) click to toggle source
# File lib/evoc/evaluate.rb, line 52
def self.applicable(rec:)
  if rec.is_a?(Array)
    (rec <=> []).abs
  else
    raise Evoc::Exceptions::FormatError.new "Wrong format given to #{__method__}, expected an array, input was: #{input}"
  end
end
average_precision(recommendation,expected_outcome) click to toggle source

calculate the average precision of the result based on an expected outcome @param [Array] recommendation a sorted array @param [Array] expected_outcome an array of items @return [Float] the average precision

# File lib/evoc/evaluate.rb, line 281
def self.average_precision(recommendation,expected_outcome)
    raise Error.new "#average_precision has been deprecated, use #ap instead"
    if !expected_outcome.is_a?(Array) then expected_outcome = [expected_outcome] end
    if (expected_outcome.size > 0) & !recommendation.empty?
      average_precision = 0
      correct_items = []
      total_items_considered = []
      # sort rules by weight
      # we first group rules with equal weights
      # and then sort the groups by weight
      recommendation.each do |items|
        if !items.is_a?(Array) then items = [items] end
        if items.first.class != expected_outcome.first.class
            raise ArgumentError, "Expected outcome was of type #{expected_outcome.first.class}, while the item in the recommendation was of type #{items.first.class}"
        end
        # skip already considered items
        if (new_items = items - total_items_considered).size > 0
          new_items.each {|item| total_items_considered << item}
          if correct_in_rule = (items & expected_outcome)
            if correct_in_rule.size > 0
              # make sure that the new items havent already been added earlier
              new_correct = (correct_in_rule - correct_items)
              # add new items
              new_correct.each {|item| correct_items << item}
              change_in_recall = new_correct.size.to_r/expected_outcome.size
              precision_at_k = correct_items.size.to_r/total_items_considered.size
              average_precision += (precision_at_k * change_in_recall)
            end
          end
        end
      end
      average_precision.to_f
    else
      nil
    end
end
discernibility(rec:) click to toggle source
# File lib/evoc/evaluate.rb, line 32
def self.discernibility(rec:)
  # AP is 0 for the empty list
  if rec.is_a?(Array) && rec.empty? # array and empty
    return nil
  end
  self.validateInput(rec)

  rec_size = 0
  rec_clusters = 0

  rec.each do |c|
    rec_clusters = rec_clusters + 1
    c.each do |e|
      rec_size = rec_size + 1
    end
  end
  return (rec_clusters/rec_size).to_f

end
f1(rec:,exp:) click to toggle source

@return the f1 score (preision/recall harmonic mean)

# File lib/evoc/evaluate.rb, line 62
def self.f1(rec:,exp:)
  # AP is 0 for the empty list
  if rec.is_a?(Array) && rec.empty? # array and empty
    return nil
  end
  self.validateInput(rec)

  rec_size = 0
  rec_correct = 0

  rec.each do |c|
    c.each do |e|
      rec_size = rec_size + 1
      rec_correct = rec_correct + e
    end
  end
  return (2*rec_correct/(rec_size + exp)).to_f
end
first_relevant(rec:) click to toggle source

@return the rank of the first relevant itemjk

# File lib/evoc/evaluate.rb, line 105
def self.first_relevant(rec:)
  # AP is 0 for the empty list
  if rec.is_a?(Array) && rec.empty? # array and empty
    return nil
  end
  self.validateInput(rec)

  last_checked = 1
  rec.each do |c|
    c.each do |e|
      if e == 1
        return last_checked
      end
      last_checked = last_checked + 1
    end
  end
  return nil 
end
last_relevant(rec:) click to toggle source

@return the rank of the last relevant itemjk

# File lib/evoc/evaluate.rb, line 126
def self.last_relevant(rec:)
  # AP is 0 for the empty list
  if rec.is_a?(Array) && rec.empty? # array and empty
    return nil
  end
  self.validateInput(rec)

  size = rec.inject(0) {|sum,c| sum + c.size}
  last_checked = size
  rec.reverse_each do |c|
    c.reverse_each do |e|
      if e == 1
        return last_checked
      end
      last_checked = last_checked - 1
    end
  end
  return nil 
end
mean_confidence(rules:) click to toggle source
# File lib/evoc/evaluate.rb, line 23
def self.mean_confidence(rules:)
  if rules.empty? then return nil end
  return (rules.inject(0) {|sum,r| sum + r.m_confidence.value}/rules.size).to_f
end
mean_confidence10(rules:) click to toggle source
# File lib/evoc/evaluate.rb, line 28
def self.mean_confidence10(rules:)
  return self.mean_confidence(rules: Evoc::RuleStore.sort_on(rules: rules,measures: ['m_confidence']).take(10).flatten.take(10))
end
mean_support(rules:) click to toggle source
# File lib/evoc/evaluate.rb, line 14
def self.mean_support(rules:)
  if rules.empty? then return nil end
  return (rules.inject(0) {|sum,r| sum + r.m_support.value}/rules.size).to_f
end
mean_support10(rules:) click to toggle source
# File lib/evoc/evaluate.rb, line 19
def self.mean_support10(rules:)
  return self.mean_support(rules: Evoc::RuleStore.sort_on(rules: rules,measures: ['m_support']).take(10).flatten.take(10))
end
precision(rec:,exp: nil) click to toggle source
# File lib/evoc/evaluate.rb, line 164
def self.precision(rec:,exp: nil)
  # AP is 0 for the empty list
  if rec.is_a?(Array) && rec.empty? # array and empty
    return nil
  end
  self.validateInput(rec)

  size_rec = rec.inject(0) {|sum,c| sum + c.size}
  num_correct_in_rec = rec.inject(0) {|sum,c| sum + c.inject(&:+)}

  return (num_correct_in_rec/size_rec).to_f
end
precision10(rec:,exp: nil) click to toggle source
# File lib/evoc/evaluate.rb, line 155
def self.precision10(rec:,exp: nil)
  # AP is 0 for the empty list
  if rec.is_a?(Array) && rec.empty? # array and empty
    return nil
  end
  self.validateInput(rec)
  return self.precision(rec: [rec.take(10).flatten.take(10)])
end
recall(rec:,exp: nil) click to toggle source
# File lib/evoc/evaluate.rb, line 177
def self.recall(rec:,exp: nil)
  # AP is 0 for the empty list
  if rec.is_a?(Array) && rec.empty? # array and empty
    return nil
  end
  self.validateInput(rec)

  num_correct_in_rec = rec.inject(0) {|sum,c| sum + c.inject(&:+)}

  if exp.nil?
    return num_correct_in_rec
  else
    if num_correct_in_rec > exp
      raise ArgumentError, "Found more relevant items than the provided number of relevant items"
    end
    return (num_correct_in_rec/exp).to_f
  end
end
recall10(rec:,exp: nil) click to toggle source
# File lib/evoc/evaluate.rb, line 146
def self.recall10(rec:,exp: nil)
  # AP is 0 for the empty list
  if rec.is_a?(Array) && rec.empty? # array and empty
    return nil
  end
  self.validateInput(rec)
  return self.recall(rec: [rec.take(10).flatten.take(10)],exp: exp)
end
relevant_ranks(rec:) click to toggle source

@return an array containg the rank of each consequtive expected outcome

# File lib/evoc/evaluate.rb, line 83
def self.relevant_ranks(rec:)
  # AP is 0 for the empty list
  if rec.is_a?(Array) && rec.empty? # array and empty
    return []
  end
  self.validateInput(rec)

  ranks = []
  last_checked = 1
  rec.each do |c|
    c.each do |e|
      if e == 1
        ranks << last_checked
      end
      last_checked = last_checked + 1
    end
  end
  return ranks 
end
t_ap(rec:,exp: nil) click to toggle source

r_p : relevant items in previous groups i_p : index previous group r_g : relevant items in group n_g : items in group i : index of current item

# File lib/evoc/evaluate.rb, line 205
def self.t_ap(rec:,exp: nil)
  # AP is 0 for the empty list
  if rec.is_a?(Array) && rec.empty? # array and empty
    return nil
  end
  self.validateInput(rec)

  ap = 0
  r_p = 0
  i_p = 0
  rec.each do |cluster|
    r_g = cluster.inject(&:+).to_r
    n_g = cluster.size.to_r
    cluster.each_with_index do |_,i|
      i = i_p + i + 1
      chance_relevant = r_g/n_g
      avg_previous_rel = if (n_g == 1)
                           (r_p + 1) * (1/i)
                         else
                           (r_p + (i - i_p - 1)*((r_g-1)/(n_g-1)) + 1) * (1/i)
                         end

      item_ap_contribution = chance_relevant * avg_previous_rel

      ap = ap + item_ap_contribution
    end
    r_p = r_p + r_g
    i_p = i_p + n_g
  end
  # if the number of relevant documents is not supplied
  # assume that the recommendation contains all relevant documents
  if exp.nil?
    exp = r_p
  else
    if r_p > exp
      raise ArgumentError, "Found more relevant items than the provided number of relevant items"
    end
  end
  return (r_p == 0 ? 0 : (ap/exp).to_f)
end
validateInput(input) click to toggle source
# File lib/evoc/evaluate.rb, line 5
def self.validateInput(input)
  # verify format
  if !input.is_a?(Array) ||                              # not an array
     !input.first.is_a?(Array) ||                        # not containg an array
     ![0,1].include?(input.first.first)                  # items are not 0s and 1s
    raise Evoc::Exceptions::FormatError.new "Wrong format given to #{__method__}, expected list of list of 0s and 1s, input was: #{input}"
  end
end