class SidekiqUtils::LatencyAlert

Constants

REDIS_KEY

Public Class Methods

check!() click to toggle source
# File lib/sidekiq_utils/latency_alert.rb, line 8
def check!
  alerts = {}
  Sidekiq::Queue.all.each do |queue|
    threshold = config['alert_thresholds'][queue.name].
      try!(&:to_i).try!(:minutes)
    next if threshold == :disabled
    threshold ||= config['alert_thresholds']['default'].
      try!(&:to_i).try!(:minutes) || 10.minutes
    if (latency = queue.latency) > threshold
      alerts[queue.name] = latency
    end
  end

  if alerts.blank?
    if should_alert_back_to_normal?
      Sidekiq.redis { |r| r.del(REDIS_KEY) }
      slack_alert("All queues under their thresholds.")
    end
    return false
  end

  alert_message = ["Sidekiq queue latency over threshold:"]
  alerts.each do |queue, latency|
    alert_message << "Queue #{queue} is #{formatted_latency(latency)} behind"
  end
  alert_message = alert_message.join("\n")
  if should_alert_again?(alert_message)
    slack_alert(alert_message)
  end

  true
end
config() click to toggle source
# File lib/sidekiq_utils/latency_alert.rb, line 41
def config
  @config ||= (YAML.load(ERB.new(
    File.read('config/sidekiq_utils.yml')).result) || {})
end

Private Class Methods

formatted_latency(latency) click to toggle source
# File lib/sidekiq_utils/latency_alert.rb, line 47
def formatted_latency(latency)
  latency_days = (latency.to_f / 1.day).floor
  if latency < 1.day
    formatted_latency =
      ActionController::Base.helpers.distance_of_time_in_words(latency)
  else
    formatted_latency = "#{latency_days} #{"day".pluralize(latency_days)}"
  end
  if latency > 1.day
    latency_in_day = latency - latency_days * 1.day
    if latency_in_day >= 45.minutes
      formatted_latency += " and " +
        ActionController::Base.helpers.
          distance_of_time_in_words(latency_in_day)
    end
  end
  formatted_latency
end
should_alert_again?(message) click to toggle source
# File lib/sidekiq_utils/latency_alert.rb, line 91
def should_alert_again?(message)
  Sidekiq.redis do |redis|
    last_alert = redis.get(REDIS_KEY)
    last_alert = JSON.parse(last_alert) if last_alert
    if !last_alert ||
        last_alert['message_hash'] != Digest::SHA1.hexdigest(message) ||
        last_alert['time'] < (config['repeat_alert_every'] || 60).to_i.minutes.ago.to_i
      redis.set(REDIS_KEY, {
        'message_hash' => Digest::SHA1.hexdigest(message),
        'time' => Time.now.to_i,
      }.to_json)
      true
    else
      false
    end
  end
end
should_alert_back_to_normal?() click to toggle source
# File lib/sidekiq_utils/latency_alert.rb, line 85
def should_alert_back_to_normal?
  Sidekiq.redis do |redis|
    redis.get(REDIS_KEY).present?
  end
end
slack_alert(alert_message) click to toggle source
# File lib/sidekiq_utils/latency_alert.rb, line 66
def slack_alert(alert_message)
  return unless config['slack'].present?
  require 'slack-notifier'
  @slack ||= Slack::Notifier.new(
    "https://#{config['slack']['team']}.slack.com/services/hooks/incoming-webhook?"\
      "token=#{config['slack']['token']}",
    username: config['slack']['username'] || 'Sidekiq alerts',
    icon_emoji: ":#{config['slack']['icon'] || 'alarm_clock'}:")

  if defined?(Rails) && !Rails.env.production?
    channels = ['#test-channel']
  else
    channels = config['slack']['channels_to_alert']
  end
  Array.wrap(channels).each do |slack_name|
    @slack.ping(alert_message, channel: slack_name)
  end
end