class Flapjack::Gateways::Oobetet::Notifier

Attributes

siblings[RW]

Public Class Methods

new(options = {}) click to toggle source
# File lib/flapjack/gateways/oobetet.rb, line 24
def initialize(options = {})
  @lock = options[:lock]
  @config = options[:config]

  @hostname = Socket.gethostname

  unless @config['watched_check']
    raise RuntimeError, 'Flapjack::Oobetet: watched_check must be defined in the config'
  end
  @check_matcher = '"' + @config['watched_check'] + '"'

  @flapjack_ok = true
  @last_alert = nil
  @last_breach = nil
end

Public Instance Methods

start() click to toggle source
# File lib/flapjack/gateways/oobetet.rb, line 40
def start
  loop do
    @lock.synchronize do
      check_timers
    end

    Kernel.sleep 10
  end
end
stop_type() click to toggle source
# File lib/flapjack/gateways/oobetet.rb, line 50
def stop_type
  :exception
end

Private Instance Methods

check_timers() click to toggle source
# File lib/flapjack/gateways/oobetet.rb, line 56
def check_timers
  if @siblings
    @time_checker ||= @siblings.detect {|sib| sib.respond_to?(:breach?) }
    @bot ||= @siblings.detect {|sib| sib.respond_to?(:announce) }
  end

  t = Time.now
  breach = @time_checker.breach?(t) if @time_checker

  if @last_breach && !breach
    emit_jabber("Flapjack Self Monitoring is OK")
    emit_pagerduty("Flapjack Self Monitoring is OK", 'resolve')
  end

  @last_breach = breach
  return unless breach

  Flapjack.logger.error("Self monitoring has detected the following breach: #{breach}")
  summary = "Flapjack Self Monitoring is Critical: #{breach} for #{@check_matcher}, " +
            "from #{@hostname} at #{t}"

  if @last_alert.nil? || @last_alert < (t.to_i - 55)

    announced_jabber    = emit_jabber(summary)
    announced_pagerduty = emit_pagerduty(summary, 'trigger')

    @last_alert = t.to_i if announced_jabber || announced_pagerduty

    if @last_alert.nil? || @last_alert < (t.to_i - 55)
      msg = "NOTICE: Self monitoring has detected a failure and is unable to tell " +
            "anyone about it. DON'T PANIC."
      Flapjack.logger.error msg
    end
  end
end
emit_jabber(summary) click to toggle source
# File lib/flapjack/gateways/oobetet.rb, line 92
def emit_jabber(summary)
  return if @bot.nil?
  @bot.announce(summary)
  true
end
emit_pagerduty(summary, event_type = 'trigger') click to toggle source
# File lib/flapjack/gateways/oobetet.rb, line 98
def emit_pagerduty(summary, event_type = 'trigger')
  return if @config['pagerduty_contact'].nil?
  status, response = send_pagerduty_event(:service_key  => @config['pagerduty_contact'],
                                          :incident_key => "Flapjack Self Monitoring from #{@hostname}",
                                          :event_type   => event_type,
                                          :description  => summary)
  unless '200'.eql?(status)
    Flapjack.logger.error("pagerduty returned #{status} #{response.inspect}")
    return false
  end

  Flapjack.logger.debug("successfully sent pagerduty event")
  true
end
send_pagerduty_event(opts = {}) click to toggle source

TODO trap Oj JSON errors FIXME common code with the pagerduty gateway, move to shared module

# File lib/flapjack/gateways/oobetet.rb, line 115
def send_pagerduty_event(opts = {})
  event = { 'service_key'  => opts[:service_key],
            'incident_key' => opts[:incident_key],
            'event_type'   => opts[:event_type],
            'description'  => opts[:description] }

  uri = URI::HTTPS.build(:host => 'events.pagerduty.com',
                         :path => '/generic/2010-04-15/create_event.json',
                         :port => 443)
  http = Net::HTTP.new(uri.host, uri.port)
  http.use_ssl = true
  http.verify_mode = OpenSSL::SSL::VERIFY_PEER
  request = Net::HTTP::Post.new(uri.request_uri)
  request.body = Flapjack.dump_json(event)
  http_response = http.request(request)

  response = Flapjack.load_json(http_response.body)
  status   = http_response.code
  Flapjack.logger.debug "send_pagerduty_event got a return code of #{status} - #{response.inspect}"
  [status, response]
end