class Hatchet::Reaper

Delete apps

Delete a single app:

@reaper.destroy_with_log(id: id, name: name, reason: "console")

Clear out all apps older than HATCHET_ALIVE_TTL_MINUTES:

@reaper.destroy_older_apps

If you need to clear up space or wait for space to be cleared up then:

@reaper.clean_old_or_sleep

Notes:

Constants

DEFAULT_REGEX
HATCHET_APP_LIMIT
MUTEX_FILE

Protect against parallel deletion on the same machine via concurrent processes

Does not protect against distributed systems on different machines trying to delete the same applications

TTL_MINUTES

Attributes

hatchet_app_limit[RW]
io[RW]

Public Class Methods

new(api_rate_limit: , regex: DEFAULT_REGEX, io: STDOUT, hatchet_app_limit: HATCHET_APP_LIMIT, initial_sleep: 10) click to toggle source
# File lib/hatchet/reaper.rb, line 40
def initialize(api_rate_limit: , regex: DEFAULT_REGEX, io: STDOUT, hatchet_app_limit:  HATCHET_APP_LIMIT, initial_sleep: 10)
  @io = io
  @apps = []
  @regex = regex
  @limit = hatchet_app_limit
  @api_rate_limit = api_rate_limit
  @reaper_throttle = ReaperThrottle.new(initial_sleep: initial_sleep)
end

Public Instance Methods

destroy_all(force_refresh: @apps.empty?) click to toggle source

No guardrails, will delete all apps that match the hatchet namespace

# File lib/hatchet/reaper.rb, line 111
def destroy_all(force_refresh: @apps.empty?)
  MUTEX_FILE.flock(File::LOCK_EX)

  refresh_app_list if force_refresh

  while app = @apps.pop
    begin
      destroy_with_log(name: app["name"], id: app["id"], reason: "destroy all")
    rescue AlreadyDeletedError => e
      handle_conflict(
        conflict_message: e.message,
        strategy: :refresh_api_and_continue
      )
    end
  end
ensure
  MUTEX_FILE.flock(File::LOCK_UN)
end
destroy_older_apps(minutes: TTL_MINUTES, force_refresh: @apps.empty?, on_conflict: :refresh_api_and_continue) click to toggle source

Destroys apps that are older than the given argument (expecting integer minutes)

This method might be running concurrently on multiple processes or multiple machines.

When a duplicate destroy is detected we can move forward with a conflict strategy:

  • ‘:refresh_api_and_continue`: Sleep to see if another process will clean up everything for us and then re-populate apps from the API and continue.

  • ‘:stop_if_under_limit`: Sleep to allow other processes to continue. Then if apps list is under the limit, assume someone else is already cleaning up for us and that we’re good to move ahead to try to create an app. Otherwise if we’re at or over the limit sleep, refresh the app list, and continue attempting to delete apps.

# File lib/hatchet/reaper.rb, line 79
def destroy_older_apps(minutes: TTL_MINUTES, force_refresh: @apps.empty?, on_conflict: :refresh_api_and_continue)
  MUTEX_FILE.flock(File::LOCK_EX)

  refresh_app_list if force_refresh

  while app = @apps.pop
    age = AppAge.new(created_at: app["created_at"], ttl_minutes: minutes)
    if !age.can_delete?
      @apps.push(app)
      break
    else
      begin
        destroy_with_log(
          id: app["id"],
          name: app["name"],
          reason: "app age (#{age.in_minutes}m) is older than #{minutes}m"
        )
      rescue AlreadyDeletedError => e
        if handle_conflict(
          strategy: on_conflict,
          conflict_message: e.message,
        ) == :stop
          break
        end
      end
    end
  end
ensure
  MUTEX_FILE.flock(File::LOCK_UN)
end
destroy_with_log(name:, id:, reason: ) click to toggle source
# File lib/hatchet/reaper.rb, line 184
def destroy_with_log(name:, id:, reason: )
  message = "Destroying #{name.inspect}: #{id}, (#{@apps.length}/#{@limit}) reason: #{reason}"

  @api_rate_limit.call.app.delete(id)

  io.puts message
rescue Excon::Error::NotFound, Excon::Error::Forbidden => e
  status = e.response.status
  request_id = e.response.headers["Request-Id"]
  message = "Possible duplicate destroy attempted #{name.inspect}: #{id}, status: #{status}, request_id: #{request_id}"
  raise AlreadyDeletedError.new(message)
end
sleep_if_over_limit(reason: ) click to toggle source
# File lib/hatchet/reaper.rb, line 49
    def sleep_if_over_limit(reason: )
      if @apps.length >= @limit
        age = AppAge.new(created_at: @apps.last["created_at"], ttl_minutes: TTL_MINUTES)
        @reaper_throttle.call(max_sleep: age.sleep_for_ttl) do |sleep_for|
          io.puts <<-EOM.strip_heredoc
            WARNING: Hatchet app limit reached (#{@apps.length}/#{@limit})
            All known apps are younger than #{TTL_MINUTES} minutes.
            Sleeping (#{sleep_for}s)

            Reason: #{reason}
          EOM

          sleep(sleep_for)
        end
      end
    end

Private Instance Methods

get_heroku_apps() click to toggle source
# File lib/hatchet/reaper.rb, line 172
        def get_heroku_apps
  @api_rate_limit.call.app.list
end
handle_conflict(conflict_message:, strategy:) click to toggle source

Will sleep with backoff and emit a warning message returns :continue or :stop symbols :stop indicates execution should stop

# File lib/hatchet/reaper.rb, line 133
            def handle_conflict(conflict_message:, strategy:)
      message = String.new(<<-EOM.strip_heredoc)
        WARNING: Possible race condition detected: #{conflict_message}
        Hatchet app limit (#{@apps.length}/#{@limit}), using strategy #{strategy}
      EOM

      conflict_state = if :refresh_api_and_continue ==  strategy
        message << "\nSleeping, refreshing app list, and continuing."
        :continue
      elsif :stop_if_under_limit == strategy && @apps.length >= @limit
        message << "\nSleeping, refreshing app list, and continuing. Not under limit."
        :continue
      elsif :stop_if_under_limit == strategy
        message << "\nHalting deletion of older apps. Under limit."
        :stop
      else
        raise "No such strategy: #{strategy}, plese use :stop_if_under_limit or :refresh_api_and_continue"
      end

      @reaper_throttle.call(max_sleep: TTL_MINUTES) do |sleep_for|
        io.puts <<-EOM.strip_heredoc
          #{message}
          Sleeping (#{sleep_for}s)
        EOM

        sleep(sleep_for)
      end

      case conflict_state
      when :continue
        refresh_app_list
      when :stop
      else
        raise "Unknown state #{conflict_state}"
      end

      conflict_state
    end
refresh_app_list() click to toggle source
# File lib/hatchet/reaper.rb, line 176
        def refresh_app_list
  @apps = get_heroku_apps.
    filter {|app| app["name"].match(@regex) }.
    map {|app| app["created_at"] = DateTime.parse(app["created_at"].to_s); app }.
    sort_by { |app| app["created_at"] }.
    reverse # Ascending order, oldest is last
end