module Elasticshelf::Client

Attributes

client[R]
cutoff_days[RW]
date_separator[RW]
errors[R]
get_info[R]
ignore_unavailable[RW]
include_global_state[RW]
indices[RW]
indices_affected[R]
indices_closed[R]
indices_deleted[R]
indices_expired[R]
indices_prefix[RW]
indices_rejected[R]
indices_withered[R]
repo[RW]
repo_compressed[RW]
repo_location[RW]
repo_type[RW]
results[R]
snapshot[RW]
snapshot_indices_expired_name[RW]
version[R]
wait_for_completion[RW]

Public Instance Methods

close_expired_indices() click to toggle source
# File lib/elasticshelf.rb, line 100
def close_expired_indices
  find_expired_indices if blank? @indices_expired
  @indices_expired.each do |k,v|
    close_index(k)
    puts "closed index '#{k}' expired #{v} days ago"
  end
end
close_index(index_name) click to toggle source
# File lib/elasticshelf.rb, line 94
def close_index(index_name)
  try do
    @indices_closed[index_name] = @client.indices.close(index: index_name)
  end
end
cutoff_days_as_date() click to toggle source
# File lib/elasticshelf.rb, line 345
def cutoff_days_as_date
  return {"error" => "cutoff_days must be zero or greater"} if @cutoff_days.to_i < 0
  cutoff_date = DateTime.now.new_offset(0) - @cutoff_days
  set_time_to_beginning_of_day(cutoff_date.strftime("%Y.%m.%d"))
end
cutoff_days_as_date_to_s() click to toggle source
# File lib/elasticshelf.rb, line 351
def cutoff_days_as_date_to_s
  cdad = cutoff_days_as_date
  cdad = cdad.strftime("%Y.%m.%d") if cdad.is_a? DateTime
  cdad
end
delete_expired_indices() click to toggle source
# File lib/elasticshelf.rb, line 118
def delete_expired_indices
  reset_errors_results
  find_expired_indices if blank? @indices_expired
  @indices_expired.each do |k,v|
    delete_index(k)
    puts "deleted '#{k}' expired #{v} days ago"
  end
end
delete_index(index_name) click to toggle source
# File lib/elasticshelf.rb, line 108
def delete_index(index_name)
  if ['_all','*'].include? index_name.downcase
    @errors = {"error" => "deleting all indices is not allowed"}
    return
  end
  try do
    @indices_deleted[index_name] = @client.indices.delete(index: index_name)
  end
end
find_expired_indices() click to toggle source
# File lib/elasticshelf.rb, line 52
def find_expired_indices
  try do
    return {} if blank? @indices_prefix
    return {} if blank? @date_separator
    return {} if @cutoff_days.nil?
    return {} if @cutoff_days.to_i < 0
    @indices_expired = {}
    # cutoff_date = cutoff_days_as_date
    # cutoff_date_beginning_of_day = set_time_to_beginning_of_day(cutoff_date.strftime("%Y.%m.%d"))
    cutoff_date_beginning_of_day = cutoff_days_as_date
    required_date_parts = 3 # for now, we only do days not hours
    # the following should only return 'closed' indices but it returns all indices ???
    # sorted_indices = @client.indices.get_settings(:index => @indices_prefix, :expand_wildcards => 'closed').keys
    # sorted_indices = @client.indices.get_settings(:index => @indices_prefix, :expand_wildcards => 'open').keys
    sorted_indices = @client.indices.get_settings(:index => @indices_prefix, :expand_wildcards => 'open,closed').keys
    sorted_indices.each do |index_name|
      unprefixed_index_name = index_name.slice(@indices_prefix.size-1..-1)
      date_parts = unprefixed_index_name.split(@date_separator)
      next unless date_parts.size == required_date_parts
      index_time = set_time_to_beginning_of_day(unprefixed_index_name, separator=@date_separator)
      expiry = (cutoff_date_beginning_of_day - index_time).to_i
      if expiry > 0
        @indices_expired[index_name] = expiry
      end
    end
  end
end
get_cluster_state(index_name, metric='metadata') click to toggle source
# File lib/elasticshelf.rb, line 162
def get_cluster_state(index_name, metric='metadata')
  try do
    index_metadata = @client.cluster.state(:index => index_name, :metric => metric)
    index_metadata
  end
end
get_elasticsearch_version() click to toggle source
# File lib/elasticshelf.rb, line 321
def get_elasticsearch_version
  try do
    info_hash = @client.info
    return nil if blank? info_hash
    return nil unless info_hash['status'] == 200
    return nil if blank? info_hash['version']
    info_hash['version']['number']
  end
end
get_index_state(index_name, metric='metadata') click to toggle source
# File lib/elasticshelf.rb, line 155
def get_index_state(index_name, metric='metadata')
  try do
    index_metadata = @client.cluster.state(:index => index_name, :metric => metric)
    index_metadata['metadata']['indices'][index_name]['state']
  end
end
get_lucene_version() click to toggle source
# File lib/elasticshelf.rb, line 331
def get_lucene_version
  try do
    info_hash = @client.info
    return nil if blank? info_hash
    return nil unless info_hash['status'] == 200
    return nil if blank? info_hash['version']
    info_hash['version']['lucene_version']
  end
end
get_settings_index(index_name) click to toggle source
# File lib/elasticshelf.rb, line 127
def get_settings_index(index_name)
  reset_errors_results
  try do
    @results[index_name] = @client.indices.get_settings(:index => index_name)
  end
end
index_closed?(index_name) click to toggle source
# File lib/elasticshelf.rb, line 80
def index_closed?(index_name)
  # catch exceptions when index doesn't exist
  try do
    index_metadata = @client.cluster.state(:index => index_name, :metric => 'metadata')
    index_metadata['metadata']['indices'][index_name]['state'] == 'close'
  end
end
indices_prefix=(str) click to toggle source
# File lib/elasticshelf.rb, line 48
def indices_prefix=(str)
  @indices_prefix = str.chomp('*') + '*'
end
new(arguments={}) click to toggle source
# File lib/elasticshelf.rb, line 24
def new(arguments={})
  @client = Elasticsearch::Transport::Client.new(arguments)
  @wait_for_completion = true
  @ignore_unavailable = true
  @include_global_state = false
  @snapshot_indices_expired_name = nil
  @date_separator = '.'
  @indices_prefix = 'logstash-'
  @cutoff_days = -1
  @repo_type = 'fs'
  @repo_compressed = true
  @indices = []
  @indices_affected = {}
  @indices_rejected = {}
  @indices_expired = {}
  @indices_closed = {}
  @indices_deleted = {}
  @indices_withered = {}
  @results = {}
  @errors = {}
  self
end
open_index(index_name) click to toggle source
# File lib/elasticshelf.rb, line 88
def open_index(index_name)
  try do
    @results[index_name] = @client.indices.open(index: index_name)
  end
end
snapshot_create() click to toggle source
# File lib/elasticshelf.rb, line 240
def snapshot_create
  # naming errors as seen in elasticsearch log:
  #   Invalid snapshot name [2014-04-01 20:12:07 UTC], must not contain whitespace
  #   Invalid snapshot name [2014-04-01_20:12:38_UTC], must be lowercase
  try do
    if blank? @indices
      # the whole cluster:
      @results = @client.snapshot.create(repository: @repo,
        snapshot: @snapshot,
        wait_for_completion: @wait_for_completion,
        body: {
          ignore_unavailable: @ignore_unavailable,
          include_global_state: @include_global_state
        }
      )
    else
      @results = @client.snapshot.create(repository: @repo,
        snapshot: @snapshot,
        wait_for_completion: @wait_for_completion,
        body: {
          indices: @indices,
          ignore_unavailable: @ignore_unavailable,
          include_global_state: @include_global_state
        }
      )
    end
  end
end
snapshot_create_repository() click to toggle source
# File lib/elasticshelf.rb, line 169
def snapshot_create_repository
  @results = @errors = {}
  # ensure the location is accessible by Elasticsearch
  @repo_type = 'fs' if blank? @repo_type
  @repo_location = '/var/lib/elasticsearch_snapshots' if blank? @repo_location
  @repo_compressed = true if blank? @repo_compressed
  try do
    @results = @client.snapshot.create_repository(
      repository: @repo,
      body: {
        type: @repo_type,
        settings: {
          location: @repo_location,
          compress: @repo_compressed
        }
      }
    )
  end
end
snapshot_delete() click to toggle source
# File lib/elasticshelf.rb, line 280
def snapshot_delete
  try do
    @results = @client.snapshot.delete(repository: @repo, snapshot: @snapshot)
  end
end
snapshot_delete_repository() click to toggle source
# File lib/elasticshelf.rb, line 200
def snapshot_delete_repository
  @results = @errors = {}
  try do
    @results = @client.snapshot.delete_repository(repository: @repo)
  end
end
snapshot_expired_indices() click to toggle source
# File lib/elasticshelf.rb, line 207
def snapshot_expired_indices
  @results = @errors = {}
  find_expired_indices
  if blank? @indices_expired
    @results = {"result" => "no expired indices found"}
    return
  end
  open_indices = []
  @indices_expired.keys.each do |index_name|
    # only open indices can be used in a snapshot:
    if index_closed?(index_name)
      @indices_rejected[index_name] = "closed index can't snapshot"
    else
      @indices_affected[index_name] = "open can snapshot"
      open_indices << index_name
    end
  end
  if blank? open_indices
    @results = {"result" => "none of the expired indices are open so can't snapshot"}
    return
  end
  @indices = open_indices.join(',')
  # allow user to override this name:
  if blank? @snapshot_indices_expired_name
    @snapshot = (Time.now.utc.to_s.gsub(' ', '_') +
                 "_#{@indices_prefix.chomp('*')}" +
                 "_cutoff_#{@cutoff_days}_days").downcase
  else
    @snapshot = @snapshot_indices_expired_name
  end
  @results = snapshot_create
end
snapshot_get() click to toggle source
# File lib/elasticshelf.rb, line 269
def snapshot_get
  reset_errors_results
  try do
    if blank? @snapshot
      @results = @client.snapshot.get(repository: @repo, snapshot: '_all')
    else
      @results = @client.snapshot.get(repository: @repo, snapshot: @snapshot)
    end
  end
end
snapshot_get_repository() click to toggle source
# File lib/elasticshelf.rb, line 189
def snapshot_get_repository
  @results = @errors = {}
  try do
    if @repo
      @results = @client.snapshot.get_repository(repository: @repo)
    else
      @results = @client.snapshot.get_repository
    end
  end
end
snapshot_restore() click to toggle source
# File lib/elasticshelf.rb, line 286
def snapshot_restore
  try do
    if blank? @indices
      # the whole snapshot:
      @results = @client.snapshot.restore(
        repository: @repo,
        snapshot: @snapshot,
        wait_for_completion: @wait_for_completion,
        body: {
          ignore_unavailable: @ignore_unavailable,
          include_global_state: @include_global_state
        }
      )
    else
      # note: @indices must be a string of comma separated index names
      @results = @client.snapshot.restore(
        repository: @repo,
        snapshot: @snapshot,
        wait_for_completion: @wait_for_completion,
        body: {
          indices: @indices,
          ignore_unavailable: @ignore_unavailable,
          include_global_state: @include_global_state
        }
      )
    end
  end
end
wither_expired_indices() click to toggle source
# File lib/elasticshelf.rb, line 146
def wither_expired_indices
  reset_errors_results
  find_expired_indices if blank? @indices_expired
  @indices_expired.each do |k,v|
    wither_index(k)
    puts "withered '#{k}' expired #{v} days ago"
  end
end
wither_index(index_name) click to toggle source
# File lib/elasticshelf.rb, line 134
def wither_index(index_name)
  try do
    # don't try to disable bloom filter on a closed index, it will re-open it:
    if index_closed?(index_name)
      @errors = {"failed" => "Unable to wither (disable bloom filter) index #{index_name}: It's already closed."}
      return
    else
      @indices_withered[index_name] = @client.indices.put_settings(:index => index_name, :body => 'index.codec.bloom.load=false')
    end
  end
end

Private Instance Methods

blank?(some_object) click to toggle source
# File lib/elasticshelf.rb, line 371
def blank?(some_object)
  some_object = some_object.strip if some_object.is_a? String
  some_object.respond_to?(:empty?) ? some_object.empty? : !some_object
end
reset_errors_results() click to toggle source
# File lib/elasticshelf.rb, line 376
def reset_errors_results
  @results = {}
  @errors = {}
end
set_time_to_beginning_of_day(index_timestamp, separator='.') click to toggle source
# File lib/elasticshelf.rb, line 367
def set_time_to_beginning_of_day(index_timestamp, separator='.')
  return DateTime.strptime(index_timestamp, ['%Y', '%m', '%d'].join(separator))
end
try() { || ... } click to toggle source
# File lib/elasticshelf.rb, line 359
def try(&block)
  yield
rescue Exception => ex
  # a consistent response for all exceptions similar to Elasticsearch:
  @errors = {"error" => "#{ex.class}: #{ex.message}"}
  # raise ex
end