module Elasticshelf::Client
Attributes
client[R]
cutoff_days[RW]
date_separator[RW]
errors[R]
get_info[R]
include_global_state[RW]
indices[RW]
indices_affected[R]
indices_closed[R]
indices_deleted[R]
indices_expired[R]
indices_prefix[RW]
indices_rejected[R]
indices_withered[R]
repo[RW]
repo_compressed[RW]
repo_location[RW]
repo_type[RW]
results[R]
snapshot[RW]
snapshot_indices_expired_name[RW]
version[R]
wait_for_completion[RW]
Public Instance Methods
close_expired_indices()
click to toggle source
# File lib/elasticshelf.rb, line 100 def close_expired_indices find_expired_indices if blank? @indices_expired @indices_expired.each do |k,v| close_index(k) puts "closed index '#{k}' expired #{v} days ago" end end
close_index(index_name)
click to toggle source
# File lib/elasticshelf.rb, line 94 def close_index(index_name) try do @indices_closed[index_name] = @client.indices.close(index: index_name) end end
cutoff_days_as_date()
click to toggle source
# File lib/elasticshelf.rb, line 345 def cutoff_days_as_date return {"error" => "cutoff_days must be zero or greater"} if @cutoff_days.to_i < 0 cutoff_date = DateTime.now.new_offset(0) - @cutoff_days set_time_to_beginning_of_day(cutoff_date.strftime("%Y.%m.%d")) end
cutoff_days_as_date_to_s()
click to toggle source
# File lib/elasticshelf.rb, line 351 def cutoff_days_as_date_to_s cdad = cutoff_days_as_date cdad = cdad.strftime("%Y.%m.%d") if cdad.is_a? DateTime cdad end
delete_expired_indices()
click to toggle source
# File lib/elasticshelf.rb, line 118 def delete_expired_indices reset_errors_results find_expired_indices if blank? @indices_expired @indices_expired.each do |k,v| delete_index(k) puts "deleted '#{k}' expired #{v} days ago" end end
delete_index(index_name)
click to toggle source
# File lib/elasticshelf.rb, line 108 def delete_index(index_name) if ['_all','*'].include? index_name.downcase @errors = {"error" => "deleting all indices is not allowed"} return end try do @indices_deleted[index_name] = @client.indices.delete(index: index_name) end end
find_expired_indices()
click to toggle source
# File lib/elasticshelf.rb, line 52 def find_expired_indices try do return {} if blank? @indices_prefix return {} if blank? @date_separator return {} if @cutoff_days.nil? return {} if @cutoff_days.to_i < 0 @indices_expired = {} # cutoff_date = cutoff_days_as_date # cutoff_date_beginning_of_day = set_time_to_beginning_of_day(cutoff_date.strftime("%Y.%m.%d")) cutoff_date_beginning_of_day = cutoff_days_as_date required_date_parts = 3 # for now, we only do days not hours # the following should only return 'closed' indices but it returns all indices ??? # sorted_indices = @client.indices.get_settings(:index => @indices_prefix, :expand_wildcards => 'closed').keys # sorted_indices = @client.indices.get_settings(:index => @indices_prefix, :expand_wildcards => 'open').keys sorted_indices = @client.indices.get_settings(:index => @indices_prefix, :expand_wildcards => 'open,closed').keys sorted_indices.each do |index_name| unprefixed_index_name = index_name.slice(@indices_prefix.size-1..-1) date_parts = unprefixed_index_name.split(@date_separator) next unless date_parts.size == required_date_parts index_time = set_time_to_beginning_of_day(unprefixed_index_name, separator=@date_separator) expiry = (cutoff_date_beginning_of_day - index_time).to_i if expiry > 0 @indices_expired[index_name] = expiry end end end end
get_cluster_state(index_name, metric='metadata')
click to toggle source
# File lib/elasticshelf.rb, line 162 def get_cluster_state(index_name, metric='metadata') try do index_metadata = @client.cluster.state(:index => index_name, :metric => metric) index_metadata end end
get_elasticsearch_version()
click to toggle source
# File lib/elasticshelf.rb, line 321 def get_elasticsearch_version try do info_hash = @client.info return nil if blank? info_hash return nil unless info_hash['status'] == 200 return nil if blank? info_hash['version'] info_hash['version']['number'] end end
get_index_state(index_name, metric='metadata')
click to toggle source
# File lib/elasticshelf.rb, line 155 def get_index_state(index_name, metric='metadata') try do index_metadata = @client.cluster.state(:index => index_name, :metric => metric) index_metadata['metadata']['indices'][index_name]['state'] end end
get_lucene_version()
click to toggle source
# File lib/elasticshelf.rb, line 331 def get_lucene_version try do info_hash = @client.info return nil if blank? info_hash return nil unless info_hash['status'] == 200 return nil if blank? info_hash['version'] info_hash['version']['lucene_version'] end end
get_settings_index(index_name)
click to toggle source
# File lib/elasticshelf.rb, line 127 def get_settings_index(index_name) reset_errors_results try do @results[index_name] = @client.indices.get_settings(:index => index_name) end end
index_closed?(index_name)
click to toggle source
# File lib/elasticshelf.rb, line 80 def index_closed?(index_name) # catch exceptions when index doesn't exist try do index_metadata = @client.cluster.state(:index => index_name, :metric => 'metadata') index_metadata['metadata']['indices'][index_name]['state'] == 'close' end end
indices_prefix=(str)
click to toggle source
# File lib/elasticshelf.rb, line 48 def indices_prefix=(str) @indices_prefix = str.chomp('*') + '*' end
new(arguments={})
click to toggle source
# File lib/elasticshelf.rb, line 24 def new(arguments={}) @client = Elasticsearch::Transport::Client.new(arguments) @wait_for_completion = true @ignore_unavailable = true @include_global_state = false @snapshot_indices_expired_name = nil @date_separator = '.' @indices_prefix = 'logstash-' @cutoff_days = -1 @repo_type = 'fs' @repo_compressed = true @indices = [] @indices_affected = {} @indices_rejected = {} @indices_expired = {} @indices_closed = {} @indices_deleted = {} @indices_withered = {} @results = {} @errors = {} self end
open_index(index_name)
click to toggle source
# File lib/elasticshelf.rb, line 88 def open_index(index_name) try do @results[index_name] = @client.indices.open(index: index_name) end end
snapshot_create()
click to toggle source
# File lib/elasticshelf.rb, line 240 def snapshot_create # naming errors as seen in elasticsearch log: # Invalid snapshot name [2014-04-01 20:12:07 UTC], must not contain whitespace # Invalid snapshot name [2014-04-01_20:12:38_UTC], must be lowercase try do if blank? @indices # the whole cluster: @results = @client.snapshot.create(repository: @repo, snapshot: @snapshot, wait_for_completion: @wait_for_completion, body: { ignore_unavailable: @ignore_unavailable, include_global_state: @include_global_state } ) else @results = @client.snapshot.create(repository: @repo, snapshot: @snapshot, wait_for_completion: @wait_for_completion, body: { indices: @indices, ignore_unavailable: @ignore_unavailable, include_global_state: @include_global_state } ) end end end
snapshot_create_repository()
click to toggle source
# File lib/elasticshelf.rb, line 169 def snapshot_create_repository @results = @errors = {} # ensure the location is accessible by Elasticsearch @repo_type = 'fs' if blank? @repo_type @repo_location = '/var/lib/elasticsearch_snapshots' if blank? @repo_location @repo_compressed = true if blank? @repo_compressed try do @results = @client.snapshot.create_repository( repository: @repo, body: { type: @repo_type, settings: { location: @repo_location, compress: @repo_compressed } } ) end end
snapshot_delete()
click to toggle source
# File lib/elasticshelf.rb, line 280 def snapshot_delete try do @results = @client.snapshot.delete(repository: @repo, snapshot: @snapshot) end end
snapshot_delete_repository()
click to toggle source
# File lib/elasticshelf.rb, line 200 def snapshot_delete_repository @results = @errors = {} try do @results = @client.snapshot.delete_repository(repository: @repo) end end
snapshot_expired_indices()
click to toggle source
# File lib/elasticshelf.rb, line 207 def snapshot_expired_indices @results = @errors = {} find_expired_indices if blank? @indices_expired @results = {"result" => "no expired indices found"} return end open_indices = [] @indices_expired.keys.each do |index_name| # only open indices can be used in a snapshot: if index_closed?(index_name) @indices_rejected[index_name] = "closed index can't snapshot" else @indices_affected[index_name] = "open can snapshot" open_indices << index_name end end if blank? open_indices @results = {"result" => "none of the expired indices are open so can't snapshot"} return end @indices = open_indices.join(',') # allow user to override this name: if blank? @snapshot_indices_expired_name @snapshot = (Time.now.utc.to_s.gsub(' ', '_') + "_#{@indices_prefix.chomp('*')}" + "_cutoff_#{@cutoff_days}_days").downcase else @snapshot = @snapshot_indices_expired_name end @results = snapshot_create end
snapshot_get()
click to toggle source
# File lib/elasticshelf.rb, line 269 def snapshot_get reset_errors_results try do if blank? @snapshot @results = @client.snapshot.get(repository: @repo, snapshot: '_all') else @results = @client.snapshot.get(repository: @repo, snapshot: @snapshot) end end end
snapshot_get_repository()
click to toggle source
# File lib/elasticshelf.rb, line 189 def snapshot_get_repository @results = @errors = {} try do if @repo @results = @client.snapshot.get_repository(repository: @repo) else @results = @client.snapshot.get_repository end end end
snapshot_restore()
click to toggle source
# File lib/elasticshelf.rb, line 286 def snapshot_restore try do if blank? @indices # the whole snapshot: @results = @client.snapshot.restore( repository: @repo, snapshot: @snapshot, wait_for_completion: @wait_for_completion, body: { ignore_unavailable: @ignore_unavailable, include_global_state: @include_global_state } ) else # note: @indices must be a string of comma separated index names @results = @client.snapshot.restore( repository: @repo, snapshot: @snapshot, wait_for_completion: @wait_for_completion, body: { indices: @indices, ignore_unavailable: @ignore_unavailable, include_global_state: @include_global_state } ) end end end
wither_expired_indices()
click to toggle source
# File lib/elasticshelf.rb, line 146 def wither_expired_indices reset_errors_results find_expired_indices if blank? @indices_expired @indices_expired.each do |k,v| wither_index(k) puts "withered '#{k}' expired #{v} days ago" end end
wither_index(index_name)
click to toggle source
# File lib/elasticshelf.rb, line 134 def wither_index(index_name) try do # don't try to disable bloom filter on a closed index, it will re-open it: if index_closed?(index_name) @errors = {"failed" => "Unable to wither (disable bloom filter) index #{index_name}: It's already closed."} return else @indices_withered[index_name] = @client.indices.put_settings(:index => index_name, :body => 'index.codec.bloom.load=false') end end end
Private Instance Methods
blank?(some_object)
click to toggle source
# File lib/elasticshelf.rb, line 371 def blank?(some_object) some_object = some_object.strip if some_object.is_a? String some_object.respond_to?(:empty?) ? some_object.empty? : !some_object end
reset_errors_results()
click to toggle source
# File lib/elasticshelf.rb, line 376 def reset_errors_results @results = {} @errors = {} end
set_time_to_beginning_of_day(index_timestamp, separator='.')
click to toggle source
# File lib/elasticshelf.rb, line 367 def set_time_to_beginning_of_day(index_timestamp, separator='.') return DateTime.strptime(index_timestamp, ['%Y', '%m', '%d'].join(separator)) end
try() { || ... }
click to toggle source
# File lib/elasticshelf.rb, line 359 def try(&block) yield rescue Exception => ex # a consistent response for all exceptions similar to Elasticsearch: @errors = {"error" => "#{ex.class}: #{ex.message}"} # raise ex end