class Vmpooler::API::V1
Public Instance Methods
account_for_starting_vm(template, vm)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 172 def account_for_starting_vm(template, vm) user = backend.hget("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}", 'user') has_token_result = has_token? backend.sadd("vmpooler__migrating__#{template}", vm) backend.hset("vmpooler__active__#{template}", vm, Time.now) backend.hset("vmpooler__vm__#{vm}", 'checkout', Time.now) if Vmpooler::API.settings.config[:auth] and has_token_result backend.hset("vmpooler__vm__#{vm}", 'token:token', request.env['HTTP_X_AUTH_TOKEN']) backend.hset("vmpooler__vm__#{vm}", 'token:user', user) if config['vm_lifetime_auth'].to_i > 0 backend.hset("vmpooler__vm__#{vm}", 'lifetime', config['vm_lifetime_auth'].to_i) end end end
atomically_allocate_vms(payload)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 199 def atomically_allocate_vms(payload) result = { 'ok' => false } failed = false vms = [] validate_token(backend) if Vmpooler::API.settings.config[:auth] and has_token? payload.each do |requested, count| count.to_i.times do |_i| vmname, vmpool, vmtemplate = fetch_single_vm(requested) if vmname account_for_starting_vm(vmpool, vmname) vms << [vmpool, vmname, vmtemplate] metrics.increment("checkout.success.#{vmpool}") update_user_metrics('allocate', vmname) if Vmpooler::API.settings.config[:config]['usage_stats'] else failed = true metrics.increment("checkout.empty.#{requested}") break end end end if failed vms.each do |(vmpool, vmname, _vmtemplate)| return_vm_to_ready_state(vmpool, vmname) end status 503 else vms.each do |(_vmpool, vmname, vmtemplate)| update_result_hosts(result, vmtemplate, vmname) end result['ok'] = true result['domain'] = config['domain'] if config['domain'] end result end
backend()
click to toggle source
# File lib/vmpooler/api/v1.rb, line 15 def backend Vmpooler::API.settings.redis end
check_ondemand_request(request_id)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 1027 def check_ondemand_request(request_id) result = { 'ok' => false } request_hash = backend.hgetall("vmpooler__odrequest__#{request_id}") if request_hash.empty? result['message'] = "no request found for request_id '#{request_id}'" return result end result['request_id'] = request_id result['ready'] = false result['ok'] = true status 202 case request_hash['status'] when 'ready' result['ready'] = true Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, _count| instances = backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}") if result.key?(platform_alias) result[platform_alias][:hostname] = result[platform_alias][:hostname] + instances else result[platform_alias] = { 'hostname': instances } end end result['domain'] = config['domain'] if config['domain'] status 200 when 'failed' result['message'] = "The request failed to provision instances within the configured ondemand_request_ttl '#{config['ondemand_request_ttl']}'" status 200 when 'deleted' result['message'] = 'The request has been deleted' status 200 else Parsing.get_platform_pool_count(request_hash['requested']) do |platform_alias, pool, count| instance_count = backend.scard("vmpooler__#{request_id}__#{platform_alias}__#{pool}") instances_pending = count.to_i - instance_count.to_i if result.key?(platform_alias) && result[platform_alias].key?(:ready) result[platform_alias][:ready] = (result[platform_alias][:ready].to_i + instance_count).to_s result[platform_alias][:pending] = (result[platform_alias][:pending].to_i + instances_pending).to_s else result[platform_alias] = { 'ready': instance_count.to_s, 'pending': instances_pending.to_s } end end end result end
checkoutlock()
click to toggle source
# File lib/vmpooler/api/v1.rb, line 43 def checkoutlock Vmpooler::API.settings.checkoutlock end
component_to_test(match, labels_string)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 239 def component_to_test(match, labels_string) return if labels_string.nil? labels_string_parts = labels_string.split(',') labels_string_parts.each do |part| key, value = part.split('=') next if value.nil? return value if key == match end 'none' end
config()
click to toggle source
# File lib/vmpooler/api/v1.rb, line 23 def config Vmpooler::API.settings.config[:config] end
count_selection(selection)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 75 def count_selection(selection) result = {} selection.uniq.each do |poolname| result[poolname] = selection.count(poolname) end result end
delete_ondemand_request(request_id)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 1080 def delete_ondemand_request(request_id) result = { 'ok' => false } platforms = backend.hget("vmpooler__odrequest__#{request_id}", 'requested') unless platforms result['message'] = "no request found for request_id '#{request_id}'" return result end if backend.hget("vmpooler__odrequest__#{request_id}", 'status') == 'deleted' result['message'] = 'the request has already been deleted' else backend.hset("vmpooler__odrequest__#{request_id}", 'status', 'deleted') Parsing.get_platform_pool_count(platforms) do |platform_alias, pool, _count| backend.smembers("vmpooler__#{request_id}__#{platform_alias}__#{pool}")&.each do |vm| backend.smove("vmpooler__running__#{pool}", "vmpooler__completed__#{pool}", vm) end backend.del("vmpooler__#{request_id}__#{platform_alias}__#{pool}") end backend.expire("vmpooler__odrequest__#{request_id}", 129_600_0) end status 200 result['ok'] = true result end
evaluate_template_aliases(template, count)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 83 def evaluate_template_aliases(template, count) template_backends = [] template_backends << template if backend.sismember('vmpooler__pools', template) selection = [] aliases = get_template_aliases(template) if aliases template_backends += aliases weighted_pools = get_pool_weights(template_backends) if weighted_pools.count > 1 && weighted_pools.count == template_backends.count pickup = Pickup.new(weighted_pools) count.to_i.times do selection << pickup.pick end else count.to_i.times do selection << template_backends.sample end end end count_selection(selection) end
extract_templates_from_query_params(params)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 976 def extract_templates_from_query_params(params) payload = {} params.split('+').each do |template| payload[template] ||= 0 payload[template] += 1 end payload end
fetch_single_vm(template)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 107 def fetch_single_vm(template) template_backends = [template] aliases = Vmpooler::API.settings.config[:alias] if aliases template_backends += aliases[template] if aliases[template].is_a?(Array) template_backends << aliases[template] if aliases[template].is_a?(String) pool_index = pool_index(pools) weighted_pools = {} template_backends.each do |t| next unless pool_index.key? t index = pool_index[t] clone_target = pools[index]['clone_target'] || config['clone_target'] next unless config.key?('backend_weight') weight = config['backend_weight'][clone_target] if weight weighted_pools[t] = weight end end if weighted_pools.count == template_backends.count pickup = Pickup.new(weighted_pools) selection = pickup.pick template_backends.delete(selection) template_backends.unshift(selection) else first = template_backends.sample template_backends.delete(first) template_backends.unshift(first) end end checkoutlock.synchronize do template_backends.each do |template_backend| vms = backend.smembers("vmpooler__ready__#{template_backend}") next if vms.empty? vms.reverse.each do |vm| ready = vm_ready?(vm, config['domain']) if ready smoved = backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__running__#{template_backend}", vm) if smoved return [vm, template_backend, template] else metrics.increment("checkout.smove.failed.#{template_backend}") return [nil, nil, nil] end else backend.smove("vmpooler__ready__#{template_backend}", "vmpooler__completed__#{template_backend}", vm) metrics.increment("checkout.nonresponsive.#{template_backend}") end end end [nil, nil, nil] end end
generate_ondemand_request(payload)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 403 def generate_ondemand_request(payload) result = { 'ok': false } requested_instances = payload.reject { |k, _v| k == 'request_id' } if too_many_requested?(requested_instances) result['message'] = "requested amount of instances exceeds the maximum #{config['max_ondemand_instances_per_request']}" status 403 return result end score = Time.now.to_i request_id = payload['request_id'] request_id ||= generate_request_id result['request_id'] = request_id if backend.exists?("vmpooler__odrequest__#{request_id}") result['message'] = "request_id '#{request_id}' has already been created" status 409 metrics.increment('ondemandrequest_generate.duplicaterequests') return result end status 201 platforms_with_aliases = [] requested_instances.each do |poolname, count| selection = evaluate_template_aliases(poolname, count) selection.map { |selected_pool, selected_pool_count| platforms_with_aliases << "#{poolname}:#{selected_pool}:#{selected_pool_count}" } end platforms_string = platforms_with_aliases.join(',') return result unless backend.zadd('vmpooler__provisioning__request', score, request_id) backend.hset("vmpooler__odrequest__#{request_id}", 'requested', platforms_string) if Vmpooler::API.settings.config[:auth] and has_token? backend.hset("vmpooler__odrequest__#{request_id}", 'token:token', request.env['HTTP_X_AUTH_TOKEN']) backend.hset("vmpooler__odrequest__#{request_id}", 'token:user', backend.hget("vmpooler__token__#{request.env['HTTP_X_AUTH_TOKEN']}", 'user')) end result['domain'] = config['domain'] if config['domain'] result[:ok] = true metrics.increment('ondemandrequest_generate.success') result end
generate_request_id()
click to toggle source
# File lib/vmpooler/api/v1.rb, line 449 def generate_request_id SecureRandom.uuid end
get_pool_weights(template_backends)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 57 def get_pool_weights(template_backends) pool_index = pool_index(pools) weighted_pools = {} template_backends.each do |t| next unless pool_index.key? t index = pool_index[t] clone_target = pools[index]['clone_target'] || config['clone_target'] next unless config.key?('backend_weight') weight = config['backend_weight'][clone_target] if weight weighted_pools[t] = weight end end weighted_pools end
get_template_aliases(template)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 47 def get_template_aliases(template) result = [] aliases = Vmpooler::API.settings.config[:alias] if aliases result += aliases[template] if aliases[template].is_a?(Array) template_backends << aliases[template] if aliases[template].is_a?(String) end result end
invalid_pool(payload)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 1019 def invalid_pool(payload) invalid = [] payload.each do |pool, _clone_target| invalid << pool unless pool_exists?(pool) end invalid end
invalid_template_or_path(payload)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 1008 def invalid_template_or_path(payload) invalid = [] payload.each do |pool, template| invalid << pool unless pool_exists?(pool) invalid << pool unless template.include? '/' invalid << pool if template[0] == '/' invalid << pool if template[-1] == '/' end invalid end
invalid_template_or_size(payload)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 995 def invalid_template_or_size(payload) invalid = [] payload.each do |pool, size| invalid << pool unless pool_exists?(pool) unless is_integer?(size) invalid << pool next end invalid << pool unless Integer(size) >= 0 end invalid end
invalid_templates(payload)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 987 def invalid_templates(payload) invalid = [] payload.keys.each do |template| invalid << template unless pool_exists?(template) end invalid end
metrics()
click to toggle source
# File lib/vmpooler/api/v1.rb, line 19 def metrics Vmpooler::API.settings.metrics end
need_auth!()
click to toggle source
# File lib/vmpooler/api/v1.rb, line 35 def need_auth! validate_auth(backend) end
need_token!()
click to toggle source
# File lib/vmpooler/api/v1.rb, line 39 def need_token! validate_token(backend) end
pool_exists?(template)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 31 def pool_exists?(template) Vmpooler::API.settings.config[:pool_names].include?(template) end
pools()
click to toggle source
# File lib/vmpooler/api/v1.rb, line 27 def pools Vmpooler::API.settings.config[:pools] end
reset_pool(payload)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 332 def reset_pool(payload) result = { 'ok' => false } payload.each do |poolname, _count| backend.sadd('vmpooler__poolreset', poolname) end status 201 result['ok'] = true result end
return_vm_to_ready_state(template, vm)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 165 def return_vm_to_ready_state(template, vm) backend.srem("vmpooler__migrating__#{template}", vm) backend.hdel("vmpooler__active__#{template}", vm) backend.hdel("vmpooler__vm__#{vm}", 'checkout', 'token:token', 'token:user') backend.smove("vmpooler__running__#{template}", "vmpooler__ready__#{template}", vm) end
sync_clone_targets()
click to toggle source
# File lib/vmpooler/api/v1.rb, line 383 def sync_clone_targets pool_index = pool_index(pools) clone_target_configs = backend.hgetall('vmpooler__config__clone_target') clone_target_configs&.each do |poolname, clone_target| next unless pool_index.include? poolname pools[pool_index[poolname]]['clone_target'] = clone_target end end
sync_pool_sizes()
click to toggle source
# File lib/vmpooler/api/v1.rb, line 373 def sync_pool_sizes pool_index = pool_index(pools) poolsize_configs = backend.hgetall('vmpooler__config__poolsize') poolsize_configs&.each do |poolname, size| next unless pool_index.include? poolname pools[pool_index[poolname]]['size'] = size.to_i end end
sync_pool_templates()
click to toggle source
# File lib/vmpooler/api/v1.rb, line 363 def sync_pool_templates pool_index = pool_index(pools) template_configs = backend.hgetall('vmpooler__config__template') template_configs&.each do |poolname, template| next unless pool_index.include? poolname pools[pool_index[poolname]]['template'] = template end end
too_many_requested?(payload)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 393 def too_many_requested?(payload) payload&.each do |poolname, count| next unless count.to_i > config['max_ondemand_instances_per_request'] metrics.increment("ondemandrequest_fail.toomanyrequests.#{poolname}") return true end false end
update_clone_target(payload)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 343 def update_clone_target(payload) result = { 'ok' => false } pool_index = pool_index(pools) pools_updated = 0 sync_clone_targets payload.each do |poolname, clone_target| unless pools[pool_index[poolname]]['clone_target'] == clone_target pools[pool_index[poolname]]['clone_target'] = clone_target backend.hset('vmpooler__config__clone_target', poolname, clone_target) pools_updated += 1 status 201 end end status 200 unless pools_updated > 0 result['ok'] = true result end
update_pool_size(payload)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 292 def update_pool_size(payload) result = { 'ok' => false } pool_index = pool_index(pools) pools_updated = 0 sync_pool_sizes payload.each do |poolname, size| unless pools[pool_index[poolname]]['size'] == size.to_i pools[pool_index[poolname]]['size'] = size.to_i backend.hset('vmpooler__config__poolsize', poolname, size) pools_updated += 1 status 201 end end status 200 unless pools_updated > 0 result['ok'] = true result end
update_pool_template(payload)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 312 def update_pool_template(payload) result = { 'ok' => false } pool_index = pool_index(pools) pools_updated = 0 sync_pool_templates payload.each do |poolname, template| unless pools[pool_index[poolname]]['template'] == template pools[pool_index[poolname]]['template'] = template backend.hset('vmpooler__config__template', poolname, template) pools_updated += 1 status 201 end end status 200 unless pools_updated > 0 result['ok'] = true result end
update_result_hosts(result, template, vm)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 189 def update_result_hosts(result, template, vm) result[template] ||= {} if result[template]['hostname'] result[template]['hostname'] = Array(result[template]['hostname']) result[template]['hostname'].push(vm) else result[template]['hostname'] = vm end end
update_user_metrics(operation, vmname)
click to toggle source
# File lib/vmpooler/api/v1.rb, line 251 def update_user_metrics(operation, vmname) backend.multi backend.hget("vmpooler__vm__#{vmname}", 'tag:jenkins_build_url') backend.hget("vmpooler__vm__#{vmname}", 'token:user') backend.hget("vmpooler__vm__#{vmname}", 'template') jenkins_build_url, user, poolname = backend.exec poolname = poolname.gsub('.', '_') if user user = user.gsub('.', '_') else user = 'unauthenticated' end metrics.increment("user.#{user}.#{operation}.#{poolname}") if jenkins_build_url if jenkins_build_url.include? 'litmus' # Very simple filter for Litmus jobs - just count them coming through for the moment. metrics.increment("usage_litmus.#{user}.#{operation}.#{poolname}") return end url_parts = jenkins_build_url.split('/')[2..-1] jenkins_instance = url_parts[0].gsub('.', '_') value_stream_parts = url_parts[2].split('_') value_stream_parts = value_stream_parts.map { |s| s.gsub('.', '_') } value_stream = value_stream_parts.shift branch = value_stream_parts.pop project = value_stream_parts.shift job_name = value_stream_parts.join('_') build_metadata_parts = url_parts[3] component_to_test = component_to_test('RMM_COMPONENT_TO_TEST_NAME', build_metadata_parts) metrics.increment("usage_jenkins_instance.#{jenkins_instance}.#{value_stream}.#{operation}.#{poolname}") metrics.increment("usage_branch_project.#{branch}.#{project}.#{operation}.#{poolname}") metrics.increment("usage_job_component.#{job_name}.#{component_to_test}.#{operation}.#{poolname}") end rescue StandardError => e puts 'd', "[!] [#{poolname}] failed while evaluating usage labels on '#{vmname}' with an error: #{e}" end