module Mobilize::Hadoop
Public Class Methods
clusters()
click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 15 def Hadoop.clusters Hadoop.config['clusters'].keys end
config()
click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 3 def Hadoop.config Base.config('hadoop') end
default_cluster()
click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 19 def Hadoop.default_cluster Hadoop.clusters.first end
exec_path(cluster)
click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 7 def Hadoop.exec_path(cluster) Hadoop.config['clusters'][cluster]['exec_path'] end
gateway_node(cluster)
click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 11 def Hadoop.gateway_node(cluster) Hadoop.config['clusters'][cluster]['gateway_node'] end
job(cluster,command,user,file_hash={})
click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 31 def Hadoop.job(cluster,command,user,file_hash={}) command = ["-",command].join unless command.starts_with?("-") Hadoop.run(cluster,"job -fs #{Hdfs.root(cluster)} #{command}",user,file_hash).ie do |r| r.class==Array ? r.first : r end end
job_list(cluster)
click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 38 def Hadoop.job_list(cluster) raw_list = Hadoop.job(cluster,"list") raw_list.split("\n")[1..-1].join("\n").tsv_to_hash_array end
job_status(cluster,hadoop_job_id)
click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 43 def Hadoop.job_status(cluster,hadoop_job_id) raw_status = Hadoop.job(cluster,"status #{hadoop_job_id}",{}) dhash_status = raw_status.strip.split("\n").map do |sline| delim_index = [sline.index("="),sline.index(":")].compact.min if delim_index key,value = [sline[0..delim_index-1],sline[(delim_index+1)..-1]] {key.strip => value.strip} end end.compact hash_status = {} dhash_status.each{|h| hash_status.merge!(h)} hash_status end
output_dir()
click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 23 def Hadoop.output_dir Hadoop.config['output_dir'] end
read_limit()
click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 27 def Hadoop.read_limit Hadoop.config['read_limit'] end
run(cluster,command,user_name,file_hash={})
click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 57 def Hadoop.run(cluster,command,user_name,file_hash={}) h_command = if command.starts_with?("hadoop") command.sub("hadoop",Hadoop.exec_path(cluster)) else "#{Hadoop.exec_path(cluster)} #{command}" end gateway_node = Hadoop.gateway_node(cluster) stage_path=nil Ssh.run(gateway_node,h_command,user_name,stage_path,file_hash) end