module Mobilize::Hadoop

Public Class Methods

clusters() click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 15
def Hadoop.clusters
  Hadoop.config['clusters'].keys
end
config() click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 3
def Hadoop.config
  Base.config('hadoop')
end
default_cluster() click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 19
def Hadoop.default_cluster
  Hadoop.clusters.first
end
exec_path(cluster) click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 7
def Hadoop.exec_path(cluster)
  Hadoop.config['clusters'][cluster]['exec_path']
end
gateway_node(cluster) click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 11
def Hadoop.gateway_node(cluster)
  Hadoop.config['clusters'][cluster]['gateway_node']
end
job(cluster,command,user,file_hash={}) click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 31
def Hadoop.job(cluster,command,user,file_hash={})
  command = ["-",command].join unless command.starts_with?("-")
  Hadoop.run(cluster,"job -fs #{Hdfs.root(cluster)} #{command}",user,file_hash).ie do |r|
    r.class==Array ? r.first : r
  end
end
job_list(cluster) click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 38
def Hadoop.job_list(cluster)
  raw_list = Hadoop.job(cluster,"list")
  raw_list.split("\n")[1..-1].join("\n").tsv_to_hash_array
end
job_status(cluster,hadoop_job_id) click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 43
def Hadoop.job_status(cluster,hadoop_job_id)
  raw_status = Hadoop.job(cluster,"status #{hadoop_job_id}",{})
  dhash_status = raw_status.strip.split("\n").map do |sline|
                   delim_index = [sline.index("="),sline.index(":")].compact.min
                   if delim_index
                     key,value = [sline[0..delim_index-1],sline[(delim_index+1)..-1]]
                     {key.strip => value.strip}
                   end
                 end.compact
  hash_status = {}
  dhash_status.each{|h| hash_status.merge!(h)}
  hash_status
end
output_dir() click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 23
def Hadoop.output_dir
  Hadoop.config['output_dir']
end
read_limit() click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 27
def Hadoop.read_limit
  Hadoop.config['read_limit']
end
run(cluster,command,user_name,file_hash={}) click to toggle source
# File lib/mobilize-hdfs/handlers/hadoop.rb, line 57
def Hadoop.run(cluster,command,user_name,file_hash={})
  h_command = if command.starts_with?("hadoop")
                command.sub("hadoop",Hadoop.exec_path(cluster))
              else
                "#{Hadoop.exec_path(cluster)} #{command}"
              end
  gateway_node = Hadoop.gateway_node(cluster)
  stage_path=nil
  Ssh.run(gateway_node,h_command,user_name,stage_path,file_hash)
end