module Hodor::Oozie

Public Class Methods

append_prefix_to_filename(file_name, prefix = '') click to toggle source
# File lib/hodor/api/oozie.rb, line 181
def append_prefix_to_filename(file_name, prefix = '')
  insert_index =  file_name.rindex(File::SEPARATOR)
  String.new(file_name).insert((insert_index.nil? ? 0 : insert_index+1) , prefix)
end
build_rest_param(filter) click to toggle source
# File lib/hodor/api/oozie.rb, line 32
def build_rest_param(filter)
  params = filter.map { |match|
    case match
    when :killed; "status%3DKILLED"
    when :succeeded; "status%3DSUCCEEDED"
    end
  }

  if params.size > 0
    filter_exp = "filter=#{params.join(';')}"
  else
    filter_exp = ""
  end
end
change_job(job_path, filter = nil) click to toggle source
# File lib/hodor/api/oozie.rb, line 99
def change_job(job_path, filter = nil)
  job_by_path(job_path, true, filter)
end
compose_job_file(direct_job = nil, prefix = '') click to toggle source

collect all job.properties.erb files up to root of repo and compose them together in top down order (i.e. deeper directories override properties in higher directories.) If direct job properties file is provided, properties will be interpolated using values in that file.

# File lib/hodor/api/oozie.rb, line 139
def compose_job_file(direct_job = nil, prefix = '')
  if direct_job.nil?
    pwd = Dir.pwd
    paths = env.paths_from_root(pwd)
    composite_jobfile = paths.inject('') { |result, path|
      jobfile = File.expand_path('job.properties.erb', path)
      if File.exists?(jobfile)
        result << "\nFrom Job File '#{jobfile}':\n"
        result << File.read(jobfile)
      end
      result
    }
    FileUtils.mkdir './.tmp' unless Dir.exists?('./.tmp')
    composite_properties_file = File.expand_path(".tmp/runjob.properties.erb", pwd)
    File.open(composite_properties_file, "w") do |f|
      f.puts composite_jobfile
    end
    out_file = composite_properties_file.sub(/\.erb$/,'')
    dest_file = generate_and_write_job_file(out_file, composite_properties_file, prefix)
  else
    raise "Job file '#{direct_job}' not found" unless File.exists?(direct_job)
    if direct_job.end_with?('.erb')
      out_file = "/tmp/#{File.basename(direct_job.sub(/.erb$/,''))}"
      dest_file = generate_and_write_job_file(out_file, direct_job, prefix)
    else
      direct_job
      out_file = append_prefix_to_filename(direct_job, prefix)
      unless prefix.blank?
        FileUtils.cp(direct_job, out_file)
        dest_file = out_file
      end
    end
  end
  dest_file
end
deploy_job(job, clean_deploy) click to toggle source
# File lib/hodor/api/oozie.rb, line 187
def deploy_job(job, clean_deploy)
  select_job(job)
  fail "No deploy section for job '#{job}'." if !env.job.has_key?("deploy")
  if env.job[:deploy].nil?
    fail "Nothing to deploy. Check the deploy section of your jobs.yml file"
  else
    env.job[:deploy].split.each { |path|
      hdfs.put_dir File.expand_path(path, env.root), { clean: clean_deploy }
    }
  end
end
env() click to toggle source
# File lib/hodor/api/oozie.rb, line 16
def env
  Hodor::Environment.instance
end
generate_and_write_job_file(file_name, in_file, prefix = '') click to toggle source
# File lib/hodor/api/oozie.rb, line 175
def generate_and_write_job_file(file_name, in_file, prefix = '')
  out_file = append_prefix_to_filename(file_name, prefix)
  File.open(out_file, 'w') { |f| f.write(env.erb_load(in_file)) }
  out_file
end
hdfs() click to toggle source
# File lib/hodor/api/oozie.rb, line 24
def hdfs
  Hodor::Hdfs.instance
end
job_by_id(job_id, filter = nil) click to toggle source
# File lib/hodor/api/oozie.rb, line 47
def job_by_id(job_id, filter = nil)
  if (job_id.nil?)
    result = Hodor::Oozie::Query.new status: [:running_first]
  else
    if job_id =~ /job_\d+/
      result = HadoopJob.new(session.current_id, job_id)
    else
      if filter
        response = session.get_job_state(job_id, build_rest_param(filter))
      else
        response = session.get_job_state(job_id)
      end
      json = JSON.parse(response)
      job_type = json["toString"]
      case job_type.split(" ").first.downcase.to_sym
      when :bundle;
        result = Bundle.new(json)
      when :coordinator;
        result = Coordinator.new(json)
      when :workflow;
        result = Workflow.new(json)
      when :action;
        result = Action.new(json)
      when :coordinatoraction;
        result = Materialization.new(json)
      else
      end
    end
  end
end
job_by_path(job_path, make_current = false, filter = nil) click to toggle source
# File lib/hodor/api/oozie.rb, line 78
def job_by_path(job_path, make_current = false, filter = nil)
  if job_path.nil? || job_path.eql?(".")
    movement = :none
  elsif job_path.eql?("/")
    movement = :root
  elsif job_path.eql?("b") || job_path.eql?("back") || 
    job_path.eql?("u") || job_path.eql?("up") || job_path.eql?("..")
    movement = :up
  elsif job_path.eql?("d") || job_path.eql?("down") || 
    job_path.eql?("f") || job_path.eql?("forward") || job_path.length < 5
    movement = :down
  else
    movement = :jump
  end

  job_id = session.job_relative(movement, job_path)
  job = job_by_id(job_id, filter)
  session.make_current(job) if make_current
  job
end
logger() click to toggle source
# File lib/hodor/api/oozie.rb, line 28
def logger
  env.logger
end
run_job(job = nil, dry_run = false, file_prefix = '') click to toggle source

If job references a job.properties or job.properties.erb file, that file will be used directly to interpolate job property values.

# File lib/hodor/api/oozie.rb, line 201
def run_job(job = nil, dry_run = false, file_prefix = '')
  if job && (job =~ /job.properties.erb$/ || job =~ /job.properties/)
    jobfile = compose_job_file(job, file_prefix)
  else
    select_job(job)
    jobfile = compose_job_file(nil, file_prefix)
  end
  unless dry_run
    runfile = env.deploy_tmp_file(jobfile)
    env.ssh "oozie job -oozie :oozie_url -config #{runfile} -run", echo: true, echo_cmd: true
  end
  jobfile
end
select_job(job) click to toggle source
# File lib/hodor/api/oozie.rb, line 103
def select_job(job)
  # load jobs.yml file
  pwd = Dir.pwd
  if File.exists?("jobs.yml")
    jobs = env.yml_load(File.expand_path('jobs.yml', pwd))
    marked_jobs = jobs.keys.select { |key| key.start_with?('^') }
    marked_jobs.each { |mark|
      jobs[mark[1..-1]] = jobs[mark]
    }
    if job.nil?
      # No job explicitly specified, so look for a
      # marked job (i.e. job starting with ^)
      jobs.each_pair { |key, val|
        if key.to_s.strip.start_with?('^')
          job = key.to_s
        end
      }
      fail "You must specify which job from jobs.yml to run" if !job
    end
    jobs = jobs.symbolize_keys
    if !jobs.has_key?(job.to_sym)
      caret = "^#{job.to_s}"
      fail "Job '#{job}' was not defined in jobs.yml" if !jobs.has_key?(caret.to_sym)
    end
    selected_job = jobs[job.to_sym]
    env.select_job(selected_job)
  else
    fail "No jobs.yml file exists in the current directory. You must specify a jobs.yml file"
  end
end
session() click to toggle source
# File lib/hodor/api/oozie.rb, line 20
def session
  Hodor::Oozie::Session.instance
end