class Hodor::Environment

Attributes

logger[R]
options[RW]

Public Class Methods

new() click to toggle source
# File lib/hodor/environment.rb, line 87
def initialize
  @options = {} 
end

Public Instance Methods

[](key) click to toggle source
# File lib/hodor/environment.rb, line 201
def [](key)
  target_cluster[key]
end
abs_path(path) click to toggle source
# File lib/hodor/environment.rb, line 179
def abs_path(path)
  "#{root}/#{path}"
end
clean?() click to toggle source
# File lib/hodor/environment.rb, line 79
def clean?
  options[:clean]
end
deploy_tmp_file(local_file, opts = {}) click to toggle source
# File lib/hodor/environment.rb, line 315
def deploy_tmp_file local_file, opts = {}
  deploy_path = "/tmp/#{File.basename(local_file, ".*")}-#{username}-#{hostname}#{File.extname(local_file)}"
  run_local %Q[scp #{local_file} #{settings[:ssh_user]}@#{settings[:ssh_host]}:#{deploy_path}],
              echo: true, echo_cmd: true
  deploy_path
end
dryrun?() click to toggle source
# File lib/hodor/environment.rb, line 75
def dryrun?
  options[:dryrun]
end
env() click to toggle source
# File lib/hodor/environment.rb, line 205
def env
  target_cluster
end
erb_load(filename, suppress_erb=false) click to toggle source
# File lib/hodor/environment.rb, line 49
def erb_load(filename, suppress_erb=false)
  if File.exists?(filename)
    file_contents = File.read(filename)
    sub_content = suppress_erb ? file_contents : erb_sub(file_contents)
    sub_content
  elsif !filename.start_with?(root)
    erb_load(File.join(root, filename))
  end
end
erb_sub(erb_body) click to toggle source
# File lib/hodor/environment.rb, line 45
def erb_sub(erb_body)
  ERB.new(erb_body).result(self.instance_eval { binding })
end
extract_sudoer(trailing) click to toggle source

user_args

strip off the "-u <username>" argument, which hadoop commands don't understand.
The username has to be set using an environment variable instead.  This is a
convience method to facilitate this swapping around that is necessary in several
hadoop commands (fs, oozie etc.).
# File lib/hodor/environment.rb, line 292
def extract_sudoer(trailing)
  username_next = false
  username = nil # nil assignment avoids "unused variable" warning
  args = []
  trailing.each { |arg|
    if arg.eql?("-u")
      username_next = arg.eql?("-u")
    elsif username_next
      username = arg
      username_next = false
    else
      args << arg
    end
  }

  return [username, args]
end
hadoop_env() click to toggle source
# File lib/hodor/environment.rb, line 83
def hadoop_env
  ENV['HADOOP_ENV'] || 'sandbox'
end
has_key?(key) click to toggle source
# File lib/hodor/environment.rb, line 217
def has_key? key
  target_cluster.has_key? key
end
hostname() click to toggle source
# File lib/hodor/environment.rb, line 229
def hostname
  Socket.gethostname
end
job() click to toggle source
# File lib/hodor/environment.rb, line 213
def job
  @job || {}
end
kvp_expand(script) click to toggle source

Run an ssh command, performing any optional variable expansion on the command line that might be necessary.

The following variable expansions are supported: env.ssh %Q[ssh ${ssh_addr} …] # calls “ssh_addr” function env.ssh %Q[ssh ${env} …] # retrieves value from hash env.ssh %Q[ssh :ssh_user …] # retrieves value from hash env.ssh %Q[ssh #{env.ssh_addr} …] # skip variable expansion.

Use normal string interpolation instead
# File lib/hodor/environment.rb, line 252
def kvp_expand(script)
  script.gsub!(/:[^\s]+|\$\{.+?\}/) { |match|
    begin
      if match.start_with?(':')
        k = match[1..-1].to_sym
        if settings.has_key?(k)
          val = settings[k]
        else
          val = match
        end
      else
        cv = match.split(/\{|\}/)
        cv = cv[1].split(/\[|\]/)
        fn = cv[0].to_sym
        if self.respond_to?(fn)
          rtn = self.send(fn)
          if cv.size == 1
            val = rtn
          else
            k = cv[1]
            k = k[1..-1].to_sym if k.start_with?(':')
            val = rtn[k]
          end
          val
        else 
          match
        end
      end
    rescue StandardError
      match
    end
  }
  script
end
load_settings() click to toggle source
# File lib/hodor/environment.rb, line 91
def load_settings
  target_env = hadoop_env.to_sym
  @clusters = yml_load('config/clusters.yml')

  @target_cluster = @clusters[target_env]
  if @target_cluster.nil?
    raise "The target environment '#{target_env}' was not defined in the config/clusters.yml file. Aborting..."
  end

  if File.exist?('config/local.yml')
    @target_cluster.merge! yml_load('config/local.yml')
  end

  @target_cluster[:target] = target_env

  @loaded = true
end
logger_id() click to toggle source
# File lib/hodor/environment.rb, line 31
def logger_id
  'MainLogger'
end
path_on_disc(path = nil) click to toggle source
# File lib/hodor/environment.rb, line 155
def path_on_disc(path = nil)
  relpath = path_on_github(path)
  "#{root}/#{relpath}".sub(/\/\//, '/').sub(/\/$/, '')
end
path_on_github(path = nil) click to toggle source
# File lib/hodor/environment.rb, line 126
def path_on_github(path = nil)
  if path 
    if path.start_with?('/')
      abspath = true
      lpath = path
    else
      relpath = true
    end
  end
  lpath ||= FileUtils.pwd

  lpath = lpath.sub(root, '')
  git_path = relpath ? "#{lpath}/#{path}" : lpath
  git_path = git_path.sub(/\/\//, '/').sub(/\/\.\//, '/').sub(/\/\.$/, '').sub(/\/$/, '')

  if git_path.end_with?('..')
    up_index = git_path.rindex(/[^\.]\//)
    if up_index > 0
      last_path = git_path[0..up_index]
      up_path = git_path[up_index+2..-1]
      up_paths = up_path.split('/')
      abs_path = File.expand_path(File.join(up_paths), "#{root}/#{last_path}".sub(/\/\//, '/'))
      git_path = path_on_github(abs_path)
    end
  end

  git_path
end
paths_from_root(end_path) click to toggle source
# File lib/hodor/environment.rb, line 183
def paths_from_root(end_path)
  # returns an array of paths from the root of the repo
  paths = []
  curpath = end_path
  loop do
    paths << curpath
    break if curpath == root || curpath.length < root.length || curpath.length == 0
    curpath = File.dirname(curpath)
  end
  paths.reverse
end
prefs() click to toggle source
# File lib/hodor/environment.rb, line 109
def prefs
  if @prefs.nil?
    preffile = "#{Etc.getpwuid.dir}/.hodor.yml"
    @prefs = yml_load(preffile) if File.exists?(preffile)
    @prefs ||= {}
  end
  @prefs
end
pwd(path = nil) click to toggle source
# File lib/hodor/environment.rb, line 160
def pwd(path = nil)
  if path 
    if path.start_with?('/')
      abspath = true
      lpwd = path
    else
      relpath = true
    end
  end
  lpwd ||= FileUtils.pwd
  rpwd = lpwd.sub(root, '')
  if rpwd.length < lpwd.length
    lpwd = rpwd[1..-1] if rpwd.start_with?('/')
  else
    lpwd = rpwd
  end
  relpath ? "#{lpwd}/#{path}" : lpwd
end
reset() click to toggle source
# File lib/hodor/environment.rb, line 118
def reset
  @clusters = nil
  @target_cluster = nil
  @loaded = false
  @jobs = nil
  @run = nil
end
root() click to toggle source
# File lib/hodor/environment.rb, line 21
def root
  begin
    @root = run_local "git rev-parse --show-toplevel", raise_on_error: true
  rescue Hodor::Cli::AbnormalExitStatus => ex
    puts "#{ex.message.strip}\nHodor must be run inside a Git working tree. Aborting..."
    Kernel.exit
  end if @root.nil?
  @root
end
run_local(command_line, opts = {}) click to toggle source

Alternative to system() that (optionally) echos STDOUT as it is appended, rather than after the command completes.

command_line - the shell command and arguments to execute

--terse     => if --terse appears on the command line, only
               the native output of the command is printed.
               I.e. the extra output of log4r is suppressed.

opts - options to the function, that include:

[:echo]     => true  - append stdout and stderr as it is generated
            => false - execute the command silently
[:echo_cmd] => true  - log the command to be executed
            => false - remain silent
[:raise_on_error]  => true - failed commands raise an exception
                   => false - remain silent
[:suppress_expansion]
    => true - don't expand key-value pairs in the command line
    => false - expand key-value pairs
[:sudo]
    => true - invoke with sudo, extracting username from -u argument
    => false - run without sudo
[:ssh]
    => true - prefix the command with ssh to run remotely
    => false - don't prefix command line with ssh

Returns stdout/stderr as a string

# File lib/hodor/environment.rb, line 348
def run_local command_line, opts = {}
  if opts[:sudo]
    username, args = extract_sudoer(command_line)
    command_line = "sudo -u #{username} #{args}" if username
  end

  if opts[:ssh]
    ssh_prefix = "ssh #{settings[:ssh_user]}@#{settings[:ssh_host]} "
    ssh_prefix << "-p #{settings[:ssh_port]} -T " unless settings[:ssh_port].nil?
    command_line = ssh_prefix + command_line
  end

  command_line = kvp_expand(command_line) unless opts[:suppress_expansion]
  native_output_only = command_line.include?('--terse')
  if native_output_only
    command_line.sub!(' --terse', '')
    opts[:echo] = true
    opts[:echo_cmd] = false
  end
  echo_command_output = opts[:echo] || false
  command_line = "#{command_line}"
  logger.sshcmd "$ #{command_line}" if opts[:echo_cmd]
  command_output = ""
  status = Open4::popen4(command_line) do |pid, stdin, stdout, stderr|
    command_output = capture_output(stdout, stderr, echo_command_output, native_output_only)
  end
  if status.exitstatus != 0
    raise Hodor::Cli::AbnormalExitStatus.new(status.exitstatus, command_output) if opts[:raise_on_error]
  end
  command_output.strip
rescue Hodor::Cli::AbnormalExitStatus
  raise
rescue Errno::ENOENT
  raise Hodor::Cli::CommandNotFound, "Bash Error.  Command or file arguments not found." if opts[:raise_on_error]
end
select_job(job) click to toggle source
# File lib/hodor/environment.rb, line 209
def select_job(job)
  @job = job
end
settings() click to toggle source
# File lib/hodor/environment.rb, line 221
def settings
  target_cluster
end
silent?() click to toggle source
# File lib/hodor/environment.rb, line 67
def silent?
  options[:silent]
end
ssh(script, opts = {}) click to toggle source
# File lib/hodor/environment.rb, line 310
def ssh script, opts = {}
  opts[:ssh] = true
  run_local script, opts
end
ssh_addr() click to toggle source

Compute SSH command (user, machine and port part)

# File lib/hodor/environment.rb, line 238
def ssh_addr
  va = "#{ssh_user}@#{settings[:ssh_host]}"
  va << " -p #{settings[:ssh_port] || 22}" 
end
ssh_user() click to toggle source
# File lib/hodor/environment.rb, line 225
def ssh_user
  env[:ssh_user]
end
target_cluster() click to toggle source
# File lib/hodor/environment.rb, line 195
def target_cluster
  load_settings if !@loaded || !@target_cluster
  raise "No settings for target cluster '#{hadoop_env}' were loaded" if !@loaded || !@target_cluster
  @target_cluster
end
terse?() click to toggle source
# File lib/hodor/environment.rb, line 63
def terse?
  options[:terse]
end
username() click to toggle source
# File lib/hodor/environment.rb, line 233
def username
  Etc.getpwuid(Process.uid).name
end
verbose?() click to toggle source
# File lib/hodor/environment.rb, line 71
def verbose?
  options[:verbose]
end
yml_load(filename) click to toggle source
# File lib/hodor/environment.rb, line 59
def yml_load(filename) #, suppress_erb=false)
  YAML.load(erb_load(filename, false)) # suppress_erb))
end

Private Instance Methods

capture_output(stdout, stderr, echo_command_output, native_output_only) click to toggle source
# File lib/hodor/environment.rb, line 386
def capture_output stdout, stderr, echo_command_output, native_output_only
  stdout_lines = ""
  stderr_lines = ""
  command_output = ""
  loop do
    begin
      # check whether stdout, stderr or both are
      #  ready to be read from without blocking
      IO.select([stdout,stderr]).flatten.compact.each { |io|
        # stdout, if ready, goes to stdout_lines
        stdout_lines += io.readpartial(1024) if io.fileno == stdout.fileno
        # stderr, if ready, goes to stdout_lines
        stderr_lines += io.readpartial(1024) if io.fileno == stderr.fileno
      }
      break if stdout.closed? && stderr.closed?
    rescue EOFError
      # Note, readpartial triggers the EOFError too soon.  Continue to flush the
      # pending io (via readpartial) until we have received all characters
      # out from the IO socket.
      break if stdout_lines.length == 0  &&  stderr_lines.length == 0
    ensure
      # if we acumulated any complete lines (\n-terminated)
      #  in either stdout/err_lines, output them now
      stdout_lines.sub!(/.*\n/) {
        command_output << $&
        if echo_command_output
          if native_output_only 
            puts $&.strip 
          else
            logger.stdout $&.strip
          end
      end
      ''
      }
      stderr_lines.sub!(/.*\n/) {
        command_output << $&
        if echo_command_output
          if native_output_only 
            puts $&.strip 
          else
            logger.stderr $&.strip
          end
      end
      ''
      }
    end
  end
  command_output
end