class Hasta::Tasks::Runner

Rakes task that runs a local test of an EMR job

Attributes

definition_file[RW]

Path to the AWS Data Pipeline definition file

job_id[RW]

The id of the EMR job to perform

name[RW]

Name of task.

default:

:runner
project_root[RW]

The root directory of the project containing the EMR code that is being tested

scheduled_start_time[RW]

The Scheduled Start Time to use when evaluating the definition

default:

Time.now
verbose[RW]

Use verbose output. If this is set to true, the task will print the local and remote paths of each step file it uploads to S3.

default:

true

Public Class Methods

new(*args, &task_block) click to toggle source
# File lib/hasta/tasks/runner.rb, line 45
def initialize(*args, &task_block)
  setup_ivars(args)

  desc "Runs the specified EMR job"
  task name, [:job_id, :scheduled_start_time] do |_, task_args|
    RakeFileUtils.send(:verbose, verbose) do
      if task_block
        task_block.call(*[self, task_args].slice(0, task_block.arity))
      end

      run_task verbose
    end
  end
end

Public Instance Methods

run_task(verbose) click to toggle source
# File lib/hasta/tasks/runner.rb, line 67
def run_task(verbose)
  Hasta.configure do |config|
    config.project_root = project_root
  end

  definition = Hasta::EmrJobDefinition.load(definition_file, job_id, scheduled_start_time)
  runner = Hasta::Runner.new(definition.id, definition.mapper, definition.reducer)

  result = runner.run(
    definition.data_sources,
    definition.data_sink,
    definition.ruby_files,
    definition.env
  )
end
setup_ivars(args) click to toggle source
# File lib/hasta/tasks/runner.rb, line 60
def setup_ivars(args)
  @name = args.shift || :runner
  @verbose = true
  @path = "definitions"
  @scheduled_start_time = Time.now
end