module Hasta::IdentityReducer

Used by any EMR job that requires an identity reducer

Public Class Methods

reduce(_, data_source, data_sink = InMemoryDataSink.new) click to toggle source
# File lib/hasta/identity_reducer.rb, line 9
def self.reduce(_, data_source, data_sink = InMemoryDataSink.new)
  Hasta.logger.debug "Starting Identity Reducer"
  SortedDataSource.new(data_source).each_line do |line|
    data_sink << line.rstrip
  end

  data_sink.close.tap { Hasta.logger.debug "Finished Identity Reducer" }
end