class DataAnon::Strategy::Base

Attributes

errors[RW]
fields[RW]
fields_missing_strategy[RW]
user_strategies[RW]

Public Class Methods

new(source_database, destination_database, name, user_strategies) click to toggle source
# File lib/strategy/base.rb, line 8
def initialize source_database, destination_database, name, user_strategies
  @name = name
  @user_strategies = user_strategies
  @fields = {}
  @source_database = source_database
  @destination_database = destination_database
  @fields_missing_strategy = DataAnon::Core::FieldsMissingStrategy.new name
  @errors = DataAnon::Core::TableErrors.new(@name)
  @primary_keys = []
end
using(field_strategy) click to toggle source
# File lib/strategy/base.rb, line 63
def self.using field_strategy
  @temp_fields.each { |f| @table_fields[f] = field_strategy }
end
whitelist?() click to toggle source
# File lib/strategy/base.rb, line 19
def self.whitelist?
  false
end

Public Instance Methods

anonymize(*fields, &block) click to toggle source
# File lib/strategy/base.rb, line 56
def anonymize *fields, &block
  if block.nil?
    fields.each { |f| @fields[f] = DataAnon::Strategy::Field::DefaultAnon.new(@user_strategies) }
    temp = self
    return Class.new do
      @temp_fields = fields
      @table_fields = temp.fields
      def self.using field_strategy
        @temp_fields.each { |f| @table_fields[f] = field_strategy }
      end
    end
  else
    fields.each { |f| @fields[f] = DataAnon::Strategy::Field::Anonymous.new(&block) }
  end
end
batch_size(size) click to toggle source
# File lib/strategy/base.rb, line 32
def batch_size size
  @batch_size = size
end
continue(&block) click to toggle source
# File lib/strategy/base.rb, line 52
def continue &block
  @continue_block = block
end
default_strategy(field_name) click to toggle source
# File lib/strategy/base.rb, line 76
def default_strategy field_name
  @fields_missing_strategy.missing field_name
  DataAnon::Strategy::Field::DefaultAnon.new(@user_strategies)
end
dest_table() click to toggle source
# File lib/strategy/base.rb, line 81
def dest_table
  return @dest_table unless @dest_table.nil?
  table_klass = Utils::DestinationTable.create @name, @primary_keys
  table_klass.establish_connection @destination_database if @destination_database
  @dest_table = table_klass
end
is_primary_key?(field) click to toggle source
# File lib/strategy/base.rb, line 72
def is_primary_key? field
  @primary_keys.select { |key| field == key }.length > 0
end
limit(limit) click to toggle source
# File lib/strategy/base.rb, line 36
def limit limit
  @limit = limit
end
primary_key(*fields) click to toggle source
# File lib/strategy/base.rb, line 28
def primary_key *fields
  @primary_keys = fields
end
process() click to toggle source
# File lib/strategy/base.rb, line 95
def process
  logger.debug "Processing table #{@name} with fields strategies #{@fields}"
  total = source_table.count

  if total > 0
    progress = progress_bar.new(@name, total)
    if @primary_keys.empty? || !@batch_size.present?
      process_table progress
    elsif @thread_num.present?
      process_table_in_threads progress
    else
      process_table_in_batches progress
    end
    progress.close
  end
  if source_table.respond_to?("connection_handler")
    source_table.connection_handler.clear_all_connections!
  end
end
process_fields(&block) click to toggle source
# File lib/strategy/base.rb, line 23
def process_fields &block
  self.instance_eval &block
  self
end
process_record_if(index, record) click to toggle source
# File lib/strategy/base.rb, line 188
def process_record_if index, record
  return if @skip_block && @skip_block.call(index, record)
  return if @continue_block && !@continue_block.call(index, record)

  process_record index, record
end
process_table(progress) click to toggle source
# File lib/strategy/base.rb, line 115
def process_table progress
  index = 0

  source_table_limited.each do |record|
    index += 1
    begin
      process_record_if index, record
    rescue => exception
      @errors.log_error record, exception
    end
    progress.show index
  end
end
process_table_in_batches(progress) click to toggle source
# File lib/strategy/base.rb, line 129
def process_table_in_batches progress
  logger.info "Processing table #{@name} records in batch size of #{@batch_size}"
  index = 0

  source_table_limited.find_each(:batch_size => @batch_size) do |record|
    index += 1
    begin
      process_record_if index, record
    rescue => exception
      @errors.log_error record, exception
    end
    progress.show index
  end
end
process_table_in_threads(progress) click to toggle source
# File lib/strategy/base.rb, line 144
def process_table_in_threads progress
  logger.info "Processing table #{@name} records in batch size of #{@batch_size} [THREADS]"

  index = 0
  threads = []

  source_table.find_in_batches(batch_size: @batch_size) do |records|
    until threads.count(&:alive?) <= @thread_num
      thr = threads.delete_at 0
      thr.join
      progress.show index
    end

    thr = Thread.new {
      records.each do |record|
        begin
          process_record_if index, record
          index += 1
        rescue => exception
          puts exception.inspect
          @errors.log_error record, exception
        end
      end
    }
    threads << thr
  end

  until threads.empty?
    thr = threads.delete_at 0
    thr.join
    progress.show index
  end
end
progress_bar() click to toggle source
# File lib/strategy/base.rb, line 195
def progress_bar
  @progress_bar || DataAnon::Utils::ProgressBar
end
progress_bar_class(progress_bar) click to toggle source
# File lib/strategy/base.rb, line 199
def progress_bar_class progress_bar
  @progress_bar = progress_bar
end
skip(&block) click to toggle source
# File lib/strategy/base.rb, line 48
def skip &block
  @skip_block = block
end
source_table() click to toggle source
# File lib/strategy/base.rb, line 88
def source_table
  return @source_table unless @source_table.nil?
  table_klass = Utils::SourceTable.create @name, @primary_keys
  table_klass.establish_connection @source_database
  @source_table = table_klass
end
source_table_limited() click to toggle source
# File lib/strategy/base.rb, line 178
def source_table_limited
  @source_table_limited ||= begin
    if @limit.present?
      source_table.all.limit(@limit).order(created_at: :desc)
    else
      source_table.all
    end
  end
end
thread_num(thread_num) click to toggle source
# File lib/strategy/base.rb, line 40
def thread_num thread_num
  @thread_num = thread_num
end
whitelist(*fields) click to toggle source
# File lib/strategy/base.rb, line 44
def whitelist *fields
  fields.each { |f| @fields[f] = DataAnon::Strategy::Field::Whitelist.new }
end