module RailsRedshiftReplicator

Constants

VERSION

Public Class Methods

add_replicable(hash) click to toggle source
# File lib/rails_redshift_replicator.rb, line 126
def add_replicable(hash)
  logger.debug I18n.t(:replicable_added, table_name: hash.keys.first, scope: :rails_redshift_replicator) 
  RailsRedshiftReplicator.replicables.merge! hash
end
analyze(*args) click to toggle source
# File lib/rails_redshift_replicator.rb, line 176
def analyze(*args)
  Tools::Analyze.new(*args).perform
end
base_exporter_types() click to toggle source

Lists exporters names

# File lib/rails_redshift_replicator.rb, line 181
def base_exporter_types
  [
    'identity_replicator',
    'timed_replicator',
    'full_replicator'
  ]
end
check_args(tables) click to toggle source
# File lib/rails_redshift_replicator.rb, line 164
def check_args(tables)
  if tables == []
    error_message = I18n.t(:must_specify_tables, scope: :rails_redshift_replicator)
    logger.error error_message
    raise StandardError.new(error_message)
  end
end
connection() click to toggle source

Redshift connection @return [PG::Connection]

# File lib/rails_redshift_replicator.rb, line 224
def connection
  @redshift ||= PG.connect(redshift_connection_params)
end
debug_mode=(value) click to toggle source
# File lib/rails_redshift_replicator.rb, line 112
def debug_mode=(value)
  logger.level = value == true ? Logger::DEBUG : Logger::WARN
  @@debug_mode = value
end
define_defaults() click to toggle source

@note Useful for testing

# File lib/rails_redshift_replicator.rb, line 34
def define_defaults
  @@replicables = {}.with_indifferent_access
  @@logger = RLogger.new(STDOUT).tap{ |l| l.level = Logger::WARN }
  
  # Connection parameters for Redshift. Defaults to environment variables.
  @@redshift_connection_params = {
    host: ENV['RRR_REDSHIFT_HOST'],
    dbname: ENV['RRR_REDSHIFT_DATABASE'],
    port: ENV['RRR_REDSHIFT_PORT'],
    user: ENV['RRR_REDSHIFT_USER'],
    password: ENV['RRR_REDSHIFT_PASSWORD']
  }

  # AWS S3 Replication bucket credentials. Defaults to environment variables.
  @@aws_credentials = {
    key: ENV['RRR_AWS_ACCESS_KEY_ID'],
    secret: ENV['RRR_AWS_SECRET_ACCESS_KEY']
  }

  # AWS S3 replication bucket parameters.
  # region defaults to environment variable or US East (N. Virginia)
  # bucket defaults to environment variable
  @@s3_bucket_params = {
    region: (ENV['RRR_REPLICATION_REGION'] || 'us-east-1'),
    bucket: ENV['RRR_REPLICATION_BUCKET'],
    prefix: ENV['RRR_REPLICATION_PREFIX']
  }

  # see [http://docs.aws.amazon.com/redshift/latest/dg/r_COPY.html]
  # You can add other keys aside from changing these.
  # The keys won't be used on the copy commands. Just their values.
  # To remove one of the defaults, set it to nil.
  # @example:
  #   @@copy_options = {
  #     statupdate: nil,
  #   }
  @@copy_options = {
    statupdate: 'STATUPDATE TRUE',
    acceptinvchars: 'ACCEPTINVCHARS',
    empty: 'EMPTYASNULL',
    truncate: 'TRUNCATECOLUMNS'
  }

  # Number of slices available on Redshift cluster. Used to split export files. Defaults to 1.
  # see [http://docs.aws.amazon.com/redshift/latest/dg/t_splitting-data-files.html]
  @@redshift_slices = 1

  # Folder to store temporary replication files until the S3 upload. Defaults to /tmp
  @@local_replication_path = '/tmp'

  # Command or path to executable that splits files
  @@split_command = 'split'

  # Command or path to executable that compresses files to gzip
  @@gzip_command = 'gzip'

  # Enable debug mode to output messages to STDOUT. Default to false
  @@debug_mode = false

  # Defines how many replication records are kept in history. Default to nil keeping full history.
  @@history_cap = nil

  # Preferred format for export file
  @@preferred_format = 'csv'

  # Maximum number of retries for a replication before cancelling and starting another
  @@max_retries = nil

  # If deletes should be tracked and propagated to redshift
  @@enable_delete_tracking = false

  # If exported files on s3 should be deleted after imported
  @@delete_s3_file_after_import = true

  return nil
end
Also aliased as: reload
export(*tables) click to toggle source

@see .replicate

# File lib/rails_redshift_replicator.rb, line 153
def export(*tables)
  check_args(tables)
  replicable_definitions(tables_to_perform(tables)).each { |_, replicable| replicable.export }
end
history_cap=(value) click to toggle source
# File lib/rails_redshift_replicator.rb, line 117
def history_cap=(value)
  @@history_cap = value && [value,2].max
end
import(*tables) click to toggle source

@see .replicate

# File lib/rails_redshift_replicator.rb, line 159
def import(*tables)
  check_args(tables)
  replicable_definitions(tables_to_perform(tables)).each { |_, replicable| replicable.import }
end
reload()
Alias for: define_defaults
reload_replicables() click to toggle source
# File lib/rails_redshift_replicator.rb, line 131
def reload_replicables
  replicables = {}
  replicables.each do |name, replicable|
    add_replicable(name => RailsRedshiftReplicator::Replicable.new(replicable.replication_type, replicable.options))
  end
end
replicable_definitions(tables) click to toggle source

@retuns [Hash] subset of key pairs of replicables

# File lib/rails_redshift_replicator.rb, line 201
def replicable_definitions(tables)
  RailsRedshiftReplicator.replicables.select { |k,_| k.to_s.in? tables.map(&:to_s) }
end
replicable_tables() click to toggle source

All replicable tables registered in RailsRedshiftReplicator eighter from the model or directly. @return [Array<String>] tables

# File lib/rails_redshift_replicator.rb, line 192
def replicable_tables
  RailsRedshiftReplicator.replicables.keys.map(&:to_s)
end
replicable_target_tables() click to toggle source
# File lib/rails_redshift_replicator.rb, line 196
def replicable_target_tables
  RailsRedshiftReplicator.replicables.map{ |k,v| v[:target_table] }
end
replicate(*tables) click to toggle source

Performs full replication (export + import) @param models [Array<Symbol>, Argument list] activerecord models to export or :all @example Replicate user and post models.

RedshiftReplicator.replicate(:user, :publication)

@example Replicate all models

RedshiftReplicator.replicate(:all)
# File lib/rails_redshift_replicator.rb, line 144
def replicate(*tables)
  check_args(tables)
  replicable_definitions(tables_to_perform(tables)).each do |_, replicable|
    replication = replicable.export
    replicable.import
  end
end
setup() { |self| ... } click to toggle source

@return [RedshiftReplicator]

# File lib/rails_redshift_replicator.rb, line 122
def setup
  yield self
end
tables_to_perform(tables) click to toggle source

Returns tables to export. :all selects all eligible @returns [Array<String>] tables to export

# File lib/rails_redshift_replicator.rb, line 207
def tables_to_perform(tables)
  tables = Array(tables).map(&:to_s)
  if tables[0] == 'all'
    replicable_tables
  else
    (replicable_tables & tables).tap do |selected|
      warn_if_unreplicable tables-selected
    end
  end
end
vacuum(*args) click to toggle source
# File lib/rails_redshift_replicator.rb, line 172
def vacuum(*args)
  Tools::Vacuum.new(*args).perform
end
warn_if_unreplicable(tables) click to toggle source
# File lib/rails_redshift_replicator.rb, line 218
def warn_if_unreplicable(tables)
  tables.each { |table| logger.warn I18n.t(:table_not_replicable, table_name: table, scope: :rails_redshift_replicator) }
end