module RailsRedshiftReplicator
Constants
- VERSION
Public Class Methods
add_replicable(hash)
click to toggle source
# File lib/rails_redshift_replicator.rb, line 126 def add_replicable(hash) logger.debug I18n.t(:replicable_added, table_name: hash.keys.first, scope: :rails_redshift_replicator) RailsRedshiftReplicator.replicables.merge! hash end
analyze(*args)
click to toggle source
# File lib/rails_redshift_replicator.rb, line 176 def analyze(*args) Tools::Analyze.new(*args).perform end
base_exporter_types()
click to toggle source
Lists exporters names
# File lib/rails_redshift_replicator.rb, line 181 def base_exporter_types [ 'identity_replicator', 'timed_replicator', 'full_replicator' ] end
check_args(tables)
click to toggle source
# File lib/rails_redshift_replicator.rb, line 164 def check_args(tables) if tables == [] error_message = I18n.t(:must_specify_tables, scope: :rails_redshift_replicator) logger.error error_message raise StandardError.new(error_message) end end
connection()
click to toggle source
Redshift connection @return [PG::Connection]
# File lib/rails_redshift_replicator.rb, line 224 def connection @redshift ||= PG.connect(redshift_connection_params) end
debug_mode=(value)
click to toggle source
# File lib/rails_redshift_replicator.rb, line 112 def debug_mode=(value) logger.level = value == true ? Logger::DEBUG : Logger::WARN @@debug_mode = value end
define_defaults()
click to toggle source
@note Useful for testing
# File lib/rails_redshift_replicator.rb, line 34 def define_defaults @@replicables = {}.with_indifferent_access @@logger = RLogger.new(STDOUT).tap{ |l| l.level = Logger::WARN } # Connection parameters for Redshift. Defaults to environment variables. @@redshift_connection_params = { host: ENV['RRR_REDSHIFT_HOST'], dbname: ENV['RRR_REDSHIFT_DATABASE'], port: ENV['RRR_REDSHIFT_PORT'], user: ENV['RRR_REDSHIFT_USER'], password: ENV['RRR_REDSHIFT_PASSWORD'] } # AWS S3 Replication bucket credentials. Defaults to environment variables. @@aws_credentials = { key: ENV['RRR_AWS_ACCESS_KEY_ID'], secret: ENV['RRR_AWS_SECRET_ACCESS_KEY'] } # AWS S3 replication bucket parameters. # region defaults to environment variable or US East (N. Virginia) # bucket defaults to environment variable @@s3_bucket_params = { region: (ENV['RRR_REPLICATION_REGION'] || 'us-east-1'), bucket: ENV['RRR_REPLICATION_BUCKET'], prefix: ENV['RRR_REPLICATION_PREFIX'] } # see [http://docs.aws.amazon.com/redshift/latest/dg/r_COPY.html] # You can add other keys aside from changing these. # The keys won't be used on the copy commands. Just their values. # To remove one of the defaults, set it to nil. # @example: # @@copy_options = { # statupdate: nil, # } @@copy_options = { statupdate: 'STATUPDATE TRUE', acceptinvchars: 'ACCEPTINVCHARS', empty: 'EMPTYASNULL', truncate: 'TRUNCATECOLUMNS' } # Number of slices available on Redshift cluster. Used to split export files. Defaults to 1. # see [http://docs.aws.amazon.com/redshift/latest/dg/t_splitting-data-files.html] @@redshift_slices = 1 # Folder to store temporary replication files until the S3 upload. Defaults to /tmp @@local_replication_path = '/tmp' # Command or path to executable that splits files @@split_command = 'split' # Command or path to executable that compresses files to gzip @@gzip_command = 'gzip' # Enable debug mode to output messages to STDOUT. Default to false @@debug_mode = false # Defines how many replication records are kept in history. Default to nil keeping full history. @@history_cap = nil # Preferred format for export file @@preferred_format = 'csv' # Maximum number of retries for a replication before cancelling and starting another @@max_retries = nil # If deletes should be tracked and propagated to redshift @@enable_delete_tracking = false # If exported files on s3 should be deleted after imported @@delete_s3_file_after_import = true return nil end
Also aliased as: reload
export(*tables)
click to toggle source
@see .replicate
# File lib/rails_redshift_replicator.rb, line 153 def export(*tables) check_args(tables) replicable_definitions(tables_to_perform(tables)).each { |_, replicable| replicable.export } end
history_cap=(value)
click to toggle source
# File lib/rails_redshift_replicator.rb, line 117 def history_cap=(value) @@history_cap = value && [value,2].max end
import(*tables)
click to toggle source
@see .replicate
# File lib/rails_redshift_replicator.rb, line 159 def import(*tables) check_args(tables) replicable_definitions(tables_to_perform(tables)).each { |_, replicable| replicable.import } end
reload_replicables()
click to toggle source
# File lib/rails_redshift_replicator.rb, line 131 def reload_replicables replicables = {} replicables.each do |name, replicable| add_replicable(name => RailsRedshiftReplicator::Replicable.new(replicable.replication_type, replicable.options)) end end
replicable_definitions(tables)
click to toggle source
@retuns [Hash] subset of key pairs of replicables
# File lib/rails_redshift_replicator.rb, line 201 def replicable_definitions(tables) RailsRedshiftReplicator.replicables.select { |k,_| k.to_s.in? tables.map(&:to_s) } end
replicable_tables()
click to toggle source
All replicable tables registered in RailsRedshiftReplicator
eighter from the model or directly. @return [Array<String>] tables
# File lib/rails_redshift_replicator.rb, line 192 def replicable_tables RailsRedshiftReplicator.replicables.keys.map(&:to_s) end
replicable_target_tables()
click to toggle source
# File lib/rails_redshift_replicator.rb, line 196 def replicable_target_tables RailsRedshiftReplicator.replicables.map{ |k,v| v[:target_table] } end
replicate(*tables)
click to toggle source
Performs full replication (export + import) @param models [Array<Symbol>, Argument list] activerecord models to export or :all @example Replicate user and post models.
RedshiftReplicator.replicate(:user, :publication)
@example Replicate all models
RedshiftReplicator.replicate(:all)
# File lib/rails_redshift_replicator.rb, line 144 def replicate(*tables) check_args(tables) replicable_definitions(tables_to_perform(tables)).each do |_, replicable| replication = replicable.export replicable.import end end
setup() { |self| ... }
click to toggle source
@return [RedshiftReplicator]
# File lib/rails_redshift_replicator.rb, line 122 def setup yield self end
tables_to_perform(tables)
click to toggle source
Returns tables to export. :all selects all eligible @returns [Array<String>] tables to export
# File lib/rails_redshift_replicator.rb, line 207 def tables_to_perform(tables) tables = Array(tables).map(&:to_s) if tables[0] == 'all' replicable_tables else (replicable_tables & tables).tap do |selected| warn_if_unreplicable tables-selected end end end
vacuum(*args)
click to toggle source
# File lib/rails_redshift_replicator.rb, line 172 def vacuum(*args) Tools::Vacuum.new(*args).perform end
warn_if_unreplicable(tables)
click to toggle source
# File lib/rails_redshift_replicator.rb, line 218 def warn_if_unreplicable(tables) tables.each { |table| logger.warn I18n.t(:table_not_replicable, table_name: table, scope: :rails_redshift_replicator) } end