class S3Rsync::Sync
Public Class Methods
new(opt)
click to toggle source
# File lib/s3rsync/sync.rb, line 9 def initialize(opt) @log = init_logger(opt['log-path']) @log.debug "opt: #{opt.inspect}" @params = {} @params[:path] = File.expand_path(opt['path']) @params[:s3_prefix] = opt['s3-prefix'] @params[:config_path] = opt['config-path'] @params[:s3cmd_conf] = @params[:config_path].sub('.json', '_s3rsync.conf') @params[:dry_run] = opt['dry-run'] @params[:make_public] = opt['make-public'] @params[:enable_lock] = opt['enable-lock'] @log.debug "@params[:config_path]: #{@params[:config_path]}" begin conf = JSON.parse File.read(@params[:config_path]) rescue Exception => e error_exit "ERROR: opening or parsing config file failed: #{e.inspect}" end @log.debug "conf: #{conf}" @params[:s3_bucket] = conf['env.static.s3.bucket'] @params[:s3_access] = conf['env.static.s3.key.user'] @params[:s3_secret] = conf['env.static.s3.key.secret'] @params[:statsd_host] = conf['env.metrics.host'] || 'localhost' @params[:statsd_port] = conf['env.metrics.port'] || 8125 @log.debug "@params: #{@params.inspect}" @statsd = init_statsd @params[:statsd_host], @params[:statsd_port] @log.debug "@statsd: #{@statsd.inspect}" init_s3cmd_conf @s3cmd = "s3cmd" end
Public Instance Methods
get_lock(name = 'common')
click to toggle source
# File lib/s3rsync/sync.rb, line 41 def get_lock(name = 'common') lockfile = Lockfile.new( "/tmp/s3rsync_#{name}.lock", :timeout => 3, :max_age => 3600, :debug => !ENV['DEBUG'].nil? ) end
init_s3cmd_conf()
click to toggle source
# File lib/s3rsync/sync.rb, line 50 def init_s3cmd_conf @log.debug "@params[:s3cmd_conf]: #{@params[:s3cmd_conf]}" if !File.exists? @params[:s3cmd_conf] @log.info "Generating dynamic s3cfg: #{@params[:s3cmd_conf]}" begin erb_path = File.join(File.dirname(__FILE__), 'templates', 's3cmd_conf.erb') erb_template = IO.read erb_path @log.debug "erb_template: #{erb_template.inspect}" erb = ERB.new erb_template rendered_template = erb.result(binding) @log.debug "rendered_template: #{rendered_template}" File.write @params[:s3cmd_conf], rendered_template rescue Exception => e error_exit "generating s3cmd config failed: #{e.inspect}" end @log.info "Dynamic s3cfg generated: #{@params[:s3cmd_conf]}" end end
run(mode = :upload)
click to toggle source
# File lib/s3rsync/sync.rb, line 69 def run(mode = :upload) init_s3cmd_conf valid_file_upload = (mode == :upload && File.exists?(@params[:path])) if !Dir.exists?(@params[:path]) && !valid_file_upload @log.error "Local dir or file not exist on disk: #{@params[:path]}" return 4 end @params[:path] += '/' if Dir.exists?(@params[:path]) sync_locations = "#{@params[:path]} s3://#{@params[:s3_bucket]}/#{@params[:s3_prefix]}#{@params[:path]}" @log.debug "original sync_locations: #{sync_locations}" if @params[:enable_lock] lockfile = get_lock(mode) @log.debug "lockfile: #{lockfile.inspect}" if lockfile.locked? @log.warn "lock file found, exiting: #{lockfile.inspect}" return 5 end end if mode == :download @log.debug "reversing sync_locations" sync_locations = sync_locations.split(' ').reverse.join(' ') @log.debug "download sync_locations: #{sync_locations}" end sync_output = 1 begin lockfile.lock if @params[:enable_lock] s3cmd_args = {} s3cmd_args['-c'] = @params[:s3cmd_conf] s3cmd_args['--dry-run'] = '' if @params[:dry_run] s3cmd_args['--acl-public'] = '' if @params[:make_public] @log.debug "s3cmd_args: #{s3cmd_args.inspect}" compiled_args = s3cmd_args.to_a.flatten.reject { |v| v.empty? }.join(' ') @log.debug "compiled_args: #{compiled_args.inspect}" cmd = "#{@s3cmd} #{compiled_args} sync #{sync_locations}" @log.debug "cmd: #{cmd}" sync_output = pass_cli cmd @log.debug "sync_output: #{sync_output.inspect}" rescue Exception => e sync_output = 1 @log.error "something wrong while trying to run the sync: #{e.inspect}" ensure lockfile.unlock if @params[:enable_lock] end if (sync_output == 1) || sync_output.downcase.include?('error') @log.error "errors found in output: #{sync_output}" return 1 else return 0 end end