class S3Uploader::Uploader
Attributes
logger[W]
Public Class Methods
new(options = {})
click to toggle source
# File lib/s3_uploader/s3_uploader.rb, line 18 def initialize(options = {}) @options = { :destination_dir => '', :threads => DEFAULT_THREADS_NUMBER, :s3_key => ENV['S3_KEY'], :s3_secret => ENV['S3_SECRET'], :public => false, :region => DEFAULT_AWS_REGION, :metadata => {}, :path_style => false, :regexp => nil, :gzip => false, :gzip_working_dir => nil, :time_range => Time.at(0)..(Time.now + (60 * 60 * 24)) }.merge(options) @logger = @options[:logger] || Logger.new(STDOUT) if @options[:gzip] && @options[:gzip_working_dir].nil? raise 'gzip_working_dir required when using gzip' end if @options[:connection] @connection = @options[:connection] else if @options[:s3_key].nil? || @options[:s3_secret].nil? raise "Missing access keys" end @connection = Fog::Storage.new({ :provider => 'AWS', :aws_access_key_id => @options[:s3_key], :aws_secret_access_key => @options[:s3_secret], :region => @options[:region], :path_style => @options[:path_style] }) end if !@options[:destination_dir].to_s.empty? && !@options[:destination_dir].end_with?('/') @options[:destination_dir] << '/' end end
Public Instance Methods
upload(source_dir, bucket)
click to toggle source
# File lib/s3_uploader/s3_uploader.rb, line 63 def upload(source_dir, bucket) raise 'Source directory is requiered' if source_dir.to_s.empty? source = source_dir.dup source << '/' unless source.end_with?('/') raise 'Source must be a directory' unless File.directory?(source) gzip_working_dir = @options[:gzip_working_dir] if @options[:gzip] && !gzip_working_dir.to_s.empty? gzip_working_dir << '/' unless gzip_working_dir.end_with?('/') if gzip_working_dir.start_with?(source) raise 'gzip_working_dir may not be located within source-folder' end end total_size = 0 files = Queue.new regexp = @options[:regexp] Dir.glob(File.join(source, '**/*')) .select { |f| !File.directory?(f) }.each do |f| if (regexp.nil? || File.basename(f).match(regexp)) && @options[:time_range].cover?(File.mtime(f)) if @options[:gzip] && File.extname(f) != '.gz' dir, base = File.split(f) dir = dir.sub(source, gzip_working_dir) gz_file = File.join(dir, [ base, '.gz' ].join) @logger.info("Compressing #{f}") FileUtils.mkdir_p(dir) Zlib::GzipWriter.open(gz_file) do |gz| gz.mtime = File.mtime(f) gz.orig_name = f File.open(f, 'rb') do |fi| while (block_in = fi.read(BLOCK_SIZE)) do gz.write block_in end end end files << gz_file total_size += File.size(gz_file) else files << f total_size += File.size(f) end end end directory = @connection.directories.new(:key => bucket) start = Time.now total_files = files.size file_number = 0 @mutex = Mutex.new threads = [] @options[:threads].times do |i| threads[i] = Thread.new do until files.empty? @mutex.synchronize do file_number += 1 Thread.current["file_number"] = file_number end file = files.pop rescue nil if file key = file.sub(source, '').sub(gzip_working_dir.to_s, '') dest = [ @options[:destination_dir], key ].join body = File.open(file) @logger.info(["[", Thread.current["file_number"], "/", total_files, "] Uploading ", key, " to s3://#{bucket}/#{dest}" ].join) directory.files.create( :key => dest, :body => body, :public => @options[:public], :metadata => @options[:metadata] ) body.close end end end end threads.each { |t| t.join } finish = Time.now elapsed = finish.to_f - start.to_f mins, secs = elapsed.divmod 60.0 @logger.info("Uploaded %d (%.#{0}f KB) in %d:%04.2f" % [total_files, total_size / KILO_SIZE, mins.to_i, secs]) end