class S3sync::Syncer
Public Class Methods
new()
click to toggle source
# File lib/s3sync/syncer.rb, line 7 def initialize @s3 = AWS::S3.new @log = S3sync::Logger.new end
Public Instance Methods
download(s3_location, local_path)
click to toggle source
# File lib/s3sync/syncer.rb, line 37 def download(s3_location, local_path) bucket_name, *folders = s3url_to_bucket_folder s3_location destination_folder = File.absolute_path(local_path) @log.info "Downloading" local_files = local_files(local_path) remote_files(bucket_name, folders) do |s3| next if FileDiff::same_file? s3, local_files[s3[:key]] begin destination_file = File.join destination_folder, s3[:key] @log.info "#{s3[:file].public_url} => #{destination_file}" s3_download s3, destination_file rescue Exception => e @log.error e end end @log.info "Done" rescue Exception => e @log.error e end
upload(local_path, s3_url)
click to toggle source
# File lib/s3sync/syncer.rb, line 12 def upload(local_path, s3_url) bucket_name, *folders = s3url_to_bucket_folder s3_url @log.info "Uploading files" local_files = local_files(local_path) remote_files(bucket_name, folders) do |s3| source = local_files[s3[:key]] local_files.delete s3[:key] if FileDiff::same_file? source, s3 end local_files.each do |key,item| begin s3_key = File.join folders, key @log.info "#{item[:file]} => s3://#{bucket_name}/#{s3_key}" s3_upload item, bucket_name, s3_key rescue Exception => e @log.error e end end @log.info "Done" rescue Exception => e @log.error e end
Private Instance Methods
ensure_folder_exists(folder)
click to toggle source
# File lib/s3sync/syncer.rb, line 65 def ensure_folder_exists(folder) FileUtils.mkdir_p(folder) unless File.directory?(folder) end
last_modified(item)
click to toggle source
# File lib/s3sync/syncer.rb, line 73 def last_modified(item) item[:last_modified] end
local_files(path)
click to toggle source
# File lib/s3sync/syncer.rb, line 100 def local_files(path) return {} unless File.directory? path results = {} Find.find(path) do |item| next if File.directory? item file_path = item.match(/#{path}\/?(.*)/)[1] results[file_path] = { key:file_path, last_modified: File.mtime(item), content_length: File.size(item), file: File.absolute_path(item) } end results end
remote_files(bucket, folders) { |remote_item(object, relative_path)| ... }
click to toggle source
Yielding the remote s3 files and doing a 2 pass filter as better performance than computing a diff of 2 complete directory listings
# File lib/s3sync/syncer.rb, line 128 def remote_files(bucket, folders) objects = @s3.buckets[bucket].objects.with_prefix(File.join(folders)) objects.each do |object| relative_path = object.key.split(/\//).drop folders.length next if object.content_length.nil? or object.content_length == 0 or relative_path.empty? yield remote_item(object, relative_path) end end
remote_item(object, relative_path)
click to toggle source
# File lib/s3sync/syncer.rb, line 117 def remote_item(object, relative_path) last_modified = Time.parse object.metadata['last_modified'] rescue 0 { key:File.join(relative_path), last_modified:last_modified, content_length:object.content_length, file:object } end
s3_download(item, destination_file)
click to toggle source
# File lib/s3sync/syncer.rb, line 77 def s3_download(item, destination_file) ensure_folder_exists File.dirname(destination_file) File.open(destination_file, 'wb') do |f| item[:file].read {|chunk| f.write chunk } end update_last_modified destination_file, last_modified(item) end
s3_upload(item, bucket, key)
click to toggle source
# File lib/s3sync/syncer.rb, line 87 def s3_upload(item, bucket, key) object = @s3.buckets[bucket].objects[key] object.write(:file => item[:file]) object.metadata['last_modified'] = last_modified(item) end
s3url_to_bucket_folder(s3_location)
click to toggle source
# File lib/s3sync/syncer.rb, line 93 def s3url_to_bucket_folder(s3_location) s3_path = s3_location.match(/s3:\/\/(.*)/)[1] rescue nil raise 'Invalid S3 url must be s3://bucket form' unless s3_path s3_path.split /\// end
safe_encode(text)
click to toggle source
# File lib/s3sync/syncer.rb, line 61 def safe_encode(text) text.force_encoding('ASCII-8BIT').encode('UTF-8', :invalid => :replace, :undef => :replace, :replace => '?') end
update_last_modified(file, time)
click to toggle source
# File lib/s3sync/syncer.rb, line 69 def update_last_modified(file, time) FileUtils.touch file, mtime:time end