class S3Ranger::SyncCommand
Public Class Methods
cmp(list1, list2)
click to toggle source
# File lib/s3ranger/sync.rb, line 124 def self.cmp list1, list2 l1 = {}; list1.each {|e| l1[e.path] = e} l2 = {}; list2.each {|e| l2[e.path] = e} same, to_add_to_2 = [], [] l1.each do |key, value| value2 = l2.delete key if value2.nil? to_add_to_2 << value elsif value2.size == value.size same << value else to_add_to_2 << value end end to_remove_from_2 = l2.values [same, to_add_to_2, to_remove_from_2] end
new(args, source, destination)
click to toggle source
# File lib/s3ranger/sync.rb, line 146 def initialize args, source, destination @args = args @source = source @destination = destination end
parse_params(args)
click to toggle source
# File lib/s3ranger/sync.rb, line 183 def self.parse_params args # Reading the arbitrary parameters from the command line and getting # modifiable copies to parse source, destination = args; return nil if source.nil? or destination.nil? # Sync from one s3 to another is currently not supported if remote_prefix? source and remote_prefix? destination raise WrongUsage.new(nil, 'Both arguments can\'t be on S3') end # C'mon, there's rsync out there if !remote_prefix? source and !remote_prefix? destination raise WrongUsage.new(nil, 'One argument must be on S3') end source, destination = process_destination source, destination return [Location.new(*source), Location.new(*destination)] end
process_destination(source, destination)
click to toggle source
# File lib/s3ranger/sync.rb, line 237 def self.process_destination source, destination source, destination = source.dup, destination.dup # don't repeat slashes source.squeeze! '/' destination.squeeze! '/' # Making sure that local paths won't break our stuff later source.gsub!(/^\.\//, '') destination.gsub!(/^\.\//, '') # Parsing the final destination destination = process_file_destination source, destination, "" # here's where we find out what direction we're going source_is_s3 = remote_prefix? source # canonicalize the S3 stuff remote_prefix = source_is_s3 ? source : destination bucket, remote_prefix = remote_prefix.split ":" remote_prefix ||= "" # Just making sure we preserve the direction if source_is_s3 [[remote_prefix, bucket], destination] else [source, [remote_prefix, bucket]] end end
process_file_destination(source, destination, file="")
click to toggle source
# File lib/s3ranger/sync.rb, line 208 def self.process_file_destination source, destination, file="" if not file.empty? sub = (remote_prefix? source) ? source.split(":")[1] : source file = file.gsub(/^#{sub}/, '') end # no slash on end of source means we need to append the last src dir to # dst prefix testing for empty isn't good enough here.. needs to be # "empty apart from potentially having 'bucket:'" if source =~ %r{/$} File.join [destination, file] else if remote_prefix? source _, name = source.split ":" File.join [destination, File.basename(name || ""), file] else source = /^\/?(.*)/.match(source)[1] # Corner case: the root of the remote path is empty, we don't want to # add an unnecessary slash here. if destination.end_with? ':' File.join [destination + source, file] else File.join [destination, source, file] end end end end
remote_prefix?(prefix)
click to toggle source
# File lib/s3ranger/sync.rb, line 202 def self.remote_prefix?(prefix) # allow for dos-like things e.g. C:\ to be treated as local even with # colon. prefix.include? ':' and not prefix.match '^[A-Za-z]:[\\\\/]' end
Public Instance Methods
download_files(destination, source, list)
click to toggle source
# File lib/s3ranger/sync.rb, line 314 def download_files destination, source, list list.each {|e| path = File.join destination.path, e.path if @args.verbose puts " + #{source}#{e.path} => #{path}" end unless @args.dry_run obj = @args.s3.buckets[source.bucket].objects[e.path] # Making sure this new file will have a safe shelter FileUtils.mkdir_p File.dirname(path) # Downloading and saving the files File.open(path, 'wb') do |file| obj.read do |chunk| file.write chunk end end end } end
read_tree_remote(location)
click to toggle source
# File lib/s3ranger/sync.rb, line 267 def read_tree_remote location dir = location.path dir += '/' if not dir.empty? or dir.end_with?('/') @args.s3.buckets[location.bucket].objects.with_prefix(dir || "").to_a.collect {|obj| Node.new location.path, obj.key, obj.content_length } end
read_trees(source, destination)
click to toggle source
# File lib/s3ranger/sync.rb, line 275 def read_trees source, destination if source.local? source_tree = LocalDirectory.new(source.path).list_files destination_tree = read_tree_remote destination else source_tree = read_tree_remote source destination_tree = LocalDirectory.new(destination.path).list_files end [source_tree, destination_tree] end
remove_files(remote, list)
click to toggle source
# File lib/s3ranger/sync.rb, line 301 def remove_files remote, list if @args.verbose list.each {|e| puts " - #{remote}#{e.path}" } end unless @args.dry_run @args.s3.buckets[remote.bucket].objects.delete_if { |obj| list.include? obj.key } end end
remove_local_files(destination, source, list)
click to toggle source
# File lib/s3ranger/sync.rb, line 338 def remove_local_files destination, source, list list.each {|e| path = File.join destination.path, e.path if @args.verbose puts " * #{e.path} => #{path}" end unless @args.dry_run FileUtils.rm_rf path end } end
run()
click to toggle source
# File lib/s3ranger/sync.rb, line 152 def run # Reading the source and destination using our helper method if (source, destination, bucket = self.class.parse_params [@source, @destination]).nil? raise WrongUsage.new(nil, 'Need a source and a destination') end # Getting the trees source_tree, destination_tree = read_trees source, destination # Getting the list of resources to be exchanged between the two peers _, to_add, to_remove = self.class.cmp source_tree, destination_tree # Removing the items matching the exclude pattern if requested to_add.select! { |e| begin (e.path =~ /#{@args.exclude}/).nil? rescue RegexpError => exc raise WrongUsage.new nil, exc.message end } if @args.exclude # Calling the methods that perform the actual IO if source.local? upload_files destination, to_add remove_files destination, to_remove unless @args.keep else download_files destination, source, to_add remove_local_files destination, source, to_remove unless @args.keep end end
upload_files(remote, list)
click to toggle source
# File lib/s3ranger/sync.rb, line 287 def upload_files remote, list list.each do |e| if @args.verbose puts " + #{e.full} => #{remote}#{e.path}" end unless @args.dry_run if File.file? e.path @args.s3.buckets[remote.bucket].objects[e.path].write Pathname.new(e.path), :acl => @args.acl end end end end