class S3Ranger::SyncCommand

Public Class Methods

cmp(list1, list2) click to toggle source
# File lib/s3ranger/sync.rb, line 124
def self.cmp list1, list2
  l1 = {}; list1.each {|e| l1[e.path] = e}
  l2 = {}; list2.each {|e| l2[e.path] = e}

  same, to_add_to_2 = [], []

  l1.each do |key, value|
    value2 = l2.delete key
    if value2.nil?
      to_add_to_2 << value
    elsif value2.size == value.size
      same << value
    else
      to_add_to_2 << value
    end
  end

  to_remove_from_2 = l2.values

  [same, to_add_to_2, to_remove_from_2]
end
new(args, source, destination) click to toggle source
# File lib/s3ranger/sync.rb, line 146
def initialize args, source, destination
  @args = args
  @source = source
  @destination = destination
end
parse_params(args) click to toggle source
# File lib/s3ranger/sync.rb, line 183
def self.parse_params args
  # Reading the arbitrary parameters from the command line and getting
  # modifiable copies to parse
  source, destination = args; return nil if source.nil? or destination.nil?

  # Sync from one s3 to another is currently not supported
  if remote_prefix? source and remote_prefix? destination
    raise WrongUsage.new(nil, 'Both arguments can\'t be on S3')
  end

  # C'mon, there's rsync out there
  if !remote_prefix? source and !remote_prefix? destination
    raise WrongUsage.new(nil, 'One argument must be on S3')
  end

  source, destination = process_destination source, destination
  return [Location.new(*source), Location.new(*destination)]
end
process_destination(source, destination) click to toggle source
# File lib/s3ranger/sync.rb, line 237
def self.process_destination source, destination
  source, destination = source.dup, destination.dup

  # don't repeat slashes
  source.squeeze! '/'
  destination.squeeze! '/'

  # Making sure that local paths won't break our stuff later
  source.gsub!(/^\.\//, '')
  destination.gsub!(/^\.\//, '')

  # Parsing the final destination
  destination = process_file_destination source, destination, ""

  # here's where we find out what direction we're going
  source_is_s3 = remote_prefix? source

  # canonicalize the S3 stuff
  remote_prefix = source_is_s3 ? source : destination
  bucket, remote_prefix = remote_prefix.split ":"
  remote_prefix ||= ""

  # Just making sure we preserve the direction
  if source_is_s3
    [[remote_prefix, bucket], destination]
  else
    [source, [remote_prefix, bucket]]
  end
end
process_file_destination(source, destination, file="") click to toggle source
# File lib/s3ranger/sync.rb, line 208
def self.process_file_destination source, destination, file=""
  if not file.empty?
    sub = (remote_prefix? source) ? source.split(":")[1] : source
    file = file.gsub(/^#{sub}/, '')
  end

  # no slash on end of source means we need to append the last src dir to
  # dst prefix testing for empty isn't good enough here.. needs to be
  # "empty apart from potentially having 'bucket:'"
  if source =~ %r{/$}
    File.join [destination, file]
  else
    if remote_prefix? source
      _, name = source.split ":"
      File.join [destination, File.basename(name || ""), file]
    else
      source = /^\/?(.*)/.match(source)[1]

      # Corner case: the root of the remote path is empty, we don't want to
      # add an unnecessary slash here.
      if destination.end_with? ':'
        File.join [destination + source, file]
      else
        File.join [destination, source, file]
      end
    end
  end
end
remote_prefix?(prefix) click to toggle source
# File lib/s3ranger/sync.rb, line 202
def self.remote_prefix?(prefix)
  # allow for dos-like things e.g. C:\ to be treated as local even with
  # colon.
  prefix.include? ':' and not prefix.match '^[A-Za-z]:[\\\\/]'
end

Public Instance Methods

download_files(destination, source, list) click to toggle source
# File lib/s3ranger/sync.rb, line 314
def download_files destination, source, list
  list.each {|e|
    path = File.join destination.path, e.path

    if @args.verbose
      puts " + #{source}#{e.path} => #{path}"
    end

    unless @args.dry_run
      obj = @args.s3.buckets[source.bucket].objects[e.path]

      # Making sure this new file will have a safe shelter
      FileUtils.mkdir_p File.dirname(path)

      # Downloading and saving the files
      File.open(path, 'wb') do |file|
        obj.read do |chunk|
          file.write chunk
        end
      end
    end
  }
end
read_tree_remote(location) click to toggle source
# File lib/s3ranger/sync.rb, line 267
def read_tree_remote location
  dir = location.path
  dir += '/' if not dir.empty? or dir.end_with?('/')
  @args.s3.buckets[location.bucket].objects.with_prefix(dir || "").to_a.collect {|obj|
    Node.new location.path, obj.key, obj.content_length
  }
end
read_trees(source, destination) click to toggle source
# File lib/s3ranger/sync.rb, line 275
def read_trees source, destination
  if source.local?
    source_tree = LocalDirectory.new(source.path).list_files
    destination_tree = read_tree_remote destination
  else
    source_tree = read_tree_remote source
    destination_tree = LocalDirectory.new(destination.path).list_files
  end

  [source_tree, destination_tree]
end
remove_files(remote, list) click to toggle source
# File lib/s3ranger/sync.rb, line 301
def remove_files remote, list

  if @args.verbose
    list.each {|e|
      puts " - #{remote}#{e.path}"
    }
  end

  unless @args.dry_run
    @args.s3.buckets[remote.bucket].objects.delete_if { |obj| list.include? obj.key }
  end
end
remove_local_files(destination, source, list) click to toggle source
# File lib/s3ranger/sync.rb, line 338
def remove_local_files destination, source, list
  list.each {|e|
    path = File.join destination.path, e.path

    if @args.verbose
      puts " * #{e.path} => #{path}"
    end

    unless @args.dry_run
      FileUtils.rm_rf path
    end
  }
end
run() click to toggle source
# File lib/s3ranger/sync.rb, line 152
def run
  # Reading the source and destination using our helper method
  if (source, destination, bucket = self.class.parse_params [@source, @destination]).nil?
    raise WrongUsage.new(nil, 'Need a source and a destination')
  end

  # Getting the trees
  source_tree, destination_tree = read_trees source, destination

  # Getting the list of resources to be exchanged between the two peers
  _, to_add, to_remove = self.class.cmp source_tree, destination_tree

  # Removing the items matching the exclude pattern if requested
  to_add.select! { |e|
    begin
      (e.path =~ /#{@args.exclude}/).nil?
    rescue RegexpError => exc
      raise WrongUsage.new nil, exc.message
    end
  } if @args.exclude

  # Calling the methods that perform the actual IO
  if source.local?
    upload_files destination, to_add
    remove_files destination, to_remove unless @args.keep
  else
    download_files destination, source, to_add
    remove_local_files destination, source, to_remove unless @args.keep
  end
end
upload_files(remote, list) click to toggle source
# File lib/s3ranger/sync.rb, line 287
def upload_files remote, list
  list.each do |e|
    if @args.verbose
      puts " + #{e.full} => #{remote}#{e.path}"
    end

    unless @args.dry_run
      if File.file? e.path
        @args.s3.buckets[remote.bucket].objects[e.path].write Pathname.new(e.path), :acl => @args.acl
      end
    end
  end
end