class S3sync::Syncer

Public Class Methods

new() click to toggle source
# File lib/s3sync/syncer.rb, line 7
def initialize
  @s3 = AWS::S3.new
  @log = S3sync::Logger.new
end

Public Instance Methods

download(s3_location, local_path) click to toggle source
# File lib/s3sync/syncer.rb, line 37
def download(s3_location, local_path)
  bucket_name, *folders = s3url_to_bucket_folder s3_location
  destination_folder = File.absolute_path(local_path)

  @log.info "Downloading"
  
  local_files = local_files(local_path)
  remote_files(bucket_name, folders) do |s3|
    next if FileDiff::same_file? s3, local_files[s3[:key]]
    begin
      destination_file = File.join destination_folder, s3[:key]
      @log.info "#{s3[:file].public_url} => #{destination_file}"
      s3_download s3, destination_file
    rescue Exception => e
      @log.error e
    end
  end
  @log.info "Done"
rescue Exception => e
  @log.error e
end
upload(local_path, s3_url) click to toggle source
# File lib/s3sync/syncer.rb, line 12
def upload(local_path, s3_url)
  bucket_name, *folders = s3url_to_bucket_folder s3_url

  @log.info "Uploading files" 

  local_files = local_files(local_path)
  remote_files(bucket_name, folders) do |s3|
    source = local_files[s3[:key]]
    local_files.delete s3[:key] if FileDiff::same_file? source, s3
  end

  local_files.each do |key,item|
    begin
      s3_key = File.join folders, key
      @log.info "#{item[:file]} => s3://#{bucket_name}/#{s3_key}"
      s3_upload item, bucket_name, s3_key
    rescue Exception => e
      @log.error e
    end
  end
  @log.info "Done"
rescue Exception => e
  @log.error e
end

Private Instance Methods

ensure_folder_exists(folder) click to toggle source
# File lib/s3sync/syncer.rb, line 65
def ensure_folder_exists(folder)
  FileUtils.mkdir_p(folder) unless File.directory?(folder)
end
last_modified(item) click to toggle source
# File lib/s3sync/syncer.rb, line 73
def last_modified(item)
  item[:last_modified]
end
local_files(path) click to toggle source
# File lib/s3sync/syncer.rb, line 100
def local_files(path)
  return {} unless File.directory? path

  results = {}
  Find.find(path) do |item|
    next if File.directory? item
    
    file_path  = item.match(/#{path}\/?(.*)/)[1]
    results[file_path] = { key:file_path, 
                           last_modified: File.mtime(item), 
                           content_length: File.size(item),
                           file: File.absolute_path(item) }
  end

  results
end
remote_files(bucket, folders) { |remote_item(object, relative_path)| ... } click to toggle source

Yielding the remote s3 files and doing a 2 pass filter as better performance than computing a diff of 2 complete directory listings

# File lib/s3sync/syncer.rb, line 128
def remote_files(bucket, folders)
  objects = @s3.buckets[bucket].objects.with_prefix(File.join(folders))
  objects.each do |object|
    relative_path = object.key.split(/\//).drop folders.length
    next if object.content_length.nil? or object.content_length == 0 or relative_path.empty?
    yield remote_item(object, relative_path)
  end
end
remote_item(object, relative_path) click to toggle source
# File lib/s3sync/syncer.rb, line 117
def remote_item(object, relative_path)
  last_modified = Time.parse object.metadata['last_modified'] rescue 0

  { key:File.join(relative_path), 
    last_modified:last_modified, 
    content_length:object.content_length, 
    file:object }
end
s3_download(item, destination_file) click to toggle source
# File lib/s3sync/syncer.rb, line 77
def s3_download(item, destination_file)
  ensure_folder_exists File.dirname(destination_file)

  File.open(destination_file, 'wb') do |f|
    item[:file].read {|chunk| f.write chunk }
  end

  update_last_modified destination_file, last_modified(item)
end
s3_upload(item, bucket, key) click to toggle source
# File lib/s3sync/syncer.rb, line 87
def s3_upload(item, bucket, key)
  object = @s3.buckets[bucket].objects[key]
  object.write(:file => item[:file])
  object.metadata['last_modified'] = last_modified(item)
end
s3url_to_bucket_folder(s3_location) click to toggle source
# File lib/s3sync/syncer.rb, line 93
def s3url_to_bucket_folder(s3_location)
  s3_path = s3_location.match(/s3:\/\/(.*)/)[1] rescue nil
  raise 'Invalid S3 url must be s3://bucket form' unless s3_path

  s3_path.split /\//
end
safe_encode(text) click to toggle source
# File lib/s3sync/syncer.rb, line 61
def safe_encode(text)
  text.force_encoding('ASCII-8BIT').encode('UTF-8', :invalid => :replace, :undef => :replace, :replace => '?')
end
update_last_modified(file, time) click to toggle source
# File lib/s3sync/syncer.rb, line 69
def update_last_modified(file, time)
  FileUtils.touch file, mtime:time
end