class RestFtpDaemon::Remote::RemoteS3

Constants

S3_MAX_COUNT
S3_MIN_PART

Defaults

Attributes

client[R]

Class options

target[R]

Public Class Methods

handles() click to toggle source

URI schemes handled by this plugin

# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 23
def self.handles
  [URI::S3]
end
new(target, job, config) click to toggle source
Calls superclass method
# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 27
def initialize target, job, config
  super
end

Public Instance Methods

connect() click to toggle source
Calls superclass method
# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 31
def connect
  super

  # Connect remote server
  @client = Aws::S3::Client.new(
    region: @target.aws_region,
    credentials: Aws::Credentials.new(@target.aws_id, @target.aws_secret),
    http_wire_trace: debug_enabled
    )

rescue Exception => exception
  raise RemoteConnectError, "#{exception.class}: #{exception.message}"
end
connected?() click to toggle source
# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 84
def connected?
  !@client.nil?
end
move(source, target) click to toggle source
# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 68
def move source, target
  # Identify the source object
  # obj = @client.bucket(source.aws_bucket).object(source.path_rel)
  # raise RestFtpDaemon::AssertionFailed, "move: object not found" unless obj

  # Move the file
  # log_debug "move: copy bucket[#{source.aws_bucket}] source[#{source.path_rel}] target[#{target.path_rel}]"
  @client.copy_object(bucket: source.aws_bucket, key: target.path_rel, copy_source: "#{source.aws_bucket}/#{source.path_rel}")
  # log_debug "move: delete bucket[#{source.aws_bucket}] source[#{source.path_rel}]"
  @client.delete_object(bucket: source.aws_bucket, key: source.path_rel)
  # log_debug "move: done"

  # Move the file
  # obj.move_to(target.path_rel, :bucket_name => target.aws_bucket)
end
push(source, target, &callback) click to toggle source
# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 54
def push source, target, &callback
  # Push init
  raise RestFtpDaemon::AssertionFailed, "push/client" if @client.nil?

  # Do the transfer, passing the file to the best method
  File.open(source.path_abs, 'r', encoding: 'BINARY') do |file|
    if file.size >= S3_MIN_PART
      upload_multipart  file, target.aws_bucket, target.path_rel, target.name, &callback
    else
      upload_onefile    file, target.aws_bucket, target.path_rel, target.name, &callback
    end
  end
end
size_if_exists(target) click to toggle source
# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 45
def size_if_exists target
  log_debug "size_if_exists rel[#{target.path_rel}]"
  object = @client.get_object(bucket: target.aws_bucket, key: target.path_rel)
rescue Aws::S3::Errors::NotFound => e
  return false
else
  return object.content_length
end

Private Instance Methods

compute_parts_size(filesize) click to toggle source
# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 159
def compute_parts_size filesize
  # Initial part size is minimal
  partsize_mini = S3_MIN_PART

  # Other partsize if too many blocks
  partsize_bigf = (filesize.to_f / S3_MAX_COUNT).ceil

  # Decide
  return [partsize_mini, partsize_bigf].max
end
debug_enabled() click to toggle source
# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 170
def debug_enabled
  @config[:debug_s3]
end
upload_multipart(file, s3_bucket, s3_path, s3_name) { |part_size, s3_name| ... } click to toggle source
# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 98
def upload_multipart file, s3_bucket, s3_path, s3_name, &callback
  # Init
  current_part  = 1

  # Compute parameters
  file_size     = file.size
  parts_size    = compute_parts_size(file_size)
  parts_count   = (file_size.to_f / parts_size).ceil

  # Prepare basic opts
  options = {
    bucket: s3_bucket,
    key:    s3_path,
    }

  # Declare multipart upload
  mpu_create_response = @client.create_multipart_upload(options)
  options[:upload_id] = mpu_create_response.upload_id
  log_debug "push: create_multipart_upload", {
    s3_bucket:    s3_bucket,
    s3_path:      s3_path,
    upload_id:    options[:upload_id],
    file_size:    format_bytes(file_size, "B"),
    parts_size:   format_bytes(parts_size, "B"),
    parts_count:  parts_count
    }

  # Upload each part
  file.each_part(parts_size) do |part|
    # Prepare part upload
    opts = options.merge({
      body:        part,
      part_number: current_part,
      })
    part_size = part.bytesize
    log_debug "upload_part [#{current_part}/#{parts_count}] part_size[#{part_size}]"

    # Push this over there
    resp = @client.upload_part(opts)
    
    # Send progress info upwards
    yield part_size, s3_name

    # Increment part number
    current_part += 1
  end

  # Retrieve parts and complete upload
  parts_resp = @client.list_parts(options)
  those_parts = parts_resp.parts.map do |part| 
    { part_number: part.part_number, etag: part.etag }
  end
  opts = options.merge({
    multipart_upload: {
      parts: those_parts
      } 
    })
  log_debug "complete_multipart_upload"
  mpu_complete_response = @client.complete_multipart_upload(opts)
end
upload_onefile(file, s3_bucket, s3_path, s3_name, &callback) click to toggle source
# File lib/plugins/rest-ftp-daemon/remote/remote_s3.rb, line 90
def upload_onefile file, s3_bucket, s3_path, s3_name, &callback
  log_debug "push: put_object", {
    s3_bucket:    s3_bucket,
    s3_path:      s3_path,
    }
  @client.put_object(bucket: s3_bucket, key: s3_path, body: file)
end