class Bosh::Blobstore::S3BlobstoreClient

Constants

BLANK_REGION

hack to get the v2 AWS SDK to behave with S3-compatible blobstores

DEFAULT_REGION
ENDPOINT

Attributes

simple[R]

Public Class Methods

new(options) click to toggle source

Blobstore client for S3 @param [Hash] options S3connection options @option options [Symbol] bucket_name

key that is applied before the object is sent to S3

@option options [Symbol, optional] access_key_id @option options [Symbol, optional] secret_access_key @note If access_key_id and secret_access_key are not present, the

blobstore client operates in read only mode as a
simple_blobstore_client
Calls superclass method Bosh::Blobstore::BaseClient::new
# File lib/blobstore_client/s3_blobstore_client.rb, line 27
def initialize(options)
  super(options)

  @aws_options = build_aws_options({
    bucket_name: @options[:bucket_name],
    use_ssl: @options.fetch(:use_ssl, true),
    host: @options[:host],
    port: @options[:port],
    region: @options[:region] || DEFAULT_REGION,
    s3_force_path_style: @options.fetch(:s3_force_path_style, false),
    ssl_verify_peer:  @options.fetch(:ssl_verify_peer, true),
    credentials_source: @options.fetch(:credentials_source, 'static'),
    access_key_id: @options[:access_key_id],
    secret_access_key: @options[:secret_access_key],
    signature_version: @options[:signature_version]
  })

  # using S3 without credentials is a special case:
  # it is really the simple blobstore client with a bucket name
  if read_only?
    unless @options[:bucket_name] || @options[:bucket]
      raise BlobstoreError, 'bucket name required'
    end

    @options[:bucket] ||= @options[:bucket_name]
    @options[:endpoint] ||= S3BlobstoreClient::ENDPOINT
    @simple = SimpleBlobstoreClient.new(@options)
  end

rescue Aws::S3::Errors::ServiceError => e
  raise BlobstoreError, "Failed to initialize S3 blobstore: #{e.code} : #{e.message}"
end

Public Instance Methods

create_file(object_id, file) click to toggle source

@param [File] file file to store in S3

# File lib/blobstore_client/s3_blobstore_client.rb, line 61
def create_file(object_id, file)
  raise BlobstoreError, 'unsupported action' if @simple

  object_id ||= generate_object_id

  # in Ruby 1.8 File doesn't respond to :path
  path = file.respond_to?(:path) ? file.path : file
  store_in_s3(path, full_oid_path(object_id))

  object_id
rescue Aws::S3::Errors::ServiceError => e
  raise BlobstoreError, "Failed to create object, S3 response error code #{e.code}: #{e.message}"
end
delete_object(object_id) click to toggle source

@param [String] object_id object id to delete

# File lib/blobstore_client/s3_blobstore_client.rb, line 93
def delete_object(object_id)
  raise BlobstoreError, 'unsupported action' if @simple
  object_id = full_oid_path(object_id)

  s3_object = Aws::S3::Object.new({:key => object_id}.merge(@aws_options))
  # TODO: don't blow up if we are cannot find an object we are trying to
  # delete anyway
  raise NotFound, "Object '#{object_id}' is not found" unless s3_object.exists?

  s3_object.delete
rescue Aws::S3::Errors::ServiceError => e
  raise BlobstoreError, "Failed to delete object '#{object_id}', S3 response error code #{e.code}: #{e.message}"
end
get_file(object_id, file) click to toggle source

@param [String] object_id object id to retrieve @param [File] file file to store the retrived object in

# File lib/blobstore_client/s3_blobstore_client.rb, line 77
def get_file(object_id, file)
  object_id = full_oid_path(object_id)
  return @simple.get_file(object_id, file) if @simple

  s3_object = Aws::S3::Object.new({:key => object_id}.merge(@aws_options))
  s3_object.get do |chunk|
    file.write(chunk)
  end

rescue Aws::S3::Errors::NoSuchKey => e
  raise NotFound, "S3 object '#{object_id}' not found"
rescue Aws::S3::Errors::ServiceError => e
  raise BlobstoreError, "Failed to find object '#{object_id}', S3 response error code #{e.code}: #{e.message}"
end
object_exists?(object_id) click to toggle source
# File lib/blobstore_client/s3_blobstore_client.rb, line 107
def object_exists?(object_id)
  object_id = full_oid_path(object_id)
  return simple.exists?(object_id) if simple

  # Hack to get the Aws SDK to redirect to the correct region on
  # subsequent requests
  unless @region_configured
    s3 = Aws::S3::Client.new(@aws_options.reject{|k| k == :bucket_name})
    s3.list_objects({bucket: @aws_options[:bucket_name]})
    @region_configured = true
  end

  Aws::S3::Object.new({:key => object_id}.merge(@aws_options)).exists?
end

Protected Instance Methods

aws_credentials(credentials_source, access_key_id, secret_access_key) click to toggle source
# File lib/blobstore_client/s3_blobstore_client.rb, line 159
def aws_credentials(credentials_source, access_key_id, secret_access_key)
  creds = {}
  # credentials_source could be static (default) or env_or_profile
  # static credentials must be included in aws_properties
  # env_or_profile credentials will use the Aws DefaultCredentialsProvider
  # to find Aws credentials in environment variables or EC2 instance profiles
  case credentials_source
    when 'static'
      creds[:access_key_id]     = access_key_id
      creds[:secret_access_key] = secret_access_key

    when 'env_or_profile'
      if !access_key_id.nil? || !secret_access_key.nil?
        raise BlobstoreError, "can't use access_key_id or secret_access_key with env_or_profile credentials_source"
      end
    else
      raise BlobstoreError, 'invalid credentials_source'
  end
  return creds
end
build_aws_options(options) click to toggle source
# File lib/blobstore_client/s3_blobstore_client.rb, line 180
def build_aws_options(options)
  aws_options = {
    bucket_name: options[:bucket_name],
    region: options[:region],
    force_path_style: options[:s3_force_path_style],
    ssl_verify_peer: options[:ssl_verify_peer],
  }

  unless options[:host].nil?
    host = options[:host]
    protocol = options[:use_ssl] ? 'https' : 'http'
    uri = options[:port].nil? ? host : "#{host}:#{options[:port]}"
    aws_options[:endpoint] = "#{protocol}://#{uri}"
    aws_options[:region] = BLANK_REGION
  end

  aws_options[:signature_version] = 's3' unless use_v4_signing?(options)

  creds = aws_credentials(options[:credentials_source], options[:access_key_id], options[:secret_access_key])
  aws_options.merge!(creds)

  aws_options
end
full_oid_path(object_id) click to toggle source
# File lib/blobstore_client/s3_blobstore_client.rb, line 143
def full_oid_path(object_id)
   @options[:folder] ?  @options[:folder] + '/' + object_id : object_id
end
read_only?() click to toggle source
# File lib/blobstore_client/s3_blobstore_client.rb, line 136
def read_only?
  (@options[:credentials_source] == 'static' ||
  @options[:credentials_source].nil?) &&
  @options[:access_key_id].nil? &&
  @options[:secret_access_key].nil?
end
store_in_s3(path, oid) click to toggle source

@param [String] path path to file which will be stored in S3 @param [String] oid object id @return [void]

# File lib/blobstore_client/s3_blobstore_client.rb, line 127
def store_in_s3(path, oid)
  raise BlobstoreError, "object id #{oid} is already in use" if object_exists?(oid)

  s3_object = Aws::S3::Object.new({:key => oid}.merge(@aws_options))
  multipart_threshold = @options.fetch(:s3_multipart_threshold, 16_777_216)
  s3_object.upload_file(path, {content_type: "application/octet-stream", multipart_threshold: multipart_threshold})
  nil
end
use_v4_signing?(options) click to toggle source
# File lib/blobstore_client/s3_blobstore_client.rb, line 147
def use_v4_signing?(options)
  case options[:signature_version]
    when '4'
      true
    when '2'
      false
    else
      region = options[:region]
      (region == 'eu-central-1' || region == 'cn-north-1')
  end
end