class Pupistry::StorageAWS

Pupistry::StorageAWS

Attributes

bucket[RW]
s3[RW]

Public Class Methods

new(mode) click to toggle source
# File lib/pupistry/storage_aws.rb, line 13
def initialize(mode)
  # mode is either "build" or "agent", depending which we load a different
  # set of permissions. Awareness of both is intentional, since we want the
  # build machines to known the agent creds so we can generate bootstrap
  # template files.

  unless defined? $config['general']['s3_bucket']
    $logger.fatal 'You must set the AWS s3_bucket'
    exit 0
  end

  # Define AWS configuration
  if defined? $config[mode]['access_key_id']
    if $config[mode]['access_key_id'] == ''
      $logger.debug 'No AWS IAM credentials specified, defaulting to environmental discovery'
      $logger.debug 'If you get weird permissions errors, try setting the credentials explicity in config first.'
    else
      $logger.debug 'Loading AWS credentials from configuration file'

      AWS.config(
        access_key_id:     $config[mode]['access_key_id'],
        secret_access_key: $config[mode]['secret_access_key'],
        region:            $config[mode]['region'],
        proxy_uri:         $config[mode]['proxy_uri']
      )
    end
  else
    $logger.debug 'No AWS IAM credentials specified, defaulting to environmental discovery'
    $logger.debug 'If you get weird permissions errors, try setting the credentials explicity in config first.'
  end

  # Setup S3 bucket
  if defined? $config['general']['s3_endpoint'] and $config['general']['s3_endpoint'] != nil
    $logger.debug 'Connecting to alternative endpoint ' + $config['general']['s3_endpoint']
    @s3 = AWS::S3.new(
            s3_endpoint: $config['general']['s3_endpoint'],
            s3_force_path_style: true,
          )
  else
    @s3 = AWS::S3.new
  end
  @bucket = @s3.buckets[$config[mode]['s3_bucket']]
end

Public Instance Methods

download(src, dest = 'stream') click to toggle source
# File lib/pupistry/storage_aws.rb, line 96
def download(src, dest = 'stream')
  $logger.debug "Downloading file s3://#{$config['general']['s3_bucket']}/#{$config['general']['s3_prefix']}#{src} to #{dest}"

  begin
    # Generate the object name/key based on the relative file name and path.
    s3_obj_name = "#{$config['general']['s3_prefix']}#{src}"
    s3_obj      = @s3.buckets[$config['general']['s3_bucket']].objects[s3_obj_name]

    # Download the file
    if dest == 'stream'
      # Return the contents rather than writing to disk. We assume stream mode
      # if the dest filename was unspecified
      return s3_obj.read
    else
      # Download to an ondisk file
      File.open(dest, 'wb') do |file|
        s3_obj.read do |chunk|
          file.write(chunk)
        end
      end
    end

  rescue AWS::S3::Errors::NoSuchKey
    $logger.debug 'No such file exists for download, this is normal at times.'
    return false

  rescue AWS::S3::Errors::NoSuchBucket
    $logger.fatal "S3 bucket #{$config['general']['s3_bucket']} does not exist"
    exit 0

  rescue AWS::S3::Errors::AccessDenied
    $logger.fatal "Access to S3 bucket #{$config['general']['s3_bucket']} denied"
    exit 0

  rescue AWS::S3::Errors::InvalidObjectState
    $logger.warn "Unable to download \"#{src}\", it has been archived off into Glacier and would need to be recovered first."

    # Do we need to restore it?
    begin
      if s3_obj.restore_in_progress?
        $logger.warn "A restore of this file is currently in progress, but can take up to 4 hours - please re-try later."
      else
        # Not being restored currently, let's file a request. This allows
        # us to cater for situations where a bunch of servers need to get
        # an old manifest/file, however the fastest solution is to simply
        # do a new `pupistry push` from a workstation to upload a new
        # manifest file.
        if s3_obj.restore(:days => 30)
          $logger.warn "Recover request has been issued, this could take up to 4 hours to complete."
          $logger.warn "Note that doing a `pupistry push` from the workstation would solve this faster."
        end
      end
    rescue StandardError => e
      $logger.error "Glacier restore request for #{src} failed. (#{e.class}), best option is to push the latest manifest from a workstation with `pupistry push`."
    end

    return false

  rescue AWS::S3::Errors::PermanentRedirect => e
    $logger.error "The wrong endpoint has been specified (or autodetected) for #{$config['general']['s3_bucket']}."
    raise e

  rescue AWS::S3::Errors::SignatureDoesNotMatch => e
    $logger.error "IAM signature error when accessing #{$config['general']['s3_bucket']}, probably invalid IAM credentials"
    raise e

  rescue AWS::S3::Errors::MissingCredentialsError
    $logger.error 'AWS credentials not supplied. You must either:'
    $logger.error 'a) Specify them in the config file for Pupistry'
    $logger.error 'b) Use IAM roles with an EC2 instance.'
    $logger.error 'c) Set them in ENV as AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY'
    return false

  rescue StandardError => e
    raise e
  end
end
upload(src, dest) click to toggle source
# File lib/pupistry/storage_aws.rb, line 57
def upload(src, dest)
  $logger.debug "Pushing file #{src} to s3://#{$config['general']['s3_bucket']}/#{$config['general']['s3_prefix']}#{dest}"

  begin
    # Generate the object name/key based on the relative file name and path.
    s3_obj_name = "#{$config['general']['s3_prefix']}#{dest}"
    s3_obj      = @s3.buckets[$config['general']['s3_bucket']].objects[s3_obj_name]

    # Perform S3 upload
    s3_obj.write(file: src)

  rescue AWS::S3::Errors::NoSuchBucket
    $logger.fatal "S3 bucket #{$config['general']['s3_bucket']} does not exist"
    exit 0

  rescue AWS::S3::Errors::AccessDenied
    $logger.fatal "Access to S3 bucket #{$config['general']['s3_bucket']} denied"
    exit 0

  rescue AWS::S3::Errors::PermanentRedirect => e
    $logger.error "The wrong endpoint has been specified (or autodetected) for #{$config['general']['s3_bucket']}."
    raise e

  rescue AWS::S3::Errors::SignatureDoesNotMatch => e
    $logger.error "IAM signature error when accessing #{$config['general']['s3_bucket']}, probably invalid IAM credentials"
    raise e

  rescue AWS::S3::Errors::MissingCredentialsError
    $logger.error 'AWS credentials not supplied. You must either:'
    $logger.error 'a) Specify them in the config file for Pupistry'
    $logger.error 'b) Use IAM roles with an EC2 instance.'
    $logger.error 'c) Set them in ENV as AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY'
    return false

  rescue StandardError => e
    raise e
  end
end