class MU::Cloud::AWS::AmazonEndpoint

Wrapper class for the EC2 API, so that we can catch some common transient endpoint errors without having to spray rescues all over the codebase.

Attributes

account[R]
credentials[R]

Public Class Methods

new(region: nil, api: "EC2", credentials: nil) click to toggle source

Create an AWS API client @param region [String]: Amazon region so we know what endpoint to use @param api [String]: Which API are we wrapping?

# File modules/mu/providers/aws.rb, line 1612
def initialize(region: nil, api: "EC2", credentials: nil)
  @cred_obj = MU::Cloud::AWS.loadCredentials(credentials)
  @credentials = MU::Cloud::AWS.credConfig(credentials, name_only: true)
  @api_name = api

  if !@cred_obj
    raise MuError, "Unable to locate valid AWS credentials for #{api} API. #{credentials ? "Credentials requested were '#{credentials}'": ""}"
  end

  params = {}
  region ||= MU::Cloud::AWS.credConfig(credentials)['region']
  region ||= MU.myRegion

  if region
    @region = region
    params[:region] = @region
  end

  params[:credentials] = @cred_obj

  MU.log "Initializing #{api} object with credentials #{credentials}", MU::DEBUG, details: params
  require "aws-sdk-#{api.downcase}"

  @api = Object.const_get("Aws::#{api}::Client").new(params)
end

Public Instance Methods

method_missing(method_sym, *arguments) click to toggle source

Catch-all for AWS client methods. Essentially a pass-through with some rescues for known silly endpoint behavior.

# File modules/mu/providers/aws.rb, line 1641
        def method_missing(method_sym, *arguments)
          # make sure error symbols are loaded for our exception handling later
          require "aws-sdk-lambda"
          require "aws-sdk-rds"
          require "aws-sdk-ec2"
          require "aws-sdk-route53"
          require "aws-sdk-iam"
          require "aws-sdk-efs"
          require "aws-sdk-pricing"
          require "aws-sdk-apigateway"
          require "aws-sdk-ecs"
          require "aws-sdk-eks"
          require "aws-sdk-cloudwatchlogs"
          require "aws-sdk-cloudwatchevents"
          require "aws-sdk-elasticloadbalancing"
          require "aws-sdk-elasticloadbalancingv2"
          require "aws-sdk-autoscaling"

          known_concats = {
            "Pricing" => {
              :get_products => :price_list
            }
          }

          retries = 0
          begin
            MU.log "Calling #{@api_name}.#{method_sym} in #{@region}", MU::DEBUG, details: arguments

              retval = if !arguments.nil? and arguments.size == 1
                @api.method(method_sym).call(arguments[0])
              elsif !arguments.nil? and arguments.size > 0
                @api.method(method_sym).call(*arguments)
              else
                @api.method(method_sym).call
              end

            if !retval.nil?
              begin
              page_markers = {
                :marker => :marker,
                :next_token => :next_token,
                :next_marker => :marker
              }
              paginator = nil
              new_page = nil
              page_markers.each_key { |m|
                if !retval.nil? and retval.respond_to?(m)
                  paginator = m
                  new_page = retval.send(m)
                  break
                end
              }

              if paginator and new_page and !new_page.empty?
                resp = retval.respond_to?(:__getobj__) ? retval.__getobj__ : retval
                concat_to = MU.structToHash(resp).keys.reject { |m|
                  m.to_s.match(/=$/) or m == paginator or resp.send(m).nil? or !resp.send(m).is_a?(Array)
                }

                if concat_to.empty? and known_concats[@api_name] and
                   known_concats[@api_name][method_sym]
                  concat_to << known_concats[@api_name][method_sym]
                end

                if concat_to.empty? and method_sym.to_s.match(/^(?:describe|list)_(.*)/)
                  my_attr = Regexp.last_match[1].to_sym
                  concat_to << my_attr if resp.respond_to?(my_attr)
                end

                if concat_to.size != 1
                  raise MuError.new "Tried to figure out where I might append paginated results for a #{@api_name}.#{method_sym}, but failed", details: MU.structToHash(resp).keys
                else
                  concat_to = concat_to.first
                  new_args = arguments ? arguments.dup : [{}]
                  begin
                    if new_args.is_a?(Array)
                      new_args << {} if new_args.empty?
                      if new_args.size == 1 and new_args.first.is_a?(Hash)
                        new_args[0][page_markers[paginator]] = new_page
                      else
                        MU.log "I don't know how to insert a #{paginator} into these arguments for #{method_sym}", MU::WARN, details: new_args
                      end
                    elsif new_args.is_a?(Hash)
                      new_args[page_markers[paginator]] = new_page
                    end

                    MU.log "Attempting magic pagination for #{method_sym}", MU::DEBUG, details: new_args

#                    resp = if !arguments.nil? and arguments.size == 1
#                      @api.method(method_sym).call(new_args[0])
#                    elsif !arguments.nil? and arguments.size > 0
                    resp = @api.method(method_sym).call(*new_args)
#                    end
                    break if resp.nil?
                    resp = resp.__getobj__ if resp.respond_to?(:__getobj__)
                    retval.send(concat_to).concat(resp.send(concat_to))
                    new_page = resp.send(paginator) if !resp.nil?
                  end while !resp.nil? and !new_page.nil? and !new_page.empty?
                end
              end
              rescue StandardError => e
                MU.log "Made a good-faith effort to auto-paginate API call to #{method_sym} and failed with #{e.message}", MU::DEBUG, details: arguments
                raise e
              end
            end

            return retval
          rescue Aws::Lambda::Errors::TooManyRequestsException, Aws::RDS::Errors::Throttling, Aws::EC2::Errors::InternalError, Aws::EC2::Errors::RequestLimitExceeded, Aws::EC2::Errors::Unavailable, Aws::Route53::Errors::Throttling, Aws::ElasticLoadBalancing::Errors::HttpFailureException, Aws::EC2::Errors::Http503Error, Aws::AutoScaling::Errors::Http503Error, Aws::AutoScaling::Errors::InternalFailure, Aws::AutoScaling::Errors::ServiceUnavailable, Aws::Route53::Errors::ServiceUnavailable, Aws::ElasticLoadBalancing::Errors::Throttling, Aws::RDS::Errors::ClientUnavailable, Aws::Waiters::Errors::UnexpectedError, Aws::ElasticLoadBalancing::Errors::ServiceUnavailable, Aws::ElasticLoadBalancingV2::Errors::Throttling, Seahorse::Client::NetworkingError, Aws::IAM::Errors::Throttling, Aws::EFS::Errors::ThrottlingException, Aws::Pricing::Errors::ThrottlingException, Aws::APIGateway::Errors::TooManyRequestsException, Aws::ECS::Errors::ThrottlingException, Net::ReadTimeout, Faraday::TimeoutError, Aws::CloudWatchLogs::Errors::ThrottlingException => e
            if e.class.name == "Seahorse::Client::NetworkingError" and e.message.match(/Name or service not known/)
              MU.log e.inspect, MU::ERR
              raise e
            end
            retries = retries + 1
            debuglevel = MU::DEBUG
            interval = 5 + Random.rand(4) - 2
            if retries < 10 and retries > 2
              debuglevel = MU::NOTICE
              interval = 20 + Random.rand(10) - 3
            # elsif retries >= 10 and retries <= 100
            elsif retries >= 10
              debuglevel = MU::WARN
              interval = 40 + Random.rand(15) - 5
            # elsif retries > 100
              # raise MuError, "Exhausted retries after #{retries} attempts while calling EC2's #{method_sym} in #{@region}.  Args were: #{arguments}"
            end
            MU.log "Got #{e.inspect} calling EC2's #{method_sym} in #{@region} with credentials #{@credentials}, waiting #{interval.to_s}s and retrying. Args were: #{arguments}", debuglevel, details: caller
            sleep interval
            retry
          rescue StandardError => e
            MU.log "Got #{e.inspect} calling EC2's #{method_sym} in #{@region} with credentials #{@credentials}", MU::DEBUG, details: arguments
            raise e
          end
        end