class MU::Cloud::Google::GoogleEndpoint

Wrapper class for Google APIs, so that we can catch some common transient endpoint errors without having to spray rescues all over the codebase.

Attributes

issuer[R]

Public Class Methods

new(api: "ComputeV1::ComputeService", scopes: ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/compute.readonly'], masquerade: nil, credentials: nil, auth_error_quiet: false) click to toggle source

Create a Google Cloud Platform API client @param api [String]: Which API are we wrapping? @param scopes [Array<String>]: Google auth scopes applicable to this API

# File modules/mu/providers/google.rb, line 1113
        def initialize(api: "ComputeV1::ComputeService", scopes: ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/compute.readonly'], masquerade: nil, credentials: nil, auth_error_quiet: false)
          @credentials = credentials
          @scopes = scopes.map { |s|
            if !s.match(/\//) # allow callers to use shorthand
              s = "https://www.googleapis.com/auth/"+s
            end
            s
          }
          @masquerade = masquerade
          @api = Object.const_get("Google::Apis::#{api}").new
          @api.authorization = MU::Cloud::Google.loadCredentials(@scopes, credentials: credentials)
          raise MuError, "No useable Google credentials found#{credentials ? " with set '#{credentials}'" : ""}" if @api.authorization.nil?
          if @masquerade
            begin
              @api.authorization.sub = @masquerade
              @api.authorization.fetch_access_token!
            rescue Signet::AuthorizationError => e
              if auth_error_quiet
                MU.log "Cannot masquerade as #{@masquerade} to API #{api}: #{e.message}", MU::DEBUG, details: @scopes
              else
                MU.log "Cannot masquerade as #{@masquerade} to API #{api}: #{e.message}", MU::ERROR, details: @scopes
                if e.message.match(/client not authorized for any of the scopes requested/)
# XXX it'd be helpful to list *all* scopes we like, as well as the API client's numeric id
                  MU.log "To grant access to API scopes for this service account, see:", MU::ERR, details: "https://admin.google.com/AdminHome?chromeless=1#OGX:ManageOauthClients"
                end
              end

              raise e
            end
          end
          @issuer = @api.authorization.issuer
        end

Public Instance Methods

delete(type, project, region = nil, noop = false, filter = "description eq click to toggle source

Generic wrapper for deleting Compute resources, which are consistent enough that we can get away with this. @param type [String]: The type of resource, typically the string you'll find in all of the API calls referring to it @param project [String]: The project in which we should look for the resources @param region [String]: The region in which to loop for the resources @param noop [Boolean]: If true, will only log messages about resources to be deleted, without actually deleting them @param filter [String]: The Compute API filter string to use to isolate appropriate resources

# File modules/mu/providers/google.rb, line 1153
        def delete(type, project, region = nil, noop = false, filter = "description eq #{MU.deploy_id}", credentials: nil)
          list_sym = "list_#{type.sub(/y$/, "ie")}s".to_sym
          credentials ||= @credentials
          resp = nil
          begin
            if region
              resp = MU::Cloud::Google.compute(credentials: credentials).send(list_sym, project, region, filter: filter, mu_gcp_enable_apis: false)
            else
              resp = MU::Cloud::Google.compute(credentials: credentials).send(list_sym, project, filter: filter, mu_gcp_enable_apis: false)
            end

          rescue ::Google::Apis::ClientError => e
            return if e.message.match(/^notFound: /)
          end

          if !resp.nil? and !resp.items.nil?
            threads = []
            parent_thread_id = Thread.current.object_id
            resp.items.each { |obj|
              threads << Thread.new {
                MU.dupGlobals(parent_thread_id)
                Thread.abort_on_exception = false
                MU.log "Removing #{type.gsub(/_/, " ")} #{obj.name}"
                delete_sym = "delete_#{type}".to_sym
                if !noop
                  retries = 0
                  failed = false
                  begin
                    resp = nil
                    failed = false
                    if region
                      resp = MU::Cloud::Google.compute(credentials: credentials).send(delete_sym, project, region, obj.name)
                    else
                      resp = MU::Cloud::Google.compute(credentials: credentials).send(delete_sym, project, obj.name)
                    end

                    if resp.error and resp.error.errors and resp.error.errors.size > 0
                      failed = true
                      retries += 1
                      if resp.error.errors.first.code == "RESOURCE_IN_USE_BY_ANOTHER_RESOURCE" and retries < 6
                        sleep 10
                      else
                        MU.log "Error deleting #{type.gsub(/_/, " ")} #{obj.name}", MU::ERR, details: resp.error.errors
                        Thread.abort_on_exception = false
                        raise MuError, "Failed to delete #{type.gsub(/_/, " ")} #{obj.name}"
                      end
                    else
                      failed = false
                    end
# TODO validate that the resource actually went away, because it seems not to do so very reliably
                  rescue ::Google::Apis::ClientError => e
                    raise e if !e.message.match(/(^notFound: |operation in progress)/)
                  rescue MU::Cloud::MuDefunctHabitat => e
                    # this is ok- it's already deleted
                  end while failed and retries < 6
                end
              }
            }
            threads.each do |t|
              t.join
            end

          end
        end
is_done?(retval) click to toggle source

Check whether the various types of Operation responses say they're done, without knowing which specific API they're from

# File modules/mu/providers/google.rb, line 1376
def is_done?(retval)
  (retval.respond_to?(:status) and retval.status == "DONE") or (retval.respond_to?(:done) and retval.done)
end
method_missing(method_sym, *arguments) click to toggle source

Catch-all for AWS client methods. Essentially a pass-through with some rescues for known silly endpoint behavior.

# File modules/mu/providers/google.rb, line 1221
        def method_missing(method_sym, *arguments)
          retries = 0
          actual_resource = nil

          enable_on_fail = true
          arguments.each { |arg|
            if arg.is_a?(Hash) and arg.has_key?(:mu_gcp_enable_apis)
              enable_on_fail = arg[:mu_gcp_enable_apis]
              arg.delete(:mu_gcp_enable_apis)
              
            end
          }
          arguments.delete({})
          next_page_token = nil
          overall_retval = nil

          begin
            MU.log "Calling #{method_sym}", MU::DEBUG, details: arguments
            retval = nil
            retries = 0
            wait_backoff = 5
            if next_page_token 
              if method_sym != :list_entry_log_entries
                if arguments.size == 1 and arguments.first.is_a?(Hash)
                  arguments[0][:page_token] = next_page_token
                else
                  arguments << { :page_token => next_page_token }
                end
              elsif arguments.first.class == ::Google::Apis::LoggingV2::ListLogEntriesRequest
                arguments[0] = ::Google::Apis::LoggingV2::ListLogEntriesRequest.new(
                  resource_names: arguments.first.resource_names,
                  filter: arguments.first.filter,
                  page_token: next_page_token
                )
              end
            end
            begin
              if !arguments.nil? and arguments.size == 1
                retval = @api.method(method_sym).call(arguments[0])
              elsif !arguments.nil? and arguments.size > 0
                retval = @api.method(method_sym).call(*arguments)
              else
                retval = @api.method(method_sym).call
              end
            rescue ArgumentError => e
              MU.log "#{e.class.name} calling #{@api.class.name}.#{method_sym.to_s}: #{e.message}", MU::ERR, details: arguments
              raise e
            rescue ::Google::Apis::AuthorizationError => e
              if arguments.size > 0
                raise MU::MuError, "Service account #{MU::Cloud::Google.svc_account_name} has insufficient privileges to call #{method_sym} in project #{arguments.first}"
              else
                raise MU::MuError, "Service account #{MU::Cloud::Google.svc_account_name} has insufficient privileges to call #{method_sym}"
              end
            rescue ::Google::Apis::RateLimitError, ::Google::Apis::TransmissionError, ::ThreadError, ::Google::Apis::ServerError => e
              if retries <= 10
                sleep wait_backoff
                retries += 1
                wait_backoff = wait_backoff * 2
                retry
              else
                raise e
              end
            rescue ::Google::Apis::ClientError, OpenSSL::SSL::SSLError => e
              if e.message.match(/^quotaExceeded: Request rate/)
                if retries <= 10
                  sleep wait_backoff
                  retries += 1
                  wait_backoff = wait_backoff * 2
                  retry
                else
                  raise e
                end
              elsif e.message.match(/^invalidParameter:|^badRequest:/)
                MU.log "#{e.class.name} calling #{@api.class.name}.#{method_sym.to_s}: "+e.message, MU::ERR, details: arguments
# uncomment for debugging stuff; this can occur in benign situations so we don't normally want it logging
              elsif e.message.match(/^forbidden:/)
                MU.log "#{e.class.name} calling #{@api.class.name}.#{method_sym.to_s} got \"#{e.message}\" using credentials #{@credentials}#{@masquerade ? " (OAuth'd as #{@masquerade})": ""}.#{@scopes ? "\nScopes:\n#{@scopes.join("\n")}" : "" }", MU::DEBUG, details: arguments
                raise e
              end
              @@enable_semaphores ||= {}
              max_retries = 3
              wait_time = 90
              if enable_on_fail and retries <= max_retries and e.message.match(/^accessNotConfigured/)
                enable_obj = nil

                project = if arguments.size > 0 and arguments.first.is_a?(String)
                  arguments.first
                else
                  MU::Cloud::Google.defaultProject(@credentials)
                end
# XXX validate that this actually looks like a project id, maybe
                if method_sym == :delete and !MU::Cloud::Google::Habitat.isLive?(project, @credentials)
                  MU.log "Got accessNotConfigured while attempting to delete a resource in #{project}", MU::WARN
                   
                  return
                end

                @@enable_semaphores[project] ||= Mutex.new
                enable_obj = MU::Cloud::Google.service_manager(:EnableServiceRequest).new(
                  consumer_id: "project:"+project.gsub(/^projects\/([^\/]+)\/.*/, '\1')
                )
                # XXX dumbass way to get this string
                if e.message.match(/by visiting https:\/\/console\.developers\.google\.com\/apis\/api\/(.+?)\//)

                  svc_name = Regexp.last_match[1]
                  save_verbosity = MU.verbosity
                  if !["servicemanagement.googleapis.com", "billingbudgets.googleapis.com"].include?(svc_name) and method_sym != :delete
                    retries += 1
                    @@enable_semaphores[project].synchronize {
                      MU.setLogging(MU::Logger::NORMAL)
                      MU.log "Attempting to enable #{svc_name} in project #{project.gsub(/^projects\/([^\/]+)\/.*/, '\1')}; will retry #{method_sym.to_s} in #{(wait_time/retries).to_s}s (#{retries.to_s}/#{max_retries.to_s})", MU::NOTICE
                      MU.setLogging(save_verbosity)
                      begin
                        MU::Cloud::Google.service_manager(credentials: @credentials).enable_service(svc_name, enable_obj)
                      rescue ::Google::Apis::ClientError => e
                        MU.log "Error enabling #{svc_name} in #{project.gsub(/^projects\/([^\/]+)\/.*/, '\1')} for #{method_sym.to_s}: "+ e.message, MU::ERR, details: enable_obj
                        raise e
                      end
                    }
                    sleep wait_time/retries
                    retry
                  else
                    MU.setLogging(MU::Logger::NORMAL)
                    MU.log "Google Cloud's Service Management API must be enabled manually by visiting #{e.message.gsub(/.*?(https?:\/\/[^\s]+)(?:$|\s).*/, '\1')}", MU::ERR
                    MU.setLogging(save_verbosity)
                    raise MU::MuError, "Service Management API not yet enabled for this account/project"
                  end
                elsif e.message.match(/scheduled for deletion and cannot be used for API calls/)
                  raise MuDefunctHabitat, e.message
                else
                  MU.log "Unfamiliar error calling #{method_sym.to_s} "+e.message, MU::ERR, details: arguments
                end
              elsif retries <= 10 and
                 e.message.match(/^resourceNotReady:/) or
                 (e.message.match(/^resourceInUseByAnotherResource:/) and method_sym.to_s.match(/^delete_/)) or
                 e.message.match(/SSL_connect/)
                if retries > 0 and retries % 3 == 0
                  MU.log "Will retry #{method_sym} after #{e.message} (retry #{retries})", MU::NOTICE, details: arguments
                else
                  MU.log "Will retry #{method_sym} after #{e.message} (retry #{retries})", MU::DEBUG, details: arguments
                end
                retries = retries + 1
                sleep retries*10
                retry
              else
                raise e
              end
            end

            if retval.class.name.match(/.*?::Operation$/)

              retries = 0

              # Check whether the various types of +Operation+ responses say
              # they're done, without knowing which specific API they're from
              def is_done?(retval)
                (retval.respond_to?(:status) and retval.status == "DONE") or (retval.respond_to?(:done) and retval.done)
              end

              begin
                if retries > 0 and retries % 3 == 0
                  MU.log "Waiting for #{method_sym} to be done (retry #{retries})", MU::NOTICE
                else
                  MU.log "Waiting for #{method_sym} to be done (retry #{retries})", MU::DEBUG, details: retval
                end

                if !is_done?(retval)
                  sleep 7
                  begin
                    if retval.class.name.match(/::Compute[^:]*::/)
                      resp = MU::Cloud::Google.compute(credentials: @credentials).get_global_operation(
                        arguments.first, # there's always a project id
                        retval.name
                      )
                      retval = resp
                    elsif retval.class.name.match(/::Servicemanagement[^:]*::/)
                      resp = MU::Cloud::Google.service_manager(credentials: @credentials).get_operation(
                        retval.name
                      )
                      retval = resp
                    elsif retval.class.name.match(/::Cloudresourcemanager[^:]*::/)
                      resp = MU::Cloud::Google.resource_manager(credentials: @credentials).get_operation(
                        retval.name
                      )
                      retval = resp
                      if retval.error
                        raise MuError, retval.error.message
                      end
                    elsif retval.class.name.match(/::Container[^:]*::/)
                      resp = MU::Cloud::Google.container(credentials: @credentials).get_project_location_operation(
                        retval.self_link.sub(/.*?\/projects\//, 'projects/')
                      )
                      retval = resp
                    elsif retval.class.name.match(/::Cloudfunctions[^:]*::/)
                      resp = MU::Cloud::Google.function(credentials: @credentials).get_operation(
                        retval.name
                      )
                      retval = resp
#MU.log method_sym.to_s, MU::WARN, details: retval
                      if retval.error
                        raise MuError, retval.error.message
                      end
                    else
                      pp retval
                      raise MuError, "I NEED TO IMPLEMENT AN OPERATION HANDLER FOR #{retval.class.name}"
                    end
                  rescue ::Google::Apis::ClientError => e
                    # this is ok; just means the operation is done and went away
                    if e.message.match(/^notFound:/)
                      break
                    else
                      raise e
                    end
                  end
                  retries = retries + 1
                end

              end while !is_done?(retval)

              # Most insert methods have a predictable get_* counterpart. Let's
              # take advantage.
              # XXX might want to do something similar for delete ops? just the
              # but where we wait for the operation to definitely be done
#              had_been_found = false
              if method_sym.to_s.match(/^(insert|create|patch)_/)
                get_method = method_sym.to_s.gsub(/^(insert|patch|create_disk|create)_/, "get_").to_sym
                cloud_id = if retval.respond_to?(:target_link)
                  retval.target_link.sub(/^.*?\/([^\/]+)$/, '\1')
                elsif retval.respond_to?(:metadata) and retval.metadata["target"]
                  retval.metadata["target"]
                else
                  arguments[0] # if we're lucky
                end
                faked_args = arguments.dup
                faked_args.pop
                if get_method == :get_snapshot
                  faked_args.pop
                  faked_args.pop
                end
                faked_args.push(cloud_id)
                if get_method == :get_project_location_cluster
                  faked_args[0] = faked_args[0]+"/clusters/"+faked_args[1]
                  faked_args.pop
                elsif get_method == :get_project_location_function
                  faked_args = [cloud_id]
                end
                actual_resource = @api.method(get_method).call(*faked_args)
#if method_sym == :insert_instance
#MU.log "actual_resource", MU::WARN, details: actual_resource
#end
#                had_been_found = true
                if actual_resource.respond_to?(:status) and
                  ["PROVISIONING", "STAGING", "PENDING", "CREATING", "RESTORING"].include?(actual_resource.status)
                  retries = 0
                  begin 
                    if retries > 0 and retries % 3 == 0
                      MU.log "Waiting for #{cloud_id} to get past #{actual_resource.status} (retry #{retries})", MU::NOTICE
                    else
                      MU.log "Waiting for #{cloud_id} to get past #{actual_resource.status} (retry #{retries})", MU::DEBUG, details: actual_resource
                    end
                    sleep 10
                    actual_resource = @api.method(get_method).call(*faked_args)
                    retries = retries + 1
                  end while ["PROVISIONING", "STAGING", "PENDING", "CREATING", "RESTORING"].include?(actual_resource.status)
                end
                return actual_resource
              end
            end

            # This atrocity appends the pages of list_* results
            if overall_retval
              if method_sym.to_s.match(/^list_(.*)/)
                require 'google/apis/iam_v1'
                require 'google/apis/logging_v2'
                what = Regexp.last_match[1].to_sym
                whatassign = (Regexp.last_match[1]+"=").to_sym
                if overall_retval.class == ::Google::Apis::IamV1::ListServiceAccountsResponse
                  what = :accounts
                  whatassign = :accounts=
                end
                if retval.respond_to?(what) and retval.respond_to?(whatassign)
                  if !retval.public_send(what).nil?
                    newarray = retval.public_send(what) + overall_retval.public_send(what)
                    overall_retval.public_send(whatassign, newarray)
                  end
                elsif !retval.respond_to?(:next_page_token) or retval.next_page_token.nil? or retval.next_page_token.empty?
                  MU.log "Not sure how to append #{method_sym.to_s} results to #{overall_retval.class.name} (apparently #{what.to_s} and #{whatassign.to_s} aren't it), returning first page only", MU::WARN, details: retval
                  return retval
                end
              else
                MU.log "Not sure how to append #{method_sym.to_s} results, returning first page only", MU::WARN, details: retval
                return retval
              end
            else
              overall_retval = retval
            end

            arguments.delete({ :page_token => next_page_token })
            next_page_token = nil

            if retval.respond_to?(:next_page_token) and !retval.next_page_token.nil?
              next_page_token = retval.next_page_token
              MU.log "Getting another page of #{method_sym.to_s}", MU::DEBUG, details: next_page_token
            else
              return overall_retval
            end
          rescue ::Google::Apis::ServerError, ::Google::Apis::ClientError, ::Google::Apis::TransmissionError => e
            if e.class.name == "Google::Apis::ClientError" and
               (!method_sym.to_s.match(/^insert_/) or !e.message.match(/^notFound: /) or
                (e.message.match(/^notFound: /) and method_sym.to_s.match(/^insert_/))
               )
              if e.message.match(/^notFound: /) and method_sym.to_s.match(/^insert_/) and retval
                logreq = MU::Cloud::Google.logging(:ListLogEntriesRequest).new(
                  resource_names: ["projects/"+arguments.first],
                  filter: %Q{labels."compute.googleapis.com/resource_id"="#{retval.target_id}" OR labels."ssl_certificate_id"="#{retval.target_id}"} # XXX I guess we need to cover all of the possible keys, ugh
                )
                logs = MU::Cloud::Google.logging(credentials: @credentials).list_entry_log_entries(logreq)
                details = nil
                if logs.entries
                  details = logs.entries.map { |err| err.json_payload }
                  details.reject! { |err| err["error"].nil? or err["error"].size == 0 }
                end

                raise MuError, "#{method_sym.to_s} of #{retval.target_id} appeared to succeed, but then the resource disappeared! #{details.to_s}"
              end
              raise e
            end
            retries = retries + 1
            debuglevel = MU::DEBUG
            interval = 5 + Random.rand(4) - 2
            if retries < 10 and retries > 2
              debuglevel = MU::NOTICE
              interval = 20 + Random.rand(10) - 3
            # elsif retries >= 10 and retries <= 100
            elsif retries >= 10
              debuglevel = MU::WARN
              interval = 40 + Random.rand(15) - 5
            # elsif retries > 100
              # raise MuError, "Exhausted retries after #{retries} attempts while calling Compute's #{method_sym} in #{@region}.  Args were: #{arguments}"
            end

            MU.log "Got #{e.inspect} calling Google's #{method_sym}, waiting #{interval.to_s}s and retrying. Called from: #{caller[1]}", debuglevel, details: arguments
            sleep interval
            MU.log method_sym.to_s.bold+" "+e.inspect, MU::WARN, details: arguments
            retry
          end while !next_page_token.nil?
        end