class MU::Cloud::AWS
Support for Amazon Web Services as a provisioning layer.
Attributes
Public Class Methods
Fetch the AWS
account number where this Mu master resides. If it's not in AWS
at all, or otherwise cannot be determined, return nil. here. XXX account for Google
and non-cloud situations XXX this needs to be “myAccountNumber” or somesuch XXX and maybe do the IAM thing for arbitrary, non-resident accounts
# File modules/mu/providers/aws.rb, line 775 def self.account_number require "aws-sdk-ec2" return nil if credConfig.nil? return @@my_acct_num if @@my_acct_num loadCredentials # XXX take optional credential set argument # begin # user_list = MU::Cloud::AWS.iam(region: credConfig['region']).list_users.users ## rescue ::Aws::IAM::Errors => e # XXX why does this NameError here? # rescue StandardError => e # MU.log "Got #{e.inspect} while trying to figure out our account number", MU::WARN, details: caller # end # if user_list.nil? or user_list.size == 0 resp = MU::Cloud::AWS.getAWSMetaData("network/interfaces/macs/") return nil if !resp mac = resp.split(/\n/)[0] acct_num = MU::Cloud::AWS.getAWSMetaData("network/interfaces/macs/#{mac}owner-id") acct_num.chomp! # else # acct_num = MU::Cloud::AWS.iam(region: credConfig['region']).list_users.users.first.arn.split(/:/)[4] # end MU.setVar("acct_num", acct_num) @@my_acct_num ||= acct_num acct_num end
Amazon Certificate Manager API
# File modules/mu/providers/aws.rb, line 1077 def self.acm(region: MU.curRegion, credentials: nil) region ||= myRegion @@acm_api[credentials] ||= {} @@acm_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "ACM", region: region, credentials: credentials) @@acm_api[credentials][region] end
Resolve the administrative S3 bucket for a given credential set, or return a default. @param credentials [String] @return [String]
# File modules/mu/providers/aws.rb, line 652 def self.adminBucketName(credentials = nil) require "aws-sdk-s3" cfg = credConfig(credentials) return nil if !cfg if !cfg['log_bucket_name'] cfg['log_bucket_name'] = $MU_CFG['hostname'] MU.log "No AWS log bucket defined for credentials #{credentials}, attempting to use default of #{cfg['log_bucket_name']}", MU::WARN end resp = MU::Cloud::AWS.s3(credentials: credentials).list_buckets found = false resp.buckets.each { |b| if b.name == cfg['log_bucket_name'] found = true break end } if !found MU.log "Attempting to create log bucket #{cfg['log_bucket_name']} for credentials #{credentials}", MU::WARN begin MU::Cloud::AWS.s3(credentials: credentials).create_bucket(bucket: cfg['log_bucket_name'], acl: "private") rescue Aws::S3::Errors::BucketAlreadyExists raise MuError, "AWS credentials #{credentials} need a log bucket, and the name #{cfg['log_bucket_name']} is unavailable. Use mu-configure to edit credentials '#{credentials}' or 'hostname'" end end cfg['log_bucket_name'] end
Resolve the administrative S3 bucket for a given credential set, or return a default. @param credentials [String] @return [String]
# File modules/mu/providers/aws.rb, line 684 def self.adminBucketUrl(credentials = nil) return nil if !credConfig(credentials) "s3://"+adminBucketName(credentials)+"/" end
Amazon's API Gateway API
# File modules/mu/providers/aws.rb, line 1250 def self.apig(region: MU.curRegion, credentials: nil) region ||= myRegion @@apig_api[credentials] ||= {} @@apig_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "APIGateway", region: region, credentials: credentials) @@apig_api[credentials][region] end
If we're in AWS
and NVME-aware, return a mapping of AWS-side device names to actual NVME devices. @return [Hash]
# File modules/mu/providers/aws.rb, line 476 def self.attachedNVMeDisks if !hosted? or !File.executable?("/bin/lsblk") or !File.executable?("/sbin/nvme") return {} end map = {} devices = MU::Master.listBlockDevices return {} if !devices devices.each { |d| if d =~ /^\/dev\/nvme/ %x{/sbin/nvme id-ctrl -v #{d}}.each_line { |desc| if desc.match(/^0000: (?:[0-9a-f]{2} ){16}"(.+?)\./) virt_dev = Regexp.last_match[1] map[virt_dev] = d break end } end } map end
Amazon's Autoscaling API
# File modules/mu/providers/aws.rb, line 1099 def self.autoscale(region: MU.curRegion, credentials: nil) region ||= myRegion @@autoscale_api[credentials] ||= {} @@autoscale_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "AutoScaling", region: region, credentials: credentials) @@autoscale_api[credentials][region] end
Purge cloud-specific deploy meta-artifacts (SSH keys, resource groups, etc) @param deploy_id [MU::MommaCat]
# File modules/mu/providers/aws.rb, line 387 def self.cleanDeploy(deploy_id, credentials: nil, noop: false) if !noop MU.log "Deleting s3://#{adminBucketName(credentials)}/#{deploy_id}-secret" MU::Cloud::AWS.s3(credentials: credentials).delete_object( bucket: adminBucketName(credentials), key: "#{deploy_id}-secret" ) listRegions(credentials: credentials).each { |r| resp = MU::Cloud::AWS.ec2(region: r, credentials: credentials).describe_key_pairs( filters: [{name: "key-name", values: ["deploy-#{MU.deploy_id}"]}] ) resp.data.key_pairs.each { |keypair| MU.log "Deleting key pair #{keypair.key_name} from #{r}" MU::Cloud::AWS.ec2(region: r, credentials: credentials).delete_key_pair(key_name: keypair.key_name) if !noop } } end if hosted? MU::Cloud::AWS.openFirewallForClients end end
Amazon's CloudFormation
API
# File modules/mu/providers/aws.rb, line 1137 def self.cloudformation(region: MU.curRegion, credentials: nil) region ||= myRegion @@cloudformation_api[credentials] ||= {} @@cloudformation_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudFormation", region: region, credentials: credentials) @@cloudformation_api[credentials][region] end
Amazon's CloudFront API
# File modules/mu/providers/aws.rb, line 1202 def self.cloudfront(region: MU.curRegion, credentials: nil) region ||= myRegion @@cloudfront_api[credentials] ||= {} @@cloudfront_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudFront", region: region, credentials: credentials) @@cloudfront_api[credentials][region] end
Amazon's CloudTrail API
# File modules/mu/providers/aws.rb, line 1153 def self.cloudtrail(region: MU.curRegion, credentials: nil) region ||= myRegion @@cloudtrail_api[credentials] ||= {} @@cloudtrail_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudTrail", region: region, credentials: credentials) @@cloudtrail_api[credentials][region] end
Log
bucket policy for enabling CloudTrail logging to our log bucket in S3.
# File modules/mu/providers/aws.rb, line 431 def self.cloudtrailBucketPolicy(credentials = nil) cfg = credConfig(credentials) policy_json = '{ "Version": "2012-10-17", "Statement": [ { "Sid": "AWSCloudTrailAclCheck20131101", "Effect": "Allow", "Principal": { "AWS": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':iam::<%= MU.account_number %>:root", "Service": "cloudtrail.amazonaws.com" }, "Action": "s3:GetBucketAcl", "Resource": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(credentials)+'" }, { "Sid": "AWSCloudTrailWrite20131101", "Effect": "Allow", "Principal": { "AWS": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':iam::'+credToAcct(credentials)+':root", "Service": "cloudtrail.amazonaws.com" }, "Action": "s3:PutObject", "Resource": "arn:'+(MU::Cloud::AWS.isGovCloud?(cfg['region']) ? "aws-us-gov" : "aws")+':s3:::'+MU::Cloud::AWS.adminBucketName(credentials)+'/AWSLogs/'+credToAcct(credentials)+'/*", "Condition": { "StringEquals": { "s3:x-amz-acl": "bucket-owner-full-control" } } } ] }' ERB.new(policy_json).result end
Amazon's CloudWatch API
# File modules/mu/providers/aws.rb, line 1161 def self.cloudwatch(region: MU.curRegion, credentials: nil) region ||= myRegion @@cloudwatch_api[credentials] ||= {} @@cloudwatch_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudWatch", region: region, credentials: credentials) @@cloudwatch_api[credentials][region] end
Amazon's Cloudwatch Events API
# File modules/mu/providers/aws.rb, line 1258 def self.cloudwatch_events(region = MU.cureRegion) region ||= myRegion @@cloudwatch_events_api[credentials] ||= {} @@cloudwatch_events_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudWatchEvents", region: region, credentials: credentials) @@cloudwatch_events_api end
Amazon's CloudWatchEvents API
# File modules/mu/providers/aws.rb, line 1194 def self.cloudwatchevents(region: MU.curRegion, credentials: nil) region ||= myRegion @@cloudwatchevents_api[credentials] ||= {} @@cloudwatchevents_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudWatchEvents", region: region, credentials: credentials) @@cloudwatchevents_api[credentials][region] end
Amazon's CloudWatchLogs API
# File modules/mu/providers/aws.rb, line 1186 def self.cloudwatchlogs(region: MU.curRegion, credentials: nil) region ||= myRegion @@cloudwatchlogs_api[credentials] ||= {} @@cloudwatchlogs_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CloudWatchLogs", region: region, credentials: credentials) @@cloudwatchlogs_api[credentials][region] end
Amazon's Cognito Identity API
# File modules/mu/providers/aws.rb, line 1322 def self.cognito_ident(region: MU.curRegion, credentials: nil) region ||= myRegion @@cognito_ident_api[credentials] ||= {} @@cognito_ident_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CognitoIdentity", region: region, credentials: credentials) @@cognito_ident_api[credentials][region] end
Amazon's Cognito Identity Provider API
# File modules/mu/providers/aws.rb, line 1330 def self.cognito_user(region: MU.curRegion, credentials: nil) region ||= myRegion @@cognito_user_api[credentials] ||= {} @@cognito_user_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "CognitoIdentityProvider", region: region, credentials: credentials) @@cognito_user_api[credentials][region] end
A non-working example configuration
# File modules/mu/providers/aws.rb, line 573 def self.config_example sample = hosted_config sample ||= { "region" => "us-east-1", "account_number" => "123456789012", } # sample["access_key"] = "AKIAIXKNI3JY6JVVJIHA" # sample["access_secret"] = "oWjHT+2N3veyswy7+UA5i+H14KpvrOIZlnRlxpkw" sample["credentials_file"] = "#{Etc.getpwuid(Process.uid).dir}/.aws/credentials" sample["log_bucket_name"] = "my-mu-s3-bucket" sample end
Generate an EC2 keypair unique to this deployment, given a regular OpenSSH-style public key and a name. @param keyname [String]: The name of the key to create. @param public_key [String]: The public key @return [Array<String>]: keypairname, ssh_private_key, ssh_public_key
# File modules/mu/providers/aws.rb, line 858 def self.createEc2SSHKey(keyname, public_key, credentials: nil) require "aws-sdk-ec2" # We replicate this key in all regions if !MU::Cloud::CloudFormation.emitCloudFormation MU::Cloud::AWS.listRegions.each { |region| MU.log "Replicating #{keyname} to EC2 in #{region}", MU::DEBUG, details: @ssh_public_key begin MU::Cloud::AWS.ec2(region: region, credentials: credentials).import_key_pair( key_name: keyname, public_key_material: public_key ) rescue ::Aws::EC2::Errors::AuthFailure => e @@regions_semaphore.synchronize { @@regions.delete(region) } MU.log "#{region} threw #{e.message}, skipping", MU::ERR end } end end
Tag an EC2 resource
@param resource [String]: The cloud provider identifier of the resource to tag @param region [String]: The cloud provider region @param credentials [String]: Credentials to authorize API requests @param optional [Boolean]: Whether to apply our optional generic tags @param nametag [String]: A Name
tag to apply @param othertags [Array<Hash>]: Miscellaneous custom tags, in Basket of Kittens style @return [void]
# File modules/mu/providers/aws.rb, line 215 def self.createStandardTags(resource = nil, region: MU.curRegion, credentials: nil, optional: true, nametag: nil, othertags: nil) require "aws-sdk-ec2" tags = [] MU::MommaCat.listStandardTags.each_pair { |name, value| tags << {key: name, value: value} if !value.nil? } if optional MU::MommaCat.listOptionalTags.each { |key, value| tags << {key: name, value: value} if !value.nil? } end if nametag tags << { key: "Name", value: nametag } end if othertags othertags.each { |tag| tags << { key: tag['key'], value: tag['value'] } } end if MU::Cloud::CloudFormation.emitCloudFormation return tags end attempts = 0 begin MU::Cloud::AWS.ec2(region: region, credentials: credentials).create_tags( resources: [resource], tags: tags ) rescue Aws::EC2::Errors::ServiceError => e MU.log "Got #{e.inspect} tagging #{resource} in #{region}, will retry", MU::WARN, details: caller.concat(tags) if attempts > 1 if attempts < 5 attempts = attempts + 1 sleep 15 retry else raise e end end MU.log "Created standard tags for resource #{resource}", MU::DEBUG, details: caller end
Tag a resource. Defaults to applying our MU
deployment identifier, if no arguments other than the resource identifier are given.
@param resource [String]: The cloud provider identifier of the resource to tag @param tag_name [String]: The name of the tag to create @param tag_value [String]: The value of the tag @param region [String]: The cloud provider region @return [void]
# File modules/mu/providers/aws.rb, line 1389 def self.createTag(resource = nil, tag_name="MU-ID", tag_value=MU.deploy_id, region: MU.curRegion, credentials: nil) require "aws-sdk-ec2" attempts = 0 return nil if resource.nil? resource = [resource] if resource.is_a?(String) if !MU::Cloud::CloudFormation.emitCloudFormation begin MU::Cloud::AWS.ec2(credentials: credentials, region: region).create_tags( resources: resource, tags: [ { key: tag_name, value: tag_value } ] ) rescue Aws::EC2::Errors::ServiceError => e MU.log "Got #{e.inspect} tagging #{resource} with #{tag_name}=#{tag_value}", MU::WARN if attempts > 1 if attempts < 5 attempts = attempts + 1 sleep 15 retry else raise e end end MU.log "Created tag #{tag_name} with value #{tag_value} for resource #{resource}", MU::DEBUG else return { "Key" => tag_name, "Value" => tag_value } end end
Return the $MU_CFG data associated with a particular profile/name/set of credentials. If no account name is specified, will return one flagged as default. Returns nil if AWS
is not configured. Throws an exception if an account name is specified which does not exist. @param name [String]: The name of the key under 'aws' in mu.yaml to return @return [Hash,nil]
# File modules/mu/providers/aws.rb, line 695 def self.credConfig(name = nil, name_only: false) # If there's nothing in mu.yaml (which is wrong), but we're running # on a machine hosted in AWS, *and* that machine has an IAM profile, # fake it with those credentials and hope for the best. if !$MU_CFG['aws'] or !$MU_CFG['aws'].is_a?(Hash) or $MU_CFG['aws'].size == 0 if @@my_hosted_cfg return name_only ? "#default" : @@my_hosted_cfg end if hosted? begin iam_blob = getAWSMetaData("iam/info") if iam_blob iam_data = JSON.parse(iam_blob) if iam_data["InstanceProfileArn"] and !iam_data["InstanceProfileArn"].empty? @@my_hosted_cfg = hosted_config return name_only ? "#default" : @@my_hosted_cfg end end rescue JSON::ParserError => e end elsif ENV['AWS_ACCESS_KEY_ID'] and ENV['AWS_SECRET_ACCESS_KEY'] env_config = { "region" => ENV['EC2_REGION'] || "us-east-1", "access_key" => ENV['AWS_ACCESS_KEY_ID'], "access_secret" => ENV['AWS_SECRET_ACCESS_KEY'], "log_bucket_name" => "mu-placeholder-bucket-name" } return name_only ? "#default" : env_config end return nil end if name.nil? $MU_CFG['aws'].each_pair { |set, cfg| if cfg['default'] return name_only ? set : cfg end } else if $MU_CFG['aws'][name] return name_only ? name : $MU_CFG['aws'][name] elsif @@acct_to_profile_map[name.to_s] return name_only ? name : @@acct_to_profile_map[name.to_s] elsif name.is_a?(Integer) or name.match(/^\d+$/) # Try to map backwards from an account id, if that's what we go $MU_CFG['aws'].each_pair { |acctname, cfg| if cfg['account_number'] and name.to_s == cfg['account_number'].to_s return name_only ? acctname : $MU_CFG['aws'][acctname] end } # Check each credential sets' resident account, then $MU_CFG['aws'].each_pair { |acctname, cfg| begin MU::Cloud::AWS.iam(credentials: acctname).list_users.users # rescue ::Aws::IAM::Errors => e # XXX why does this NameError here? rescue StandardError => e MU.log e.inspect, MU::WARN, details: cfg next end acct_num = MU::Cloud::AWS.iam(credentials: acctname).list_users.users.first.arn.split(/:/)[4] cfg['account_number'] ||= acct_num.to_s if acct_num.to_s == name.to_s @@acct_to_profile_map[name.to_s] = cfg return name_only ? name.to_s : cfg end } end raise MuError, "AWS credential set #{name} was requested, but I see no such working credentials in mu.yaml" end end
Map the name of a credential set back to an AWS
account number @param name [String]
# File modules/mu/providers/aws.rb, line 627 def self.credToAcct(name = nil) creds = credConfig(name) if creds['account_number'] and !creds['account_number'].empty? return creds['account_number'] end acct_num = MU::Cloud::AWS.iam(credentials: name).list_users.users.first.arn.split(/:/)[4] acct_num.to_s end
Amazon's DynamoDB API
# File modules/mu/providers/aws.rb, line 1282 def self.dynamo(region: MU.curRegion, credentials: nil) region ||= myRegion @@dynamo_api[credentials] ||= {} @@dynamo_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "DynamoDB", region: region, credentials: credentials) @@dynamo_api[credentials][region] end
Amazon's DynamoStream API
# File modules/mu/providers/aws.rb, line 1290 def self.dynamostream(region: MU.curRegion, credentials: nil) region ||= myRegion @@dynamostream_api[credentials] ||= {} @@dynamostream_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "DynamoDBStreams", region: region, credentials: credentials) @@dynamostream_api[credentials][region] end
Amazon's EC2 API
# File modules/mu/providers/aws.rb, line 1091 def self.ec2(region: MU.curRegion, credentials: nil) region ||= myRegion @@ec2_api[credentials] ||= {} @@ec2_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "EC2", region: region, credentials: credentials) @@ec2_api[credentials][region] end
Amazon's ECS API
# File modules/mu/providers/aws.rb, line 1266 def self.ecs(region: MU.curRegion, credentials: nil) region ||= myRegion @@ecs_api[credentials] ||= {} @@ecs_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "ECS", region: region, credentials: credentials) @@ecs_api[credentials][region] end
Amazon's EFS API
# File modules/mu/providers/aws.rb, line 1234 def self.efs(region: MU.curRegion, credentials: nil) region ||= myRegion @@efs_api[credentials] ||= {} @@efs_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "EFS", region: region, credentials: credentials) @@efs_api[credentials][region] end
Amazon's EKS API
# File modules/mu/providers/aws.rb, line 1274 def self.eks(region: MU.curRegion, credentials: nil) region ||= myRegion @@eks_api[credentials] ||= {} @@eks_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "EKS", region: region, credentials: credentials) @@eks_api[credentials][region] end
Amazon's ElastiCache API
# File modules/mu/providers/aws.rb, line 1210 def self.elasticache(region: MU.curRegion, credentials: nil) region ||= myRegion @@elasticache_api[credentials] ||= {} @@elasticache_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "ElastiCache", region: region, credentials: credentials) @@elasticache_api[credentials][region] end
Amazon's Elasticsearch API
# File modules/mu/providers/aws.rb, line 1314 def self.elasticsearch(region: MU.curRegion, credentials: nil) region ||= myRegion @@elasticsearch_api[credentials] ||= {} @@elasticsearch_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "ElasticsearchService", region: region, credentials: credentials) @@elasticsearch_api[credentials][region] end
Amazon's ElasticLoadBalancing API
# File modules/mu/providers/aws.rb, line 1107 def self.elb(region: MU.curRegion, credentials: nil) region ||= myRegion @@elb_api[credentials] ||= {} @@elb_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "ElasticLoadBalancing", region: region, credentials: credentials) @@elb_api[credentials][region] end
Amazon's ElasticLoadBalancingV2 (ALB) API
# File modules/mu/providers/aws.rb, line 1115 def self.elb2(region: MU.curRegion, credentials: nil) region ||= myRegion @@elb2_api[credentials] ||= {} @@elb2_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "ElasticLoadBalancingV2", region: region, credentials: credentials) @@elb2_api[credentials][region] end
AWS
can stash API-available certificates in Amazon Certificate Manager or in IAM. Rather than make people crazy trying to get the syntax correct in our Baskets of Kittens, let's have a helper that tries to do the right thing, and only raise an exception if we need help to disambiguate. @param name [String]: The name of the cert. For IAM certs this can be any IAM name; for ACM, it's usually the domain name. If multiple matches are found, or no matches, an exception is raised. @param id [String]: The ARN of a known certificate. We just validate that it exists. This is ignored if a name parameter is supplied. @return [String]: The ARN of a matching certificate that is known to exist. If it is an ACM certificate, we also know that it is not expired.
# File modules/mu/providers/aws.rb, line 950 def self.findSSLCertificate(name: nil, id: nil, region: myRegion, credentials: nil, raise_on_missing: true) require "aws-sdk-iam" if (name.nil? or name.empty?) and (id.nil? or id.empty?) raise MuError, "Can't call findSSLCertificate without specifying either a name or an id" end if id and @@certificates[id] return [id, @@certificates[id]] end if !name.nil? and !name.empty? matches = [] acmcerts = MU::Cloud::AWS.acm(region: region, credentials: credentials).list_certificates( certificate_statuses: ["ISSUED"] ) acmcerts.certificate_summary_list.each { |cert| matches << cert.certificate_arn if cert.domain_name == name } begin iamcert = MU::Cloud::AWS.iam(credentials: credentials).get_server_certificate( server_certificate_name: name ) rescue Aws::IAM::Errors::ValidationError, Aws::IAM::Errors::NoSuchEntity # valid names for ACM certs can break here, and that's ok to ignore end if !iamcert.nil? matches << iamcert.server_certificate.server_certificate_metadata.arn end if matches.size == 1 id = matches.first elsif matches.size == 0 if raise_on_missing raise MuError, "No IAM or ACM certificate named #{name} was found in #{region}" else return nil end elsif matches.size > 1 raise MuError, "Multiple certificates named #{name} were found in #{region}. Remove extras or use ssl_certificate_id to supply the exact ARN of the one you want to use." end end domains = [] if id.match(/^arn:aws(?:-us-gov)?:acm/) resp = MU::Cloud::AWS.acm(region: region).describe_certificate( certificate_arn: id ) if resp.nil? or resp.certificate.nil? raise MuError, "No such ACM certificate '#{id}'" end domains << resp.certificate.domain_name if resp.certificate.subject_alternative_names domains.concat(resp.certificate.subject_alternative_names) end elsif id.match(/^arn:aws(?:-us-gov)?:iam/) resp = MU::Cloud::AWS.iam.list_server_certificates if resp.nil? raise MuError, "No such IAM certificate '#{id}'" end resp.server_certificate_metadata_list.each { |cert| if cert.arn == id if cert.expiration < Time.now MU.log "IAM SSL certificate #{cert.server_certificate_name} (#{id}) is EXPIRED", MU::WARN end @@certificates[id] = [cert.server_certificate_name] return [id, [cert.server_certificate_name]] end } raise MuError, "No such IAM certificate '#{id}'" else raise MuError, "The format of '#{id}' doesn't look like an ARN for either Amazon Certificate Manager or IAM" end @@certificates[id] = domains.uniq [id, domains.uniq] end
Fetch an Amazon instance metadata parameter (example: public-ipv4). @param param [String]: The parameter name to fetch @return [String, nil]
# File modules/mu/providers/aws.rb, line 1364 def self.getAWSMetaData(param) base_url = "http://169.254.169.254/latest/meta-data/" begin response = nil Timeout.timeout(1) do response = URI.open("#{base_url}/#{param}").read end response rescue OpenURI::HTTPError, Timeout::Error, SocketError, Errno::ENETUNREACH, Net::HTTPServerException, Errno::EHOSTUNREACH => e # This is normal on machines checking to see if they're AWS-hosted logger = MU::Logger.new logger.log "Failed metadata request #{base_url}/#{param}: #{e.inspect}", MU::DEBUG return nil end end
Return what we think of as a cloud object's habitat. In AWS
, this means the account_number
in which it's resident. If this is not applicable, such as for a {Habitat} or {Folder}, returns nil. @param cloudobj [MU::Cloud::AWS]: The resource from which to extract the habitat id @return [String,nil]
# File modules/mu/providers/aws.rb, line 591 def self.habitat(cloudobj, nolookup: false, deploy: nil) @@habmap ||= {} # XXX whaddabout config['habitat'] HNNNGH if cloudobj.respond_to?(:account_number) and cloudobj.account_number and !cloudobj.account_number.empty? return cloudobj.account_number elsif cloudobj.config and cloudobj.config['account'] if nolookup return cloudobj.config['account'] end if @@habmap[cloudobj.config['account']] return @@habmap[cloudobj.config['account']] end deploy ||= cloudobj.deploy if cloudobj.respond_to?(:deploy) MU.log "Incomplete implementation: MU::Cloud::AWS.habitat", MU::DEBUG, details: deploy # accountobj = accountLookup(cloudobj.config['account'], deploy, raise_on_fail: false) # if accountobj # @@habmap[cloudobj.config['account']] = accountobj.cloud_id # return accountobj.cloud_id # end end nil end
Alias for #{MU::Cloud::AWS.hosted?}
# File modules/mu/providers/aws.rb, line 469 def self.hosted MU::Cloud::AWS.hosted? end
Determine whether we (the Mu master, presumably) are hosted in this cloud. @return [Boolean]
# File modules/mu/providers/aws.rb, line 523 def self.hosted? if $MU_CFG.has_key?("aws_is_hosted") @@is_in_aws = $MU_CFG["aws_is_hosted"] return $MU_CFG["aws_is_hosted"] end require 'open-uri' if !@@is_in_aws.nil? return @@is_in_aws end begin Timeout.timeout(4) do instance_id = URI.open("http://169.254.169.254/latest/meta-data/instance-id").read if !instance_id.nil? and instance_id.size > 0 @@is_in_aws = true region = getAWSMetaData("placement/availability-zone").sub(/[a-z]$/i, "") begin validate_region(region) rescue MuError @@creds_loaded.delete("#default") @@is_in_aws = false false end return true end end rescue OpenURI::HTTPError, Timeout::Error, SocketError, Errno::EHOSTUNREACH end @@is_in_aws = false false end
If we're running this cloud, return the $MU_CFG blob we'd use to describe this environment as our target one.
# File modules/mu/providers/aws.rb, line 560 def self.hosted_config return nil if !hosted? region = getAWSMetaData("placement/availability-zone").sub(/[a-z]$/i, "") mac = getAWSMetaData("network/interfaces/macs/").split(/\n/)[0] acct_num = getAWSMetaData("network/interfaces/macs/#{mac}owner-id") acct_num.chomp! { "region" => region, "account_number" => acct_num } end
Amazon's IAM API
# File modules/mu/providers/aws.rb, line 1085 def self.iam(credentials: nil) @@iam_api[credentials] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "IAM", credentials: credentials) @@iam_api[credentials] end
Do cloud-specific deploy instantiation tasks, such as copying SSH keys around, sticking secrets in buckets, creating resource groups, etc @param deploy [MU::MommaCat]
# File modules/mu/providers/aws.rb, line 381 def self.initDeploy(deploy) end
Is the region we're dealing with a GovCloud region? @param region [String]: The region in question, defaults to the Mu Master's local region
# File modules/mu/providers/aws.rb, line 333 def self.isGovCloud?(region = myRegion) return false if !region region.match(/^us-gov-/) end
Amazon's KMS API
# File modules/mu/providers/aws.rb, line 1338 def self.kms(region: MU.curRegion, credentials: nil) region ||= myRegion @@kms_api[credentials] ||= {} @@kms_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "KMS", region: region, credentials: credentials) @@kms_api[credentials][region] end
Amazon's Lambda API
# File modules/mu/providers/aws.rb, line 1242 def self.lambda(region: MU.curRegion, credentials: nil) region ||= myRegion @@lambda_api[credentials] ||= {} @@lambda_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "Lambda", region: region, credentials: credentials) @@lambda_api[credentials][region] end
List the Availability Zones associated with a given Amazon Web Services region. If no region is given, search the one in which this MU
master server resides. @param region [String]: The region to search. @return [Array<String>]: The Availability Zones in this region.
# File modules/mu/providers/aws.rb, line 360 def self.listAZs(region: MU.curRegion, credentials: nil) cfg = credConfig(credentials) return [] if !cfg if !region.nil? and @@azs[region] return @@azs[region] end if region azs = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_availability_zones( filters: [name: "region-name", values: [region]] ) end @@azs[region] ||= [] azs.data.availability_zones.each { |az| @@azs[region] << az.zone_name if az.state == "available" } return @@azs[region] end
Return the name strings of all known sets of credentials for this cloud @return [Array<String>]
# File modules/mu/providers/aws.rb, line 640 def self.listCredentials if !$MU_CFG['aws'] return hosted? ? ["#default"] : nil end $MU_CFG['aws'].keys end
List all AWS
projects available to our credentials
# File modules/mu/providers/aws.rb, line 41 def self.listHabitats(credentials = nil, use_cache: true) cfg = credConfig(credentials) return [] if !cfg or !cfg['account_number'] [cfg['account_number']] end
Query the AWS
API for the list of valid EC2 instance types and some of their attributes. We can use this in config validation and to help “translate” machine types across cloud providers. @param region [String]: Supported machine types can vary from region to region, so we look for the set we're interested in specifically @return [Hash]
# File modules/mu/providers/aws.rb, line 885 def self.listInstanceTypes(region = myRegion) return @@instance_types if @@instance_types and @@instance_types[region] return {} if credConfig.nil? if region.nil? region = myRegion(debug: true) end return {} if region.nil? human_region = @@regionLookup[region] if human_region.nil? MU.log "Failed to map a Pricing API region name from #{region}", MU::ERR return {} end @@instance_types ||= {} @@instance_types[region] ||= {} # Pricing API isn't widely available, so ask a region we know supports # it resp = MU::Cloud::AWS.pricing(region: "us-east-1").get_products( service_code: "AmazonEC2", filters: [ { field: "productFamily", value: "Compute Instance", type: "TERM_MATCH" }, { field: "tenancy", value: "Shared", type: "TERM_MATCH" }, { field: "location", value: human_region, type: "TERM_MATCH" } ] ) resp.price_list.each { |pricing| data = JSON.parse(pricing) type = data["product"]["attributes"]["instanceType"] next if @@instance_types[region].has_key?(type) @@instance_types[region][type] = {} ["ecu", "vcpu", "memory", "storage"].each { |a| @@instance_types[region][type][a] = data["product"]["attributes"][a] } @@instance_types[region][type]["memory"].sub!(/ GiB/, "") @@instance_types[region][type]["memory"] = @@instance_types[region][type]["memory"].to_f @@instance_types[region][type]["vcpu"] = @@instance_types[region][type]["vcpu"].to_f } @@instance_types end
List the Amazon Web Services region names available to this account. The region that is local to this Mu server will be listed first. @param us_only [Boolean]: Restrict results to United States only @return [Array<String>]
# File modules/mu/providers/aws.rb, line 809 def self.listRegions(us_only = false, credentials: nil) if @@regions.size == 0 return [] if credConfig.nil? result = MU::Cloud::AWS.ec2(region: myRegion, credentials: credentials).describe_regions.regions @@regions_semaphore.synchronize { begin result.each { |r| @@regions[r.region_name] = Proc.new { listAZs(region: r.region_name, credentials: credentials) } } rescue ::Aws::EC2::Errors::AuthFailure => e MU.log "Region #{r.region_name} throws #{e.message}, ignoring it", MU::ERR end } end regions = if us_only @@regions.keys.delete_if { |r| !r.match(/^us\-/) }.uniq else @@regions.keys.uniq end # XXX GovCloud doesn't show up if you query a commercial endpoint... that's # *probably* ok for most purposes? We can't call listAZs on it from out here # apparently, so getting around it is nontrivial # if !@@regions.has_key?("us-gov-west-1") # @@regions["us-gov-west-1"] = Proc.new { listAZs("us-gov-west-1") } # end regions.sort! { |a, b| val = a <=> b if a == myRegion val = -1 elsif b == myRegion val = 1 end val } regions end
Load some credentials for using the AWS
API @param name [String]: The name of the mu.yaml AWS
credential set to use. If not specified, will use the default credentials, and set the global Aws.config credentials to those. @return [Aws::Credentials]
# File modules/mu/providers/aws.rb, line 67 def self.loadCredentials(name = nil) gem 'aws-sdk-core' @@creds_loaded ||= {} if name.nil? return @@creds_loaded["#default"] if @@creds_loaded["#default"] else return @@creds_loaded[name] if @@creds_loaded[name] end cred_cfg = credConfig(name) if cred_cfg.nil? return nil end cred_obj = nil if cred_cfg['access_key'] and cred_cfg['access_secret'] and # access key and secret just sitting in mu.yaml !cred_cfg['access_key'].empty? and !cred_cfg['access_secret'].empty? cred_obj = Aws::Credentials.new( cred_cfg['access_key'], cred_cfg['access_secret'] ) if name.nil? # Aws.config = { # access_key_id: cred_cfg['access_key'], # secret_access_key: cred_cfg['access_secret'], # region: cred_cfg['region'] # } end elsif cred_cfg['credentials_file'] and !cred_cfg['credentials_file'].empty? # pull access key and secret from an awscli-style credentials file begin File.read(cred_cfg["credentials_file"]) # make sure it's there credfile = IniFile.load(cred_cfg["credentials_file"]) if !credfile.sections or credfile.sections.size == 0 raise ::IniFile::Error, "No AWS profiles found in #{cred_cfg["credentials_file"]}" end data = credfile.has_section?("default") ? credfile["default"] : credfile[credfile.sections.first] if data["aws_access_key_id"] and data["aws_secret_access_key"] cred_obj = Aws::Credentials.new( data['aws_access_key_id'], data['aws_secret_access_key'] ) if name.nil? # Aws.config = { # access_key_id: data['aws_access_key_id'], # secret_access_key: data['aws_secret_access_key'], # region: cred_cfg['region'] # } end else MU.log "AWS credentials in #{cred_cfg["credentials_file"]} specified, but is missing aws_access_key_id or aws_secret_access_key elements", MU::WARN end rescue IniFile::Error, Errno::ENOENT, Errno::EACCES => e MU.log "AWS credentials file #{cred_cfg["credentials_file"]} is missing or invalid", MU::WARN, details: e.message end elsif cred_cfg['credentials'] and !cred_cfg['credentials'].empty? # pull access key and secret from a vault begin vault, item = cred_cfg["credentials"].split(/:/) data = if !vault or !item raise MuError.new "AWS #{name} credentials field value '#{cred_cfg["credentials"]}' malformed, should be vaultname:itemname", details: cred_cfg else MU::Groomer::Chef.getSecret(vault: vault, item: item).to_h end if data and data["access_key"] and data["access_secret"] cred_obj = Aws::Credentials.new( data['access_key'], data['access_secret'] ) if name.nil? # Aws.config = { # access_key_id: data['access_key'], # secret_access_key: data['access_secret'], # region: cred_cfg['region'] # } end else raise MuError.new "AWS #{name} credentials vault:item #{cred_cfg["credentials"]} specified, but is missing access_key or access_secret elements", details: cred_cfg end rescue MU::Groomer::MuNoSuchSecret raise MuError.new "AWS #{name} credentials vault:item #{cred_cfg["credentials"]} specified, but does not exist", details: cred_cfg end end if !cred_obj and hosted? # assume we've got an IAM profile and hope for the best ENV.delete('AWS_ACCESS_KEY_ID') ENV.delete('AWS_SECRET_ACCESS_KEY') retries = 0 begin cred_obj = Aws::InstanceProfileCredentials.new if cred_obj.nil? retries += 1 MU.log "Failed to fetch AWS instance profile credentials, attempt #{retries.to_s}/10", MU::WARN sleep 3 end end while cred_obj.nil? and retries < 10 # if name.nil? # Aws.config = {region: ENV['EC2_REGION']} # end end if cred_obj.nil? MU.log "cred_obj is nil and hosted? says #{hosted?.to_s}", MU::WARN, details: name end if name.nil? @@creds_loaded["#default"] = cred_obj else @@creds_loaded[name] = cred_obj end cred_obj end
If we've configured AWS
as a provider, or are simply hosted in AWS
, decide what our default region is.
# File modules/mu/providers/aws.rb, line 276 def self.myRegion(credentials = nil, debug: false) loglevel = debug ? MU::NOTICE : MU::DEBUG if @@myRegion_var MU.log "AWS.myRegion: returning #{@@myRegion_var} from cache", loglevel return @@myRegion_var end MU.log "AWS.myRegion: credConfig", loglevel, details: credConfig MU.log "AWS.myRegion: hosted?", loglevel, details: hosted?.to_s MU.log "AWS.myRegion: ENV['EC2_REGION']", loglevel, details: ENV['EC2_REGION'] MU.log "AWS.myRegion: $MU_CFG['aws']", loglevel, details: $MU_CFG['aws'] if credConfig.nil? and !hosted? and !ENV['EC2_REGION'] MU.log "AWS.myRegion: nothing of use set, returning", loglevel return nil end if $MU_CFG and $MU_CFG['aws'] $MU_CFG['aws'].each_pair { |credset, cfg| MU.log "AWS.myRegion: #{credset} != #{credentials} ?", loglevel, details: cfg next if credentials and credset != credentials MU.log "AWS.myRegion: validating credset #{credset}", loglevel, details: cfg next if !cfg['region'] MU.log "AWS.myRegion: validation response", loglevel, details: validate_region(cfg['region'], credentials: credset) if (cfg['default'] or !@@myRegion_var or $MU_CFG['aws'].size == 1) and validate_region(cfg['region'], credentials: credset) MU.log "AWS.myRegion: liking this set", loglevel, details: cfg @@myRegion_var = cfg['region'] break if cfg['default'] or credentials end } elsif ENV.has_key?("EC2_REGION") and !ENV['EC2_REGION'].empty? and validate_region(ENV['EC2_REGION']) and ( (ENV.has_key?("AWS_SECRET_ACCESS_KEY") and ENV.has_key?("AWS_SECRET_ACCESS_KEY") ) or (Aws.config['access_key'] and Aws.config['access_secret']) ) # Make sure this string is valid by way of the API MU.log "AWS.myRegion: using ENV", loglevel, details: ENV @@myRegion_var = ENV['EC2_REGION'] end if hosted? and !@@myRegion_var # hacky, but useful in a pinch (and if we're hosted in AWS) az_str = MU::Cloud::AWS.getAWSMetaData("placement/availability-zone") MU.log "AWS.myRegion: using hosted", loglevel, details: az_str @@myRegion_var = az_str.sub(/[a-z]$/i, "") if az_str end if credConfig and credConfig["region"] @@myRegion_var ||= credConfig["region"] end @@myRegion_var end
If we reside in this cloud, return the VPC
in which we, the Mu Master
, reside. @return [MU::Cloud::VPC]
# File modules/mu/providers/aws.rb, line 263 def self.myVPCObj return @@myVPCObj if @@myVPCObj return nil if !hosted? instance = MU.myCloudDescriptor return nil if !instance or !instance.vpc_id vpc = MU::MommaCat.findStray("AWS", "vpc", cloud_id: instance.vpc_id, dummy_ok: true, no_deploy_search: true) return nil if vpc.nil? or vpc.size == 0 @@myVPCObj = vpc.first @@myVPCObj end
Given a domain name and an ACM or IAM certificate identifier, sort out whether the domain name is “covered” by the certificate @param name [String] @param cert_id [String] @return [Boolean]
# File modules/mu/providers/aws.rb, line 1033 def self.nameMatchesCertificate(name, cert_id) _id, domains = findSSLCertificate(id: cert_id) return false if !domains domains.each { |dom| if dom == name or (dom =~ /^\*/ and name =~ /.*#{Regexp.quote(dom[1..-1])}/) return true end } false end
Punch AWS
security group holes for client nodes to talk back to us, the Mu Master
, if we're in AWS
. @return [void]
# File modules/mu/providers/aws.rb, line 1434 def self.openFirewallForClients require "aws-sdk-ec2" MU::Cloud.resourceClass("AWS", :FirewallRule) begin if File.exist?(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb") ::Chef::Config.from_file(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb") end ::Chef::Config[:environment] = MU.environment rescue LoadError # XXX why is Chef here end # This is the set of (TCP) ports we're opening to clients. We assume that # we can and and remove these without impacting anything a human has # created. my_ports = [10514] my_instance_id = MU::Cloud::AWS.getAWSMetaData("instance-id") my_client_sg_name = "Mu Client Rules for #{MU.mu_public_ip}" my_sgs = Array.new MU.setVar("curRegion", myRegion) if !myRegion.nil? MU.myCloudDescriptor.security_groups.each { |sg| my_sgs << sg.group_id } resp = MU::Cloud::AWS.ec2.describe_security_groups( filters: [ {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}, {name: "tag:Name", values: [my_client_sg_name]} ] ) if resp.nil? or resp.security_groups.nil? or resp.security_groups.size == 0 if MU.myCloudDescriptor.vpc_id.nil? sg_id = my_sgs.first resp = MU::Cloud::AWS.ec2.describe_security_groups(group_ids: [sg_id]) group = resp.security_groups.first MU.log "We don't have a security group named '#{my_client_sg_name}' available, and we are in EC2 Classic and so cannot create a new group. Defaulting to #{group.group_name}.", MU::NOTICE else group = MU::Cloud::AWS.ec2.create_security_group( group_name: my_client_sg_name, description: my_client_sg_name, vpc_id: MU.myCloudDescriptor.vpc_id ) sg_id = group.group_id my_sgs << sg_id MU::Cloud::AWS.createTag sg_id, "Name", my_client_sg_name MU::Cloud::AWS.createTag sg_id, "MU-MASTER-IP", MU.mu_public_ip MU::Cloud::AWS.ec2.modify_instance_attribute( instance_id: my_instance_id, groups: my_sgs ) end elsif resp.security_groups.size == 1 sg_id = resp.security_groups.first.group_id resp = MU::Cloud::AWS.ec2.describe_security_groups(group_ids: [sg_id]) group = resp.security_groups.first else MU.log "Found more than one security group named #{my_client_sg_name}, aborting", MU::ERR exit 1 end if !my_sgs.include?(sg_id) my_sgs << sg_id MU.log "Associating #{my_client_sg_name} with #{MU.myInstanceId}", MU::NOTICE MU::Cloud::AWS.ec2.modify_instance_attribute( instance_id: MU.myInstanceId, groups: my_sgs ) end begin MU.log "Using AWS Security Group '#{group.group_name}' (#{sg_id})" rescue NoMethodError MU.log "Using AWS Security Group #{sg_id}" end allow_ips = ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"] MU::MommaCat.listAllNodes.values.each { |data| next if data.nil? or !data.is_a?(Hash) ["public_ip_address"].each { |key| if data.has_key?(key) and !data[key].nil? and !data[key].empty? allow_ips << data[key] + "/32" end } } allow_ips.uniq! @syslog_port_semaphore.synchronize { my_ports.each { |port| begin group.ip_permissions.each { |rule| if rule.ip_protocol == "tcp" and rule.from_port == port and rule.to_port == port MU.log "Revoking old rules for port #{port.to_s} from #{sg_id}", MU::NOTICE begin MU::Cloud::AWS.ec2(region: myRegion).revoke_security_group_ingress( group_id: sg_id, ip_permissions: [ { ip_protocol: "tcp", from_port: port, to_port: port, ip_ranges: MU.structToHash(rule.ip_ranges) } ] ) rescue Aws::EC2::Errors::InvalidPermissionNotFound MU.log "Permission disappeared from #{sg_id} (port #{port.to_s}) before I could remove it", MU::WARN, details: MU.structToHash(rule.ip_ranges) end end } rescue NoMethodError # XXX this is ok end MU.log "Adding current IP list to allow rule for port #{port.to_s} in #{sg_id}", details: allow_ips allow_ips_cidr = [] allow_ips.each { |cidr| allow_ips_cidr << {"cidr_ip" => cidr} } begin MU::Cloud::AWS.ec2(region: myRegion).authorize_security_group_ingress( group_id: sg_id, ip_permissions: [ { ip_protocol: "tcp", from_port: 10514, to_port: 10514, ip_ranges: allow_ips_cidr } ] ) rescue Aws::EC2::Errors::InvalidPermissionDuplicate => e MU.log "Got #{e.inspect} in MU::Cloud::AWS.openFirewallForClients", MU::WARN, details: allow_ips_cidr end } } end
Amazon's Organizations API
# File modules/mu/providers/aws.rb, line 1354 def self.orgs(credentials: nil) @@organizations_api ||= {} # XXX org api doesn't seem to work in many regions @@organizations_api[credentials] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "Organizations", credentials: credentials, region: "us-east-1") @@organizations_api[credentials] end
Amazon's Pricing API
# File modules/mu/providers/aws.rb, line 1298 def self.pricing(region: MU.curRegion, credentials: nil) region ||= myRegion @@pricing_api[credentials] ||= {} @@pricing_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "Pricing", region: region, credentials: credentials) @@pricing_api[credentials][region] end
Amazon's RDS API
# File modules/mu/providers/aws.rb, line 1129 def self.rds(region: MU.curRegion, credentials: nil) region ||= myRegion @@rds_api[credentials] ||= {} @@rds_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "RDS", region: region, credentials: credentials) @@rds_api[credentials][region] end
Map our own idea of what a block device is called back to whatever AWS
and the operating system decided on amongst themselves. This currently exists to map generic “xvd” style names back to real NVMe devices. @param dev [String]
# File modules/mu/providers/aws.rb, line 501 def self.realDevicePath(dev) return dev if !hosted? value = nil should_retry = Proc.new { !value and MU::Master.nvme? } MU.retrier(loop_if: should_retry, wait: 5, max: 6) { map = attachedNVMeDisks value = if map[dev] map[dev] elsif map[dev.gsub(/.*?\//, '')] map[dev.gsub(/.*?\//, '')] else dev # be nice to actually handle this too end } value end
@param resources [Array<String>]: The cloud provider identifier of the resource to untag @param key [String]: The name of the tag to remove @param value [String]: The value of the tag to remove @param region [String]: The cloud provider region
# File modules/mu/providers/aws.rb, line 342 def self.removeTag(key, value, resources = [], region: myRegion) MU::Cloud::AWS.ec2(region: region).delete_tags( resources: resources, tags: [ { key: key, value: value } ] ) end
Any cloud-specific instance methods we require our resource implementations to have, above and beyond the ones specified by {MU::Cloud} @return [Array<Symbol>]
# File modules/mu/providers/aws.rb, line 189 def self.required_instance_methods [:arn] end
Given a {MU::Config::Ref} block for an IAM or ACM SSL certificate, look up and validate the specified certificate. This is intended to be invoked from resource implementations' validateConfig
methods. @param certblock [Hash,MU::Config::Ref]: @param region [String]: Default region to use when looking up the certificate, if its configuration block does not specify any @param credentials [String]: Default credentials to use when looking up the certificate, if its configuration block does not specify any @return [Boolean]
# File modules/mu/providers/aws.rb, line 1052 def self.resolveSSLCertificate(certblock, region: nil, credentials: nil) return false if !certblock ok = true certblock['region'] ||= region if !certblock['id'] certblock['credentials'] ||= credentials cert_arn, cert_domains = MU::Cloud::AWS.findSSLCertificate( name: certblock["name"], id: certblock["id"], region: certblock['region'], credentials: certblock['credentials'] ) if cert_arn certblock['id'] ||= cert_arn end ['region', 'credentials'].each { |field| certblock.delete(field) if certblock[field].nil? } [cert_arn, cert_domains] end
A hook that is always called just before any of the instance method of our resource implementations gets invoked, so that we can ensure that repetitive setup tasks (like resolving :resource_group
for Azure
resources) have always been done. @param cloudobj [MU::Cloud] @param _deploy [MU::MommaCat]
# File modules/mu/providers/aws.rb, line 53 def self.resourceInitHook(cloudobj, _deploy) class << self attr_reader :cloudformation_data attr_reader :region end return if !cloudobj cloudobj.instance_variable_set(:@cloudformation_data, {}) cloudobj.instance_variable_set(:@region, cloudobj.config['region']) end
Amazon's Route53 API
# File modules/mu/providers/aws.rb, line 1123 def self.route53(credentials: nil) @@route53_api[credentials] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "Route53", credentials: credentials) @@route53_api[credentials] end
Amazon's S3 API
# File modules/mu/providers/aws.rb, line 1145 def self.s3(region: MU.curRegion, credentials: nil) region ||= myRegion @@s3_api[credentials] ||= {} @@s3_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "S3", region: region, credentials: credentials) @@s3_api[credentials][region] end
Amazon's SNS API
# File modules/mu/providers/aws.rb, line 1218 def self.sns(region: MU.curRegion, credentials: nil) region ||= myRegion @@sns_api[credentials] ||= {} @@sns_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "SNS", region: region, credentials: credentials) @@sns_api[credentials][region] end
Amazon's SQS API
# File modules/mu/providers/aws.rb, line 1226 def self.sqs(region: MU.curRegion, credentials: nil) region ||= myRegion @@sqs_api[credentials] ||= {} @@sqs_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "SQS", region: region, credentials: credentials) @@sqs_api[credentials][region] end
Amazon's Simple Systems Manager API
# File modules/mu/providers/aws.rb, line 1306 def self.ssm(region: MU.curRegion, credentials: nil) region ||= myRegion @@ssm_api[credentials] ||= {} @@ssm_api[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "SSM", region: region, credentials: credentials) @@ssm_api[credentials][region] end
Given an AWS
region, check the API to make sure it's a valid one @param r [String] @return [String]
# File modules/mu/providers/aws.rb, line 196 def self.validate_region(r, credentials: nil) require "aws-sdk-ec2" begin MU::Cloud::AWS.ec2(region: r, credentials: credentials).describe_availability_zones.availability_zones.first.region_name rescue ::Aws::EC2::Errors::UnauthorizedOperation => e MU.log "Got '#{e.message}' trying to validate region #{r} (hosted: #{hosted?.to_s})", MU::ERR, details: loadCredentials(credentials) raise MuError, "Got '#{e.message}' trying to validate region #{r} with credentials #{credentials ? credentials : "<default>"} (hosted: #{hosted?.to_s})" end end
Is this a “real” cloud provider, or a stub like CloudFormation
?
# File modules/mu/providers/aws.rb, line 36 def self.virtual? false end
Amazon's Web Application Firewall API (Regional, for ALBs et al)
# File modules/mu/providers/aws.rb, line 1178 def self.waf(region: MU.curRegion, credentials: nil) region ||= myRegion @@waf[credentials] ||= {} @@waf[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "WAFRegional", region: region, credentials: credentials) @@waf[credentials][region] end
Amazon's Web Application Firewall API (Global, for CloudFront et al)
# File modules/mu/providers/aws.rb, line 1169 def self.wafglobal(region: MU.curRegion, credentials: nil) region ||= myRegion @@wafglobal_api[credentials] ||= {} @@wafglobal[credentials][region] ||= MU::Cloud::AWS::AmazonEndpoint.new(api: "WAF", region: region, credentials: credentials) @@wafglobal[credentials][region] end
Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it @param deploy_id [String]: The deploy for which we're writing the secret @param value [String]: The contents of the secret
# File modules/mu/providers/aws.rb, line 414 def self.writeDeploySecret(deploy, value, name = nil, credentials: nil) require "aws-sdk-s3" name ||= deploy.deploy_id+"-secret" begin MU.log "Writing #{name} to S3 bucket #{adminBucketName(credentials)}" MU::Cloud::AWS.s3(region: myRegion, credentials: credentials).put_object( acl: "private", bucket: adminBucketName(credentials), key: name, body: value ) rescue Aws::S3::Errors => e raise MU::MommaCat::DeployInitializeError, "Got #{e.inspect} trying to write #{name} to #{adminBucketName(credentials)}" end end