module StackBuilder

Constants

GEM_PATH
GEM_PATH_RUBY_VERSION
LAMBDA_HANDLER_FILE_NAME
RUBY_VERSION
S3Client

Attributes

api_gateway_deployment[RW]
api_gateway_id[RW]
app_dir[RW]
app_name[RW]
app_path[RW]
hidden_dir[RW]
lambda_handler_s3_key[RW]
lambda_handler_s3_object_version[RW]
lambda_handlers[RW]
lambda_policies[RW]
s3_bucket[RW]
stack[RW]
stack_opts[RW]

Public Instance Methods

add_api_gateway() click to toggle source

gateway

# File lib/modulator/stack/builder.rb, line 92
def add_api_gateway
  self.api_gateway_id = 'ApiGateway'
  stack.add(api_gateway_id, Humidifier::ApiGateway::RestApi.new(name: app_name, description: app_name + ' API'))
end
add_api_gateway_deployment() click to toggle source

gateway deployment

# File lib/modulator/stack/builder.rb, line 98
def add_api_gateway_deployment
  self.api_gateway_deployment = Humidifier::ApiGateway::Deployment.new(
    rest_api_id: Humidifier.ref(api_gateway_id),
    stage_name: Humidifier.ref("ApiGatewayStageName")
  )
  stack.add('ApiGatewayDeployment', api_gateway_deployment)
  stack.add_output('ApiGatewayInvokeURL',
      value: Humidifier.fn.sub("https://${#{api_gateway_id}}.execute-api.${AWS::Region}.amazonaws.com/${ApiGatewayStageName}"),
      description: 'API root url',
      export_name: app_name + 'RootUrl'
  )
  api_gateway_deployment.depends_on = []
end
add_api_gateway_resources(gateway:, lambda:) click to toggle source

gateway method

# File lib/modulator/stack/builder.rb, line 195
def add_api_gateway_resources(gateway:, lambda:)

  # example: calculator/algebra/:x/:y/sum -> module name, args, method name
  path = gateway[:path].split('/')

  # root resource
  root_resource = path.shift
  stack.add(root_resource.camelize, Humidifier::ApiGateway::Resource.new(
      rest_api_id: Humidifier.ref(api_gateway_id),
      parent_id: Humidifier.fn.get_att(["ApiGateway", "RootResourceId"]),
      path_part: root_resource
    )
  )

  # args and method name are nested resources
  parent_resource = root_resource.camelize
  path.each do |fragment|
    if fragment.start_with?(':')
      fragment = fragment[1..-1]
      dynamic_fragment = "{#{fragment}}"
    end
    stack.add(parent_resource + fragment.camelize, Humidifier::ApiGateway::Resource.new(
        rest_api_id: Humidifier.ref(api_gateway_id),
        parent_id: Humidifier.ref(parent_resource),
        path_part: dynamic_fragment || fragment
      )
    )
    parent_resource = parent_resource + fragment.camelize
  end

  # attach lambda to last resource
  id = 'EndpointFor' + (gateway[:path].gsub(':', '').gsub('/', '_')).camelize
  stack.add(id, Humidifier::ApiGateway::Method.new(
      authorization_type: 'NONE',
      http_method: gateway[:verb].to_s.upcase,
      integration: {
        integration_http_method: 'POST',
        type: "AWS_PROXY",
        uri: Humidifier.fn.sub([
          "arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31/functions/${lambdaArn}/invocations",
          'lambdaArn' => Humidifier.fn.get_att([lambda, 'Arn'])
        ])
      },
      rest_api_id: Humidifier.ref(api_gateway_id),
      resource_id: Humidifier.ref(parent_resource) # last evaluated resource
    )
  )

  # deployment depends on each endpoint
  api_gateway_deployment.depends_on << id
end
add_generic_lambda(gateway: {}, mod: {}, wrapper: {}, env: {}, settings: {}) click to toggle source

generic lambda function for gateway

# File lib/modulator/stack/builder.rb, line 127
def add_generic_lambda(gateway: {}, mod: {}, wrapper: {}, env: {}, settings: {})
  lambda_config = {}
  name_parts = mod[:name].split('::')
  {gateway: gateway, module: mod, wrapper: wrapper}.each do |env_group_prefix, env_group|
    env_group.each{|env_key, env_value| lambda_config["#{env_group_prefix}_#{env_key}"] = env_value}
  end
  env_vars = env
      .reduce({}){|env_as_string, (k, v)| env_as_string.update(k.to_s => v.to_s)}
      .merge(lambda_config)
      .merge(
        'GEM_PATH' => GEM_PATH,
        'app_dir'  => app_dir,
        'app_env'  => Humidifier.ref('AppEnvironment')
      )

  lambda_resource = generate_lambda_resource(
    description: "Lambda for #{mod[:name]}.#{mod[:method]}",
    function_name: [app_name, name_parts, mod[:method]].flatten.join('-').dasherize,
    handler: "#{LAMBDA_HANDLER_FILE_NAME}.AwsLambdaHandler.call",
    s3_key: LAMBDA_HANDLER_FILE_NAME + '.rb.zip',
    env_vars: env_vars,
    role: Humidifier.fn.get_att(['LambdaRole', 'Arn']),
    settings: settings,
    layers: [Humidifier.ref(app_name + 'Layer'), Humidifier.ref(app_name + 'GemsLayer')]
  )

  # add to stack
  ['Lambda', name_parts, mod[:method].capitalize].join.tap do |id|
    stack.add(id, lambda_resource)
    stack.add_lambda_invoke_permission(id: id, gateway: gateway)
  end
end
add_lambda(handler:, env: {}, settings: {}) click to toggle source

custom lambda function

# File lib/modulator/stack/builder.rb, line 113
def add_lambda(handler:, env: {}, settings: {})
  lambda_resource = generate_lambda_resource(
    description: "Lambda for #{handler}",
    function_name: ([app_name] << handler.split('.')).flatten.join('-').dasherize,
    handler: handler,
    s3_key: lambda_handler_s3_key,
    env_vars: env.merge('app_env' => Humidifier.ref('AppEnvironment')),
    role: Humidifier.fn.get_att(['LambdaRole', 'Arn']),
    settings: settings
  )
  stack.add(handler.gsub('.', '_').camelize, lambda_resource)
end
add_lambda_endpoint(**opts) click to toggle source
# File lib/modulator/stack/builder.rb, line 86
def add_lambda_endpoint(**opts) # gateway:, mod:, wrapper: {}, env: {}, settings: {}
  # add api resources and its lambda
  stack.add_api_gateway_resources(gateway: opts[:gateway], lambda: stack.add_generic_lambda(opts))
end
add_lambda_invoke_permission(id:, gateway:) click to toggle source

invoke permission

# File lib/modulator/stack/builder.rb, line 180
def add_lambda_invoke_permission(id:, gateway:)
  arn_path_matcher = gateway[:path].split('/').each_with_object([]) do |fragment, matcher|
    fragment = '*' if fragment.start_with?(':')
    matcher << fragment
  end.join('/')
  stack.add(id + 'InvokePermission' , Humidifier::Lambda::Permission.new(
      action: "lambda:InvokeFunction",
      function_name: Humidifier.fn.get_att([id, 'Arn']),
      principal: "apigateway.amazonaws.com",
      source_arn: Humidifier.fn.sub("arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}:${#{api_gateway_id}}/*/#{gateway[:verb]}/#{arn_path_matcher}")
    )
  )
end
add_layer(name:, description:, s3_key:, s3_object_version:) click to toggle source

add layer

# File lib/modulator/stack/uploader.rb, line 163
def add_layer(name:, description:, s3_key:, s3_object_version:)
  stack.add(name + 'Layer', Humidifier::Lambda::LayerVersion.new(
      compatible_runtimes: [RUBY_VERSION],
      layer_name: name,
      description: description,
      content: {
        s3_bucket: s3_bucket,
        s3_key: s3_key,
        s3_object_version: s3_object_version
      }
    )
  )
end
generate_lambda_resource(description:, function_name:, handler:, s3_key:, env_vars:, role:, settings:, layers: []) click to toggle source
# File lib/modulator/stack/builder.rb, line 160
def generate_lambda_resource(description:, function_name:, handler:, s3_key:, env_vars:, role:, settings:, layers: [])
  lambda_function = Humidifier::Lambda::Function.new(
    description: description,
    function_name: function_name,
    handler: handler,
    environment: {variables: env_vars},
    role: role,
    timeout: settings[:timeout] || stack_opts[:timeout] || 15,
    memory_size: settings[:memory_size] || stack_opts[:memory_size] || 128,
    runtime: RUBY_VERSION,
    code: {
      s3_bucket: s3_bucket,
      s3_key: s3_key,
      s3_object_version: lambda_handler_s3_object_version
    },
    layers: layers
  )
end
init(app_name:, s3_bucket:, **stack_opts) click to toggle source
# File lib/modulator/stack/builder.rb, line 21
def init(app_name:, s3_bucket:, **stack_opts)
  puts 'Initializing stack'
  @app_name   = app_name.camelize
  @s3_bucket  = s3_bucket
  @app_path   = Pathname.getwd
  @app_dir    = app_path.basename.to_s
  @hidden_dir = '.modulator'
  @stack_opts = stack_opts
  @lambda_handlers = stack_opts[:lambda_handlers] || []
  @lambda_policies = Array(stack_opts[:lambda_policies]) << :cloudwatch

  # create hidden dir for build artifacts
  app_path.join(hidden_dir).mkpath

  # init stack instance
  self.stack = Humidifier::Stack.new(name: app_name, aws_template_format_version: '2010-09-09')

  # app environment -  test, development, production ...
  app_envs = stack_opts[:app_envs] || ['development']
  stack.add_parameter('AppEnvironment', description: 'Application environment', type: 'String', allowed_values: app_envs, constraint_description: "Must be one of #{app_envs.join(', ')}")

  if lambda_handlers.empty?
    # api stage
    stack.add_parameter('ApiGatewayStageName', description: 'Gateway deployment stage', type: 'String', default: 'v1')

    # add gateway
    stack.add_api_gateway
    stack.add_api_gateway_deployment
  end

  # add role
  stack.add_lambda_iam_role

  # add policies to role
  stack.lambda_policies.each do |policy|
    stack.add_policy(policy) if policy.is_a?(Symbol)
    stack.add_policy(policy[:name], **policy) if policy.is_a?(Hash)
  end

  # simple lambda app
  if lambda_handlers.any?
    stack.upload_lambda_files
    lambda_handlers.each do |handler|
      stack.add_lambda(handler: handler, env: stack_opts[:env] || {}, settings: stack_opts[:settings] || {})
    end
  else
    # upload handlers and layers
    stack.upload_files
  end

  # return humidifier instance
  stack
end
upload_app_layer(sub_dirs: 'ruby/lib', add_layer_to_stack: true) click to toggle source
# File lib/modulator/stack/uploader.rb, line 113
def upload_app_layer(sub_dirs: 'ruby/lib', add_layer_to_stack: true)
  zip_file_name = app_dir + '.zip'
  app_zip_path  = app_path.join(hidden_dir, zip_file_name)

  # copy app code to ruby/lib in outside temp dir
  temp_dir_name = '.modulator_temp'
  temp_sub_dirs = sub_dirs
  temp_path     = app_path.parent.join(temp_dir_name)
  temp_path.join(temp_sub_dirs).mkpath
  FileUtils.copy_entry app_path, temp_path.join(temp_sub_dirs)

  # calculate checksum for app folder
  checksum_path = app_path.join(hidden_dir, 'app_checksum')
  old_checksum  = (checksum_path.read rescue nil)
  new_checksum  = Utils.checksum(app_path)

  if old_checksum != new_checksum
    puts '- uploading app files'
    checksum_path.write(new_checksum)
    ZipFileGenerator.new(temp_path, app_zip_path).write
    # upload zipped file
    app_layer = S3Client.put_object(
      bucket: s3_bucket,
      key: zip_file_name,
      body: app_zip_path.read
    )
    # delete zipped file
    app_zip_path.delete
  else
    puts '- using existing app files'
    app_layer = S3Client.get_object(bucket: s3_bucket, key: zip_file_name)
  end

  # delete temp dir
  FileUtils.remove_dir(temp_path)

  if add_layer_to_stack
    add_layer(
      name: app_name,
      description: "App source. MD5: #{new_checksum}",
      s3_key: zip_file_name,
      s3_object_version: app_layer.version_id
    )
  else # for simple lambda apps
    self.lambda_handler_s3_key = zip_file_name
    self.lambda_handler_s3_object_version = app_layer.version_id
  end
end
upload_files() click to toggle source
# File lib/modulator/stack/builder.rb, line 75
def upload_files
  if stack_opts[:skip_upload]
    puts 'Skipping upload'
    return
  end
  stack.upload_generic_lambda_handler
  puts 'Generating layers'
  stack.upload_gems_layer
  stack.upload_app_layer
end
upload_gems_layer() click to toggle source
# File lib/modulator/stack/uploader.rb, line 64
def upload_gems_layer
  if !app_path.join('Gemfile').exist?
    puts '- no Gemfile detected'
    return
  end

  # calculate Gemfile checksum
  checksum_path = app_path.join(hidden_dir, 'gemfile_checksum')
  old_checksum  = (checksum_path.read rescue nil)
  new_checksum  = Digest::MD5.hexdigest(File.read(app_path.join('Gemfile.lock')))

  zip_file_name = app_dir + '_gems.zip'
  gems_path     = app_path.join(hidden_dir, 'gems')
  gems_zip_path = app_path.join(hidden_dir, zip_file_name)

  if old_checksum != new_checksum
    puts '- uploading gems layer'
    checksum_path.write(new_checksum)

    # bundle gems
    Bundler.with_clean_env do
      Dir.chdir(app_path) do
        `bundle install --path=./#{hidden_dir}/gems --clean --without development`
      end
    end
    ZipFileGenerator.new(gems_path, gems_zip_path).write

    # upload zipped file
    gem_layer = S3Client.put_object(
      bucket: s3_bucket,
      key: zip_file_name,
      body: gems_zip_path.read
    )
    # delete zipped file
    FileUtils.remove_dir(gems_path)
    gems_zip_path.delete
  else
    puts '- using existing gems layer'
    gem_layer = S3Client.get_object(bucket: s3_bucket, key: zip_file_name)
  end

  add_layer(
    name: app_name + 'Gems',
    description: "App gems",
    s3_key: zip_file_name,
    s3_object_version: gem_layer.version_id
  )
end
upload_generic_lambda_handler() click to toggle source

generic handler for all lambda

# File lib/modulator/stack/uploader.rb, line 26
  def upload_generic_lambda_handler
    lambda_handler_key = LAMBDA_HANDLER_FILE_NAME + '.rb.zip'
    modulator_handler_source = Pathname.new(__FILE__).dirname.parent.join('lambda_handler.rb').read
    source = <<~SOURCE
      # DO NOT EDIT THIS FILE

      #{modulator_handler_source}
      Dir.chdir('/opt/ruby/lib')
    SOURCE

    existing_handler = S3Client.get_object(
      bucket: s3_bucket,
      key: lambda_handler_key
    ) rescue false # not found

    if existing_handler
      existing_source = Zip::InputStream.open(existing_handler.body) do |zip_file|
        zip_file.get_next_entry
        zip_file.read
      end
      self.lambda_handler_s3_object_version = existing_handler.version_id
    end

    if existing_source != source
      puts '- uploading generic lambda handler'
      source_zip_file = Zip::OutputStream.write_buffer do |zip|
        zip.put_next_entry LAMBDA_HANDLER_FILE_NAME + '.rb'
        zip.print source
      end
      new_handler = S3Client.put_object(
        bucket: s3_bucket,
        key: lambda_handler_key,
        body: source_zip_file.tap(&:rewind).read
      )
      self.lambda_handler_s3_object_version = new_handler.version_id
    end
  end
upload_lambda_files() click to toggle source

bundle gems and upload all for simple lambda apps

# File lib/modulator/stack/uploader.rb, line 11
def upload_lambda_files
  puts '- bundling dependencies'
  Bundler.with_clean_env do
    Dir.chdir(app_path) do
      `bundle install`
      `bundle install --deployment --without development`
    end
  end
  FileUtils.remove_dir(app_path.join("vendor/bundle/ruby/#{GEM_PATH_RUBY_VERSION}/cache")) # remove cache dir
  upload_app_layer(sub_dirs: '', add_layer_to_stack: false) # reuse layer upload
  FileUtils.remove_dir(app_path.join('.bundle'))
  FileUtils.remove_dir(app_path.join('vendor'))
end