class SimplyGenius::Atmos::TerraformExecutor

Public Class Methods

new(process_env: ENV) click to toggle source
# File lib/simplygenius/atmos/terraform_executor.rb, line 19
def initialize(process_env: ENV)
  @process_env = process_env
  recipe_config_path = "recipes.#{Atmos.config.working_group}"
  @recipes = Array(Atmos.config[recipe_config_path])
  if @recipes.blank?
    logger.warn("Check your configuration, there are no recipes in '#{recipe_config_path}'")
  end
  @compat11 = Atmos.config['atmos.terraform.compat11'].to_s == "true"
end

Public Instance Methods

run(*terraform_args, skip_backend: false, skip_secrets: false, get_modules: false, output_io: nil) click to toggle source
# File lib/simplygenius/atmos/terraform_executor.rb, line 29
def run(*terraform_args, skip_backend: false, skip_secrets: false, get_modules: false, output_io: nil)
  setup_working_dir(skip_backend: skip_backend)

  if get_modules
    logger.debug("Getting modules")
    get_modules_io = StringIO.new
    begin
      execute("get", output_io: get_modules_io)
    rescue TerraformExecutor::ProcessFailed => e
      logger.info(get_modules_io.string)
      raise
    end
  end

  return execute(*terraform_args, skip_secrets: skip_secrets, output_io: output_io)
end

Private Instance Methods

atmos_env() click to toggle source

TODO: Add ability to declare variables as well as set them. May need to inspect existing tf to find all declared vars so we don't double declare

# File lib/simplygenius/atmos/terraform_executor.rb, line 269
def atmos_env
  # A var value in the env is ignored if a variable declaration doesn't exist for it in a tf file.  Thus,
  # as a convenience to allow everything from atmos to be referenceable, we put everything from the atmos_config
  # in a homogenized hash named atmos_config which is declared by the atmos scaffolding.  For variables which are
  # declared, we also merge in atmos config with only the hash values homogenized (vs the entire map) so that hash
  # variables if declared in terraform can be managed from yml, set here and accessed from terraform
  #
  homogenized_config = homogenize_for_terraform(Atmos.config.to_h)
  var_hash = {
      all_env_names: Atmos.config.all_env_names,
      account_ids: Atmos.config.account_hash,
      atmos_config: homogenized_config
  }

  # Terraform > 0.12 can have more complex data types (deeply nested
  # maps), so if not enforcing 0.11 compatibility, we should preserve that
  # structure for vars, but still homogenize in compat mode.  This way,
  # assuming the var defined in tf source has constraints that match the
  # yml definition, then it will get populated correctly after all the
  # atmos niceties like merging env specific overrides into a default
  if @compat11
    homogenized_values = Hash[Atmos.config.to_h.collect {|k, v| [k, v.is_a?(Hash) ? homogenize_for_terraform(v) : v]}]
    var_hash = var_hash.merge(homogenized_values)
  else
    var_hash = var_hash.merge(Atmos.config.to_h)
  end

  env_hash = encode_tf_env(var_hash)

  # write out a file so users have some visibility into vars passed in -
  # mostly useful for debugging
  File.open(File.join(tf_recipes_dir, 'atmos-tfvars.env'), 'w') do |f|
    env_hash.each do |k, v|
      f.puts("#{k}='#{v}'")
    end
  end

  return env_hash
end
encode_tf_env(hash) click to toggle source
# File lib/simplygenius/atmos/terraform_executor.rb, line 238
def encode_tf_env(hash)
  result = {}
  hash.each do |k, v|
    result["TF_VAR_#{k}"] = encode_tf_env_value(v)
  end
  return result
end
encode_tf_env_value(v) click to toggle source
# File lib/simplygenius/atmos/terraform_executor.rb, line 220
def encode_tf_env_value(v)
  case v
  when nil
    @compat11 ? "" : JSON.generate(v)
  when Numeric, String, TrueClass, FalseClass
    v.to_s
  when Hash
    if @compat11
      hcl_hash =v.collect {|k, v| %Q("#{k.to_s}"="#{encode_tf_env_value(v)}") }.join(",")
      "{#{hcl_hash}}"
    else
      JSON.generate(v)
    end
  else
    JSON.generate(v)
  end
end
execute(*terraform_args, skip_secrets: false, output_io: nil) click to toggle source
# File lib/simplygenius/atmos/terraform_executor.rb, line 52
def execute(*terraform_args, skip_secrets: false, output_io: nil)
  cmd = tf_cmd(*terraform_args)
  logger.debug("Running terraform: #{cmd.join(' ')}")

  env = Hash[@process_env]
  env['ATMOS_ROOT'] = Atmos.config.root_dir
  env['ATMOS_CONFIG'] = Atmos.config.config_file
  if ! skip_secrets
    begin
      env = env.merge(secrets_env)
    rescue => e
      logger.debug("Secrets not available: #{e}")
    end
  end

  # TF 0.12 deprecates values for undeclared vars in an tfvars file, so
  # put it in env instead as they claim that to be the expected way to do
  # so and will continue to work
  # https://github.com/hashicorp/terraform/issues/19424
  env = env.merge(atmos_env)

  # lets tempfiles created by subprocesses be easily found by users
  env['TMPDIR'] = Atmos.config.tmp_dir

  # Lets terraform communicate back to atmos, e.g. for UI notifications
  ipc = Ipc.new(Atmos.config.tmp_dir)

  IO.pipe do |stdout, stdout_writer|
    IO.pipe do |stderr, stderr_writer|

      stdout_writer.sync = stderr_writer.sync = true

      stdout_filters = Atmos.config.plugin_manager.output_filters(:stdout, {process_env: @process_env, working_group: Atmos.config.working_group})
      stderr_filters = Atmos.config.plugin_manager.output_filters(:stderr, {process_env: @process_env, working_group: Atmos.config.working_group})

      stdout_thr = pipe_stream(stdout, output_io.nil? ? $stdout : output_io, &stdout_filters.filter_block)
      stderr_thr = pipe_stream(stderr, output_io.nil? ? $stderr : output_io, &stderr_filters.filter_block)
      status = nil
      ipc.listen do |sock_path|

        if Atmos.config['atmos.ipc.disable']
          # Using : as the command makes execution of ipc from the
          # terraform side a no-op in both cases of how we call it.  This
          # way, terraform execution continues to work when IPC is disabled
          # command = "$ATMOS_IPC_CLIENT <json_string>"
          # program = ["sh", "-c", "$ATMOS_IPC_CLIENT"]
          env['ATMOS_IPC_CLIENT'] = ":"
        else
          env['ATMOS_IPC_SOCK'] = sock_path
          env['ATMOS_IPC_CLIENT'] = ipc.generate_client_script
        end

        # Was unable to get piping to work with stdin for some reason.  It
        # worked in simple case, but started to fail when terraform config
        # got more extensive.  Thus, using spawn to redirect stdin from the
        # terminal direct to terraform, with IO.pipe to copy the outher
        # streams.  Maybe in the future we can completely disconnect stdin
        # and have atmos do the output parsing and stdin prompting
        pid = spawn(env, *cmd,
                    chdir: tf_recipes_dir,
                    :out=>stdout_writer, :err=> stderr_writer, :in => :in)

        logger.debug("Terraform started with pid #{pid}")
        begin
          _, status = Process.wait2(pid)
        rescue Interrupt
          logger.warn "Got SIGINT, sending to terraform pid=#{pid}"

          Process.kill("INT", pid)
          _, status = Process.wait2(pid)

          logger.debug "Completed signal cleanup"
          exit!(1)
        end

      end

      stdout_writer.close
      stderr_writer.close
      stdout_thr.join
      stderr_thr.join
      stdout_filters.close
      stderr_filters.close

      exitcode = status.exitstatus
      logger.debug("Terraform exited: #{exitcode}")
      if exitcode != 0
        raise ProcessFailed.new "Terraform exited with non-zero exit code: #{exitcode}"
      end

    end
  end

end
homogenize_encode(v) click to toggle source
# File lib/simplygenius/atmos/terraform_executor.rb, line 178
def homogenize_encode(v)
  case v
  when nil
    @compat11 ? "" : v
  else
    v
  end
end
homogenize_for_terraform(obj, prefix="") click to toggle source

terraform requires all values within a map to have the same type, so flatten the map - nested maps get expanded into the top level one, with their keys being appended with underscores, and lists get joined with “,” so we end up with a single hash with homogenous types

# File lib/simplygenius/atmos/terraform_executor.rb, line 192
def homogenize_for_terraform(obj, prefix="")
  if obj.is_a? Hash
    result = {}
    obj.each do |k, v|
      ho = homogenize_for_terraform(v, "#{prefix}#{k}_")
      if ho.is_a? Hash
        result = result.merge(ho)
      else
        result["#{prefix}#{k}"] = homogenize_encode(ho)
      end
    end
    return result
  elsif obj.is_a? Array
    result = []
    obj.each do |o|
      ho = homogenize_for_terraform(o, prefix)
      if ho.is_a? Hash
        result << ho.collect {|k, v| "#{k}=#{homogenize_encode(v)}"}.join(";")
      else
        result << homogenize_encode(ho)
      end
    end
    return result.join(",")
  else
    return homogenize_encode(obj)
  end
end
pipe_stream(src, dest, &block) click to toggle source
# File lib/simplygenius/atmos/terraform_executor.rb, line 348
def pipe_stream(src, dest, &block)
   Thread.new do
     block_size = 1024
     begin
       while data = src.readpartial(block_size)
         data = block.call(data, flushing: false) if block
         dest.write(data)
       end
     rescue IOError, EOFError => e
       logger.log_exception(e, "Stream failure", level: :debug)
     rescue Exception => e
       logger.log_exception(e, "Stream failure")
     ensure
       begin
         if block
           data = block.call('', flushing: true)
           dest.write(data)
         end
         dest.flush
       rescue IOError, EOFError => e
         logger.log_exception(e, "Stream failure while flushing", level: :debug)
       rescue Exception => e
         logger.log_exception(e, "Stream failure while flushing")
       end
     end
   end
end
secrets_env() click to toggle source
# File lib/simplygenius/atmos/terraform_executor.rb, line 255
def secrets_env
  # NOTE use an auto-deleting temp file if passing secrets through env ends
  # up being problematic
  # TODO fix the need for CC - TE calls for secrets which needs auth in
  # ENV, so kinda clunk to have to do both CC and pass the env in
  ClimateControl.modify(@process_env) do
    secrets = Atmos.config.provider.secret_manager.to_h
    env_secrets = encode_tf_env(secrets)
    return env_secrets
  end
end
setup_backend(skip_backend=false) click to toggle source
# File lib/simplygenius/atmos/terraform_executor.rb, line 154
def setup_backend(skip_backend=false)
  backend_file = File.join(tf_recipes_dir, 'atmos-backend.tf.json')
  backend_config = (Atmos.config["backend"] || {}).clone

  if backend_config.present? && ! skip_backend
    logger.debug("Writing out terraform state backend config")

    backend_type = backend_config.delete("type")

    backend = {
        "terraform" => {
            "backend" => {
                backend_type => backend_config
            }
        }
    }

    File.write(backend_file, JSON.pretty_generate(backend))
  else
    logger.debug("Clearing terraform state backend config")
    File.delete(backend_file) if File.exist?(backend_file)
  end
end
setup_working_dir(skip_backend: false) click to toggle source
# File lib/simplygenius/atmos/terraform_executor.rb, line 147
def setup_working_dir(skip_backend: false)
  clean_links
  link_support_dirs
  link_recipes
  setup_backend(skip_backend)
end
tf_cmd(*args) click to toggle source
# File lib/simplygenius/atmos/terraform_executor.rb, line 48
def tf_cmd(*args)
  ['terraform'] + args
end
tf_recipes_dir() click to toggle source
# File lib/simplygenius/atmos/terraform_executor.rb, line 246
def tf_recipes_dir
  @tf_recipes_dir ||= begin
    dir = File.join(Atmos.config.tf_working_dir, 'recipes')
    logger.debug("Tf recipes dir: #{dir}")
    mkdir_p(dir)
    dir
  end
end