class Pupistry::Artifact

Pupistry::Artifact

Attributes

checksum[RW]

All the functions needed for manipulating the artifats

Public Instance Methods

build_artifact() click to toggle source
# File lib/pupistry/artifact.rb, line 289
def build_artifact
  # r10k has done all the heavy lifting for us, we just need to generate a
  # tarball from the app_cache /puppetcode directory. There are some Ruby
  # native libraries, but really we might as well just use the native tools
  # since we don't want to do anything clever like in-memory assembly of
  # the file. Like r10k, if you want to convert to a nicely polished native
  # Ruby solution, patches welcome.

  $logger.info 'Creating artifact...'

  Dir.chdir($config['general']['app_cache']) do
    # Make sure there is a directory to write artifacts into
    FileUtils.mkdir_p('artifacts')

    # Build the tar file - we delibertly don't compress in a single step
    # so that we can grab the checksum, since checksum will always differ
    # post-compression.

    tar = Pupistry::Config.which_tar
    $logger.debug "Using tar at #{tar}"

    tar += " -c"
    tar += " --exclude '.git'"
    if Pupistry::HieraCrypt.is_enabled?
      # We want to exclude unencrypted hieradata (duh security) and also the node files (which aren't needed)
      tar += " --exclude 'hieradata'"
      tar += " --exclude 'hieracrypt/nodes'"
    else
      # Hieracrypt is disable, exclude any old out of date encrypted files
      tar += " --exclude 'hieracrypt/encrypted'"
    end
    tar += " -f artifacts/artifact.temp.tar puppetcode/*"

    unless system tar
      $logger.error 'Unable to create tarball'
      fail 'An unexpected error occured when executing tar'
    end

    # The checksum is important, we use it as our version for each artifact
    # so we can tell them apart in a unique way.
    @checksum = Digest::MD5.file($config['general']['app_cache'] + '/artifacts/artifact.temp.tar').hexdigest

    # Now we have the checksum, check if it's the same as any existing
    # artifacts. If so, drop out here, good to give feedback to the user
    # if nothing has changed since it's easy to forget to git push a single
    # module/change.

    if File.exist?($config['general']['app_cache'] + "/artifacts/manifest.#{@checksum}.yaml")
      $logger.error "This artifact version (#{@checksum}) has already been built, nothing todo."
      $logger.error "Did you remember to \"git push\" your module changes?"

      # TODO: Unfortunatly Hieracrypt breaks this, since the encrypted Hieradata is different
      # on every run, which results in the checksum always being different even if nothing in
      # the repo itself has changed. We need a proper fix for this at some stage, for now it's
      # covered in the readme notes for HieraCrypt as a flaw.

      # Cleanup temp file
      FileUtils.rm($config['general']['app_cache'] + '/artifacts/artifact.temp.tar')
      exit 0
    end

    # Compress the artifact now that we have taken it's checksum
    $logger.info 'Compressing artifact...'

    if system 'gzip artifacts/artifact.temp.tar'
    else
      $logger.error 'An unexpected error occured during compression of the artifact'
      fail 'An unexpected error occured during compression of the artifact'
    end
  end

  # We have the checksum, so we can now rename the artifact file
  FileUtils.mv($config['general']['app_cache'] + '/artifacts/artifact.temp.tar.gz',
               $config['general']['app_cache'] + "/artifacts/artifact.#{@checksum}.tar.gz")

  $logger.info 'Building manifest information for artifact...'

  # Create the manifest file, this is used by clients for pulling details about
  # the latest artifacts. We don't GPG sign here, but we do put in a placeholder.
  manifest = {
    'version'   => @checksum,
    'date'      => Time.new.inspect,
    'builduser' => ENV['USER'] || 'unlabled',
    'gpgsig'    => 'unsigned'
  }

  begin
    File.open("#{$config['general']['app_cache']}/artifacts/manifest.#{@checksum}.yaml", 'w') do |fh|
      fh.write YAML.dump(manifest)
    end
  rescue StandardError => e
    $logger.fatal 'Unexpected error when trying to write the manifest file'
    raise e
  end

  # This is the latest artifact, create some symlinks pointing the latest to it
  begin
    FileUtils.ln_s("manifest.#{@checksum}.yaml",
                   "#{$config['general']['app_cache']}/artifacts/manifest.latest.yaml",
                   force: true)
    FileUtils.ln_s("artifact.#{@checksum}.tar.gz",
                   "#{$config['general']['app_cache']}/artifacts/artifact.latest.tar.gz",
                   force: true)
  rescue StandardError => e
    $logger.fatal 'Something weird went really wrong trying to symlink the latest artifacts'
    raise e
  end

  $logger.info "New artifact version #{@checksum} ready for pushing"
end
clean_install() click to toggle source
# File lib/pupistry/artifact.rb, line 486
def clean_install
  # Cleanup the destination installation directory before we unpack the artifact
  # into it, otherwise long term we will end up with old deprecated files hanging
  # around.
  #
  # TODO: Do this smarter, we should track what files we drop in, and then remove
  # any that weren't touched. Need to avoid rsync and stick with native to make
  # support easier for weird/minimilistic distributions.

  if defined? $config['agent']['puppetcode'] # rubocop:disable Style/GuardClause
    if $config['agent']['puppetcode'].empty?
      $logger.error "You must configure a location for the agent's Puppet code to be deployed to"
      return false
    else
      $logger.debug "Cleaning up #{$config['agent']['puppetcode']} directory"

      if Dir.exist?($config['agent']['puppetcode'])
        FileUtils.rm_r Dir.glob($config['agent']['puppetcode'] + '/*'), secure: true
      else
        FileUtils.mkdir_p $config['agent']['puppetcode']
        FileUtils.chmod(0700, $config['agent']['puppetcode'])
      end

      return true
    end
  end
end
clean_unpack() click to toggle source
# File lib/pupistry/artifact.rb, line 514
def clean_unpack
  # Cleanup/remove any unpacked archive directories. Requires that the
  # checksum be set to the version to be purged.

  fail 'Application bug, trying to unpack no artifact' unless defined? @checksum

  if Dir.exist?($config['general']['app_cache'] + "/artifacts/unpacked.#{@checksum}/")
    $logger.debug "Cleaning up #{$config['general']['app_cache']}/artifacts/unpacked.#{@checksum}..."
    FileUtils.rm_r $config['general']['app_cache'] + "/artifacts/unpacked.#{@checksum}", secure: true
    return true
  else
    $logger.debug 'Nothing to cleanup (selected artifact is not currently unpacked)'
    return true
  end

  false
end
fetch_artifact() click to toggle source
# File lib/pupistry/artifact.rb, line 144
def fetch_artifact
  # Figure out which version to fetch (if not explicitly defined)
  if defined? @checksum
    $logger.debug "Downloading artifact version #{@checksum}"
  else
    @checksum = fetch_latest

    if defined? @checksum
      $logger.debug "Downloading latest artifact (#{@checksum})"
    else
      $logger.error 'There is not current artifact that can be fetched'
      return false
    end

  end

  # Make sure the download dir/cache exists
  FileUtils.mkdir_p $config['general']['app_cache'] + '/artifacts/' unless Dir.exist?($config['general']['app_cache'] + '/artifacts/')

  # Download files if they don't already exist
  if File.exist?($config['general']['app_cache'] + "/artifacts/manifest.#{@checksum}.yaml") &&
     File.exist?($config['general']['app_cache'] + "/artifacts/artifact.#{@checksum}.tar.gz")
    $logger.debug 'This artifact is already present, no download required.'
  else
    s3 = Pupistry::StorageAWS.new 'agent'
    s3.download "manifest.#{@checksum}.yaml", $config['general']['app_cache'] + "/artifacts/manifest.#{@checksum}.yaml"
    s3.download "artifact.#{@checksum}.tar.gz", $config['general']['app_cache'] + "/artifacts/artifact.#{@checksum}.tar.gz"
  end
end
fetch_current() click to toggle source
# File lib/pupistry/artifact.rb, line 109
def fetch_current
  # Fetch the latest on-disk YAML file and check the version metadata, used
  # to determine the latest artifact that has not yet been pushed to S3.
  # Returns the version.

  # Read the symlink information to get the latest version
  if File.exist?($config['general']['app_cache'] + '/artifacts/manifest.latest.yaml')
    manifest    = YAML.load(File.open($config['general']['app_cache'] + '/artifacts/manifest.latest.yaml'), safe: true, raise_on_unknown_tag: true)
    @checksum   = manifest['version']
  else
    $logger.error 'No artifact has been built yet. You need to run pupistry build first?'
    return false
  end
end
fetch_installed() click to toggle source
# File lib/pupistry/artifact.rb, line 124
def fetch_installed
  # Fetch the current version that is installed.

  # Make sure the Puppetcode install directory exists
  unless Dir.exist?($config['agent']['puppetcode'])
    $logger.warn "The destination path of #{$config['agent']['puppetcode']} does not appear to exist or is not readable"
    return false
  end

  # Look for a manifest file in the directory and read the version from it.
  if File.exist?($config['agent']['puppetcode'] + '/manifest.pupistry.yaml')
    manifest = YAML.load(File.open($config['agent']['puppetcode'] + '/manifest.pupistry.yaml'), safe: true, raise_on_unknown_tag: true)

    return manifest['version']
  else
    $logger.warn 'No current version installed'
    return false
  end
end
fetch_latest() click to toggle source
# File lib/pupistry/artifact.rb, line 70
def fetch_latest
  # Fetch the latest S3 YAML file and check the version metadata without writing
  # it to disk. Returns the version. Useful for quickly checking for updates :-)

  $logger.debug 'Checking latest artifact version...'

  s3        = Pupistry::StorageAWS.new 'agent'
  contents  = s3.download 'manifest.latest.yaml'

  if contents
    manifest = YAML.load(contents, safe: true, raise_on_unknown_tag: true)

    if defined? manifest['version']
      # We have a manifest version supplied, however since the manifest
      # isn't signed, there's risk of an exploited S3 bucket replacing
      # the version with injections designed to attack the shell commands
      # we call from Pupistry.
      #
      # Therefore we need to make sure the manifest version matches a
      # regex suitable for a checksum.

      if /^[A-Za-z0-9]{32}$/.match(manifest['version'])
        return manifest['version']
      else
        $logger.error 'Manifest version returned from S3 manifest.latest.yaml did not match expected regex of MD5.'
        $logger.error 'Possible bug or security incident, investigate with care!'
        $logger.error "Returned version string was: \"#{manifest['version']}\""
        exit 0
      end
    else
      return false
    end

  else
    # download did not work
    return false
  end
end
fetch_r10k() click to toggle source
# File lib/pupistry/artifact.rb, line 17
def fetch_r10k
  $logger.info 'Using r10k utility to fetch the latest Puppet code'

  unless defined? $config['build']['puppetcode']
    $logger.fatal 'You must configure the build:puppetcode config option in settings.yaml'
    fail 'Invalid Configuration'
  end

  # https://github.com/puppetlabs/r10k
  #
  # r10k does a fantastic job with all the git stuff and we want to use it
  # to download the Puppet code from all the git modules (based on following
  # the master one provided), then we can steal the Puppet code from the
  # artifact generated.
  #
  # TODO: We should re-write this to hook directly into r10k's libraries,
  # given that both Pupistry and r10k are Ruby, presumably it should be
  # doable and much more polished approach. For now the MVP is to just run
  # it via system, pull requests/patches to fix very welcome!

  # Build the r10k config to instruct it to use our cache path for storing
  # it's data and exporting the finished result.
  $logger.debug 'Generating an r10k configuration file...'
  r10k_config = {
    'cachedir' => "#{$config['general']['app_cache']}/r10kcache",
    'sources'  => {
      'puppet' => {
        'remote'  => $config['build']['puppetcode'],
        'basedir' => $config['general']['app_cache'] + '/puppetcode'
      }
    }
  }

  begin
    File.open("#{$config['general']['app_cache']}/r10kconfig.yaml", 'w') do |fh|
      fh.write YAML.dump(r10k_config)
    end
  rescue StandardError => e
    $logger.fatal 'Unexpected error when trying to write the r10k configuration file'
    raise e
  end

  # Execute R10k with the provided configuration
  $logger.debug 'Executing r10k'

  if system "r10k deploy environment -c #{$config['general']['app_cache']}/r10kconfig.yaml -pv debug"
    $logger.info 'r10k run completed'
  else
    $logger.error 'r10k run failed, unable to generate artifact'
    fail 'r10k run did not complete, unable to generate artifact'
  end
end
hieracrypt_decrypt() click to toggle source
# File lib/pupistry/artifact.rb, line 184
def hieracrypt_decrypt
  # Decrypt any encrypted Hieradata inside the currently unpacked artifact
  # before it gets copied to the installation location.

  if defined? @checksum
    Pupistry::HieraCrypt.decrypt_hieradata $config['general']['app_cache'] + "/artifacts/unpacked.#{@checksum}/puppetcode"
  else
    $logger.warn "Tried to request hieracrypt_decrypt on no artifact."
  end

end
hieracrypt_encrypt() click to toggle source
# File lib/pupistry/artifact.rb, line 174
def hieracrypt_encrypt
  # Stub function, since HieraCrypt has no association with the actual
  # artifact file, but rather the post-r10k checked data, it could be
  # invoked directly. However it's worth wrapping here incase we ever
  # do change this behavior.

  Pupistry::HieraCrypt.encrypt_hieradata

end
install() click to toggle source
# File lib/pupistry/artifact.rb, line 437
def install
  # Copy the unpacked artifact into the agent's configured location. Generally all the
  # heavy lifting is done by fetch_latest and unpack methods.

  # An application version must be specified
  fail 'Application bug, trying to install no artifact' unless defined? @checksum

  # Validate the artifact if GPG is enabled.
  if $config['general']['gpg_disable'] == true
    $logger.warn 'You have GPG validation *disabled*, whilst not critical it does weaken your security.'
    $logger.warn 'Skipping validation step...'
  else

    gpgsig = Pupistry::GPG.new @checksum

    unless gpgsig.artifact_verify
      $logger.fatal 'The GPG signature could not be validated for the artifact. This could be a bug, a file corruption or a POSSIBLE SECURITY ISSUE such as maliciously modified content.'
      fail 'Fatal unexpected error'
    end

  end

  # Make sure the artifact has been unpacked
  unless Dir.exist?($config['general']['app_cache'] + "/artifacts/unpacked.#{@checksum}")
    $logger.error "The unpacked directory expected for #{@checksum} does not appear to exist or is not readable"
    fail 'Fatal unexpected error'
  end

  # Purge any currently installed files in the directory. See clean_install
  # TODO: notes for how this could be improved.
  $logger.error 'Installation not proceeding due to issues cleaning/prepping destination dir' unless clean_install

  # Make sure the destination directory exists
  unless Dir.exist?($config['agent']['puppetcode'])
    $logger.error "The destination path of #{$config['agent']['puppetcode']} does not appear to exist or is not readable"
    fail 'Fatal unexpected error'
  end

  # Clone unpacked contents to the installation directory
  begin
    FileUtils.cp_r $config['general']['app_cache'] + "/artifacts/unpacked.#{@checksum}/puppetcode/.", $config['agent']['puppetcode']
    FileUtils.cp $config['general']['app_cache'] + "/artifacts/manifest.#{@checksum}.yaml", $config['agent']['puppetcode'] + '/manifest.pupistry.yaml'
    return true
  rescue
    $logger.fatal "An unexpected error occured when copying the unpacked artifact to #{$config['agent']['puppetcode']}"
    raise e
  end
end
push_artifact() click to toggle source
# File lib/pupistry/artifact.rb, line 195
def push_artifact
  # The push step involves 2 steps:
  # 1. GPG sign the artifact and write it into the manifest file
  # 2. Upload the manifest and archive files to S3.
  # 3. Upload a copy as the "latest" manifest file which will be hit by clients.

  # Determine which version we are uploading. Either one specifically
  # selected, otherwise find the latest one to push

  if defined? @checksum
    $logger.info "Uploading artifact version #{@checksum}."
  else
    @checksum = fetch_current

    if @checksum
      $logger.info "Uploading artifact version latest (#{@checksum})"
    else
      # If there is no current version, we can't do much....
      exit 0
    end
  end

  # Do we even need to upload? If nothing has changed....
  if @checksum == fetch_latest
    $logger.error "You've already pushed this artifact version, nothing to do."
    exit 0
  end

  # Make sure the files actually exist...
  unless File.exist?($config['general']['app_cache'] + "/artifacts/manifest.#{@checksum}.yaml")
    $logger.error "The files expected for #{@checksum} do not appear to exist or are not readable"
    fail 'Fatal unexpected error'
  end

  unless File.exist?($config['general']['app_cache'] + "/artifacts/artifact.#{@checksum}.tar.gz")
    $logger.error "The files expected for #{@checksum} do not appear to exist or are not readable"
    fail 'Fatal unexpected error'
  end

  # GPG sign the files
  if $config['general']['gpg_disable'] == true
    $logger.warn 'You have GPG signing *disabled*, whilst not critical it does weaken your security.'
    $logger.warn 'Skipping signing step...'
  else

    gpgsig = Pupistry::GPG.new @checksum

    # Sign the artifact
    unless gpgsig.artifact_sign
      $logger.fatal 'Unable to proceed with an unsigned artifact'
      exit 0
    end

    # Verify the signature - we want to make sure what we've just signed
    # can actually be validated properly :-)
    unless gpgsig.artifact_verify
      $logger.fatal 'Whilst a signature was generated, it was unable to be validated. This would suggest a bug of some kind.'
      exit 0
    end

    # Save the signature to the manifest
    unless gpgsig.signature_save
      $logger.fatal 'Unable to write the signature into the manifest file for the artifact.'
      exit 0
    end

  end

  # Upload the artifact & manifests to S3. We also make an additional copy
  # as the "latest" file which will be downloaded by all the agents checking
  # for new updates.

  s3 = Pupistry::StorageAWS.new 'build'
  s3.upload $config['general']['app_cache'] + "/artifacts/artifact.#{@checksum}.tar.gz", "artifact.#{@checksum}.tar.gz"
  s3.upload $config['general']['app_cache'] + "/artifacts/manifest.#{@checksum}.yaml", "manifest.#{@checksum}.yaml"
  s3.upload $config['general']['app_cache'] + "/artifacts/manifest.#{@checksum}.yaml", 'manifest.latest.yaml'

  # Test a read of the manifest, we do this to make sure the S3 ACLs setup
  # allow downloading of the uploaded files - helps avoid user headaches if
  # they misconfigure and then blindly trust their bootstrap config.
  #
  # Only worth doing this step if they've explicitly set their AWS IAM credentials
  # for the agent, which should be everyone except for IAM role users.

  if $config['agent']['access_key_id']
    fetch_artifact
  else
    $logger.warn "The agent's AWS credentials are unset on this machine, unable to do download test to check permissions for you."
    $logger.warn "Assuming you know what you're doing, please set if unsure."
  end

  $logger.info "Upload of artifact version #{@checksum} completed and is now latest"
end
unpack() click to toggle source
# File lib/pupistry/artifact.rb, line 400
def unpack
  # Unpack the currently selected artifact to the archives directory.

  # An application version must be specified
  fail 'Application bug, trying to unpack no artifact' unless defined? @checksum

  # Make sure the files actually exist...
  unless File.exist?($config['general']['app_cache'] + "/artifacts/manifest.#{@checksum}.yaml")
    $logger.error "The files expected for #{@checksum} do not appear to exist or are not readable"
    fail 'Fatal unexpected error'
  end

  unless File.exist?($config['general']['app_cache'] + "/artifacts/artifact.#{@checksum}.tar.gz")
    $logger.error "The files expected for #{@checksum} do not appear to exist or are not readable"
    fail 'Fatal unexpected error'
  end

  # Clean up an existing unpacked copy - in *theory* it should be same, but
  # a mistake like running out of disk could have left it in an unclean state
  # so let's make sure it's gone
  clean_unpack

  # Unpack the archive file
  FileUtils.mkdir_p($config['general']['app_cache'] + "/artifacts/unpacked.#{@checksum}")
  Dir.chdir($config['general']['app_cache'] + "/artifacts/unpacked.#{@checksum}") do
    tar = Pupistry::Config.which_tar
    $logger.debug "Using tar at #{tar}"

    if system "#{tar} -xzf ../artifact.#{@checksum}.tar.gz"
      $logger.debug "Successfully unpacked artifact #{@checksum}"
    else
      $logger.error "Unable to unpack artifact files to #{Dir.pwd}"
      fail 'An unexpected error occured when executing tar'
    end
  end
end