class Omnibus::NetFetcher

Constants

ALL_EXTENSIONS
COMPRESSED_TAR_EXTENSIONS

tar probably has compression scheme linked in, otherwise for tarballs

DIGESTS

Digest types used for verifying file checksums

TAR_EXTENSIONS
WIN_7Z_EXTENSIONS

Use 7-zip to extract 7z/zip for Windows

Public Class Methods

resolve_version(version, source) click to toggle source

Returned the resolved version for the manifest. Since this is a remote URL, there is no resolution, the version is what we said it is.

@return [String]

# File lib/omnibus/fetchers/net_fetcher.rb, line 110
def self.resolve_version(version, source)
  version
end

Public Instance Methods

checksum() click to toggle source

The checksum as defined by the user in the software definition.

@return [String]

# File lib/omnibus/fetchers/net_fetcher.rb, line 132
def checksum
  source[digest_type]
end
clean() click to toggle source

Clean the project directory if it exists and actually extract the downloaded file.

@return [true, false]

true if the project directory was removed, false otherwise
# File lib/omnibus/fetchers/net_fetcher.rb, line 64
def clean
  needs_cleaning = File.exist?(project_dir)
  if needs_cleaning
    log.info(log_key) { "Cleaning project directory `#{project_dir}'" }
    FileUtils.rm_rf(project_dir)
  end
  create_required_directories
  deploy
  needs_cleaning
end
downloaded_file() click to toggle source

The path on disk to the downloaded asset. The filename is defined by +source :cached_name+. If ommited, then it comes from the software’s +source :url+ value

@return [String]

# File lib/omnibus/fetchers/net_fetcher.rb, line 121
def downloaded_file
  filename = source[:cached_name] if source[:cached_name]
  filename ||= File.basename(source[:url], "?*")
  File.join(Config.cache_dir, filename)
end
fetch() click to toggle source

Fetch the given software definition. This method always fetches the file, even if it already exists on disk! You should use {#fetch_required?} to guard against this check in your implementation.

@return [void]

# File lib/omnibus/fetchers/net_fetcher.rb, line 82
def fetch
  log.info(log_key) { "Downloading from `#{download_url}'" }

  create_required_directories
  download
  verify_checksum!
end
fetch_required?() click to toggle source

A fetch is required if the downloaded_file (such as a tarball) does not exist on disk, or if the checksum of the downloaded file is different than the given checksum.

@return [true, false]

# File lib/omnibus/fetchers/net_fetcher.rb, line 43
def fetch_required?
  !(File.exist?(downloaded_file) && digest(downloaded_file, digest_type) == checksum)
end
version_for_cache() click to toggle source

The version for this item in the cache. This is the digest of downloaded file and the URL where it was downloaded from.

This method is called before clean but after fetch. Do not ever use the contents of the project_dir here.

@return [String]

# File lib/omnibus/fetchers/net_fetcher.rb, line 99
def version_for_cache
  "download_url:#{source[:url]}|#{digest_type}:#{checksum}"
end
version_guid() click to toggle source

The version identifier for this remote location. This is computed using the name of the software, the version of the software, and the checksum.

@return [String]

# File lib/omnibus/fetchers/net_fetcher.rb, line 53
def version_guid
  "#{digest_type}:#{checksum}"
end

Private Instance Methods

deploy() click to toggle source

Extract the downloaded file, using the magical logic based off of the ending file extension. In the rare event the file cannot be extracted, it is copied over as a raw file.

# File lib/omnibus/fetchers/net_fetcher.rb, line 186
def deploy
  if downloaded_file.end_with?(*ALL_EXTENSIONS)
    log.info(log_key) { "Extracting `#{safe_downloaded_file}' to `#{safe_project_dir}'" }
    extract
  else
    log.info(log_key) { "`#{safe_downloaded_file}' is not an archive - copying to `#{safe_project_dir}'" }

    if File.directory?(downloaded_file)
      # If the file itself was a directory, copy the whole thing over. This
      # seems unlikely, because I do not think it is a possible to download
      # a folder, but better safe than sorry.
      FileUtils.cp_r("#{downloaded_file}/.", project_dir)
    else
      # In the more likely case that we got a "regular" file, we want that
      # file to live **inside** the project directory. project_dir should already
      # exist due to create_required_directories
      FileUtils.cp(downloaded_file, project_dir)
    end
  end
end
digest_type() click to toggle source

The digest type defined in the software definition

@raise [ChecksumMissing]

if the checksum does not exist

@return [Symbol]

# File lib/omnibus/fetchers/net_fetcher.rb, line 273
def digest_type
  DIGESTS.each do |digest|
    return digest if source.key? digest
  end
  raise ChecksumMissing.new(self)
end
download() click to toggle source

Download the given file using Ruby’s OpenURI implementation. This method may emit warnings as defined in software definitions using the :warning key.

@return [void]

# File lib/omnibus/fetchers/net_fetcher.rb, line 164
def download
  log.warn(log_key) { source[:warning] } if source.key?(:warning)

  options = {}

  if source[:unsafe]
    log.warn(log_key) { "Permitting unsafe redirects!" }
    options[:allow_unsafe_redirects] = true
  end

  # Set the cookie if one was given
  options["Cookie"] = source[:cookie] if source[:cookie]
  options["Authorization"] = source[:authorization] if source[:authorization]

  download_file!(download_url, downloaded_file, options)
end
download_url() click to toggle source

The URL from which to download the software - this comes from the software’s +source :url+ value.

If S3 caching is enabled, this is the download URL for the software from the S3 bucket as defined in the {Config}.

@return [String]

# File lib/omnibus/fetchers/net_fetcher.rb, line 147
def download_url
  if Config.use_s3_caching
    S3Cache.url_for(self)
  elsif Config.use_internal_sources && !source[:internal]
    raise InternalSourceMissing.new(self)
  else
    source[:url]
  end
end
extract() click to toggle source

Extracts the downloaded archive file into project_dir.

On windows, this is a fuster cluck and we allow users to specify the preferred extractor to be used. The default is to use tar. User overrides can be set in source as:

:tar - use tar.exe and fail on errors (default strategy).
:seven_zip - use 7zip for all tar/compressed tar files on windows.
:lax_tar - use tar.exe on windows but ignore errors.

7z has issues on windows.

7z cannot extract and untar at the same time. You need to extract to a temporary location and then extract again into project_dir.

7z also doesn’t handle symlinks well. A symlink to a non-existent location simply results in a text file with the target path written in it. It does this without throwing any errors.

# File lib/omnibus/fetchers/net_fetcher.rb, line 226
def extract
  # Only used by tar
  compression_switch = ""
  compression_switch = "z"        if downloaded_file.end_with?("gz")
  compression_switch = "--lzma -" if downloaded_file.end_with?("lzma")
  compression_switch = "j"        if downloaded_file.end_with?("bz2")
  compression_switch = "J"        if downloaded_file.end_with?("xz")

  if Ohai["platform"] == "windows"
    if downloaded_file.end_with?(*TAR_EXTENSIONS) && source[:extract] != :seven_zip
      returns = [0]
      returns << 1 if source[:extract] == :lax_tar

      shellout!("tar #{compression_switch}xf #{downloaded_file} --force-local -C#{project_dir}", returns: returns)
    elsif downloaded_file.end_with?(*COMPRESSED_TAR_EXTENSIONS)
      Dir.mktmpdir do |temp_dir|
        log.debug(log_key) { "Temporarily extracting `#{safe_downloaded_file}' to `#{temp_dir}'" }

        shellout!("7z.exe x #{safe_downloaded_file} -o#{windows_safe_path(temp_dir)} -r -y")

        fname = File.basename(downloaded_file, File.extname(downloaded_file))
        fname << ".tar" if downloaded_file.end_with?("tgz", "txz")
        next_file = windows_safe_path(File.join(temp_dir, fname))

        log.debug(log_key) { "Temporarily extracting `#{next_file}' to `#{safe_project_dir}'" }
        shellout!("7z.exe x #{next_file} -o#{safe_project_dir} -r -y")
      end
    else
      shellout!("7z.exe x #{safe_downloaded_file} -o#{safe_project_dir} -r -y")
    end
  elsif downloaded_file.end_with?(".7z")
    shellout!("7z x #{safe_downloaded_file} -o#{safe_project_dir} -r -y")
  elsif downloaded_file.end_with?(".zip")
    shellout!("unzip #{safe_downloaded_file} -d #{safe_project_dir}")
  else
    shellout!("#{tar} #{compression_switch}xf #{safe_downloaded_file} -C#{safe_project_dir}")
  end
end
extract_command() click to toggle source

The command to use for extracting this piece of software.

@return [[String]]

# File lib/omnibus/fetchers/net_fetcher.rb, line 310
def extract_command; end
safe_downloaded_file() click to toggle source
# File lib/omnibus/fetchers/net_fetcher.rb, line 301
def safe_downloaded_file
  windows_safe_path(downloaded_file)
end
safe_project_dir() click to toggle source
# File lib/omnibus/fetchers/net_fetcher.rb, line 297
def safe_project_dir
  windows_safe_path(project_dir)
end
tar() click to toggle source

Primitively determine whether we should use gtar or tar to untar a file. If gtar is present, we will use gtar (AIX). Otherwise, we fallback to tar.

@return [String]

# File lib/omnibus/fetchers/net_fetcher.rb, line 318
def tar
  Omnibus.which("gtar") ? "gtar" : "tar"
end
verify_checksum!() click to toggle source

Verify the downloaded file has the correct checksum.

@raise [ChecksumMismatch]

if the checksum does not match
# File lib/omnibus/fetchers/net_fetcher.rb, line 286
def verify_checksum!
  log.info(log_key) { "Verifying checksum" }

  expected = checksum
  actual   = digest(downloaded_file, digest_type)

  if expected != actual
    raise ChecksumMismatch.new(self, expected, actual)
  end
end