class Dpkg::S3::CLI

CLI interface for dpkg-s3

Public Class Methods

exit_on_failure?() click to toggle source
# File lib/dpkg/s3/cli.rb, line 150
def self.exit_on_failure?
  true
end

Public Instance Methods

copy(package_name, to_codename, to_component) click to toggle source
# File lib/dpkg/s3/cli.rb, line 384
def copy(package_name, to_codename, to_component)
  error 'You must specify a package name.' if package_name.nil?
  error 'You must specify a codename to copy to.' if to_codename.nil?
  error 'You must specify a component to copy to.' if to_component.nil?

  arch = options[:arch]
  error 'You must specify the architecture of the package to copy.' if arch.nil?

  versions = options[:versions]
  if versions.nil?
    warn "===> WARNING: Copying all versions of #{package_name}"
  else
    log "Versions to copy: #{versions.join(', ')}"
  end

  configure_s3_client

  # retrieve the existing manifests
  log 'Retrieving existing manifests'
  from_manifest = Dpkg::S3::Manifest.retrieve(options[:codename],
                                              component, arch,
                                              options[:cache_control],
                                              false, skip_upload: options[:skip_package_upload])
  to_release = Dpkg::S3::Release.retrieve(to_codename)
  to_manifest = Dpkg::S3::Manifest.retrieve(to_codename, to_component, arch,
                                            options[:cache_control],
                                            options[:fail_if_exists],
                                            skip_upload: options[:skip_package_upload])
  packages = from_manifest.packages.select do |p|
    p.name == package_name &&
      (versions.nil? || versions.include?(p.full_version))
  end
  error 'No packages found in repository.' if packages.size.zero?

  packages.each do |package|
    begin
      to_manifest.add package, options[:preserve_versions], false
    rescue Dpkg::S3::Utils::AlreadyExistsError => e
      error("Preparing manifest failed because: #{e}")
    end
  end

  begin
    to_manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
  rescue Dpkg::S3::Utils::AlreadyExistsError => e
    error("Copying manifest failed because: #{e}")
  end
  to_release.update_manifest(to_manifest)
  to_release.write_to_s3 { |f| sublog("Transferring #{f}") }

  log 'Copy complete.'
end
delete(package) click to toggle source
# File lib/dpkg/s3/cli.rb, line 454
def delete(package)
  error('You must specify a package name.') if package.nil?

  versions = options[:versions]
  if versions.nil?
    warn("===> WARNING: Deleting all versions of #{package}")
  else
    log("Versions to delete: #{versions.join(', ')}")
  end

  arch = options[:arch]
  error('You must specify the architecture of the package to remove.') if arch.nil?

  configure_s3_client

  # retrieve the existing manifests
  log('Retrieving existing manifests')
  release = Dpkg::S3::Release.retrieve(options[:codename], options[:origin], options[:suite])
  selected_arch = if arch == 'all'
                    release.architectures
                  else
                    [arch]
                  end
  all_found = 0
  selected_arch.each do |ar|
    manifest = Dpkg::S3::Manifest.retrieve(options[:codename], component, ar, options[:cache_control], false,
                                           skip_upload: options[:skip_package_upload])

    deleted = manifest.delete_package(package, versions)
    all_found += deleted.length
    if deleted.length.zero?
      if versions.nil?
        sublog("No packages were deleted. #{package} not found in arch #{ar}.")
      else
        sublog("No packages were deleted. #{package} versions #{versions.join(', ')}
         could not be found in arch #{ar}.")
      end
      next
    else
      deleted.each do |p|
        sublog("Deleting #{p.name} version #{p.full_version} from arch #{ar}")
      end
    end

    log('Uploading new manifests to S3')
    manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
    release.update_manifest(manifest)
    release.write_to_s3 { |f| sublog("Transferring #{f}") }

    log('Update complete.')
  end
  return unless all_found.zero?

  if versions.nil?
    error("No packages were deleted. #{package} not found.")
  else
    error("No packages were deleted. #{package} versions #{versions.join(', ')} could not be found.")
  end
end
list() click to toggle source
# File lib/dpkg/s3/cli.rb, line 295
def list
  configure_s3_client

  release = Dpkg::S3::Release.retrieve(options[:codename])
  archs = release.architectures
  archs &= [options[:arch]] if options[:arch] && options[:arch] != 'all'
  widths = [0, 0]
  rows = archs.map do |arch|
    manifest = Dpkg::S3::Manifest.retrieve(options[:codename], component,
                                           arch, options[:cache_control],
                                           false, skip_upload: false)
    manifest.packages.map do |package|
      if options[:long]
        package.generate(options[:codename])
      else
        [package.name, package.full_version, package.architecture].tap do |row|
          row.each_with_index do |col, i|
            widths[i] = [widths[i], col.size].max if widths[i]
          end
        end
      end
    end
  end.flatten(1)

  if options[:long]
    $stdout.puts rows.join("\n")
  else
    rows.each do |row|
      $stdout.puts format("% -#{widths[0]}<package>s  % -#{widths[1]}<version>s  %<arch>s",
                          package: row[0], version: row[1], arch: row[2])
    end
  end
end
show(package_name, version, arch) click to toggle source
# File lib/dpkg/s3/cli.rb, line 331
def show(package_name, version, arch)
  error 'You must specify the name of the package to show.' if version.nil?
  error 'You must specify the version of the package to show.' if version.nil?
  error 'You must specify the architecture of the package to show.' if arch.nil?

  configure_s3_client

  # retrieve the existing manifests
  manifest = Dpkg::S3::Manifest.retrieve(options[:codename], component, arch,
                                         options[:cache_control], false, false)
  package = manifest.packages.detect do |p|
    p.name == package_name && p.full_version == version
  end
  error 'No such package found.' if package.nil?

  puts package.generate(options[:codename])
end
upload(*files) click to toggle source
# File lib/dpkg/s3/cli.rb, line 154
def upload(*files)
  error('You must specify at least one file to upload') if files.nil? || files.empty?

  # make sure all the files exists
  if (missing_file = files.find { |pattern| Dir.glob(pattern).empty? })
    error("File '#{missing_file}' doesn't exist")
  end

  # configure AWS::S3
  configure_s3_client

  begin
    if options[:lock]
      log('Checking for existing lock file')
      if Dpkg::S3::Lock.locked?(options[:codename], component, options[:arch], options[:cache_control])
        lock = Dpkg::S3::Lock.current(options[:codename], component, options[:arch], options[:cache_control])
        log("Repository is locked by another user: #{lock.user} at host #{lock.host}")
        log('Attempting to obtain a lock')
        Dpkg::S3::Lock.wait_for_lock(options[:codename], component, options[:arch], options[:cache_control])
      end
      log('Locking repository for updates')
      Dpkg::S3::Lock.lock(options[:codename], component, options[:arch], options[:cache_control])
      @lock_acquired = true
    end

    # retrieve the existing manifests
    log('Retrieving existing manifests')
    release = Dpkg::S3::Release.retrieve(options[:codename], options[:origin], options[:suite],
                                         options[:cache_control])
    manifests = {}
    release.architectures.each do |arch|
      manifests[arch] =
        Dpkg::S3::Manifest.retrieve(options[:codename], component, arch, options[:cache_control],
                                    options[:fail_if_exists], skip_upload: options[:skip_package_upload])
    end

    packages_arch_all = []

    # examine all the files
    files.collect { |f| Dir.glob(f) }.flatten.each do |file|
      log("Examining package file #{File.basename(file)}")
      pkg = Dpkg::S3::Package.parse_file(file)

      # copy over some options if they weren't given
      arch = options[:arch] || pkg.architecture

      # If they've specified an arch type that doesn't match the package let them know
      if options.key?('arch') && options[:arch] != pkg.architecture
        warn("You specified architecture #{options[:arch]} but package #{pkg.name} has architecture
         type of #{pkg.architecture}")
      end

      # validate we have them
      unless arch
        error("No architcture given and unable to determine one for #{file}. " \
              'Please specify one with --arch [i386|amd64|armhf].')
      end

      # If the arch is all and the list of existing manifests is none, then
      # throw an error. This is mainly the case when initializing a brand new
      # repository. With "all", we won't know which architectures they're using.
      if arch == 'all' && manifests.count.zero?
        manifests['amd64'] =
          Dpkg::S3::Manifest.retrieve(options[:codename], component, 'amd64', options[:cache_control],
                                      options[:fail_if_exists], skip_upload: options[:skip_package_upload])
        manifests['i386'] =
          Dpkg::S3::Manifest.retrieve(options[:codename], component, 'i386', options[:cache_control],
                                      options[:fail_if_exists], skip_upload: options[:skip_package_upload])
        manifests['armhf'] =
          Dpkg::S3::Manifest.retrieve(options[:codename], component, 'armhf', options[:cache_control],
                                      options[:fail_if_exists], skip_upload: options[:skip_package_upload])

        # error("Package #{File.basename(file)} had architecture \"all\", " +
        #       "however noexisting package lists exist. This can often happen " +
        #       "if the first package you are add to a new repository is an " +
        #       "\"all\" architecture file. Please use --arch [i386|amd64|armhf] or " +
        #       "another platform type to upload the file.")
      end

      # retrieve the manifest for the arch if we don't have it already
      manifests[arch] ||= Dpkg::S3::Manifest.retrieve(options[:codename], component, arch,
                                                      options[:cache_control], options[:fail_if_exists],
                                                      skip_upload: options[:skip_package_upload])

      # add package in manifests
      begin
        manifests[arch].add(pkg, options[:preserve_versions])
      rescue Dpkg::S3::Utils::AlreadyExistsError => e
        error("Preparing manifest failed because: #{e}")
      end

      # If arch is all, we must add this package in all arch available
      packages_arch_all << pkg if arch == 'all'
    end

    manifests.each do |arch, manifest|
      next if arch == 'all'

      packages_arch_all.each do |pkg|
        begin
          manifest.add(pkg, options[:preserve_versions], needs_uploading: false)
        rescue Dpkg::S3::Utils::AlreadyExistsError => e
          error("Preparing manifest failed because: #{e}")
        end
      end
    end

    # upload the manifest
    log('Uploading packages and new manifests to S3')
    manifests.each_value do |manifest|
      begin
        manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
      rescue Dpkg::S3::Utils::AlreadyExistsError => e
        error("Uploading manifest failed because: #{e}")
      end
      release.update_manifest(manifest)
    end
    release.write_to_s3 { |f| sublog("Transferring #{f}") }

    log('Update complete.')
  ensure
    if options[:lock] && @lock_acquired
      Dpkg::S3::Lock.unlock(options[:codename], component, options[:arch], options[:cache_control])
      log('Lock released.')
    end
  end
end
verify() click to toggle source
# File lib/dpkg/s3/cli.rb, line 522
def verify
  configure_s3_client

  log('Retrieving existing manifests')
  release = Dpkg::S3::Release.retrieve(options[:codename], options[:origin], options[:suite])

  release.architectures.each do |arch|
    log("Checking for missing packages in: #{options[:codename]}/#{options[:component]} #{arch}")
    manifest = Dpkg::S3::Manifest.retrieve(options[:codename], component,
                                           arch, options[:cache_control], false,
                                           skip_upload: options[:skip_package_upload])
    missing_packages = []

    manifest.packages.each do |p|
      next if Dpkg::S3::Utils.s3_exists? p.url_filename_encoded(options[:codename])

      sublog("The following packages are missing:\n\n") if missing_packages.empty?
      puts(p.generate(options[:codename]))
      puts('')

      missing_packages << p
    end

    next unless options[:sign] || (options[:fix_manifests] && !missing_packages.empty?)

    log("Removing #{missing_packages.length} package(s) from the manifest...")
    missing_packages.each { |p| manifest.packages.delete(p) }
    manifest.write_to_s3 { |f| sublog("Transferring #{f}") }
    release.update_manifest(manifest)
    release.write_to_s3 { |f| sublog("Transferring #{f}") }

    log('Update complete.')
  end
end

Private Instance Methods

component() click to toggle source
# File lib/dpkg/s3/cli.rb, line 559
def component
  return @component if @component

  @component = if (section = options[:section])
                 warn('===> WARNING: The --section/-s argument is ' \
                      'deprecated, please use --component/-m.')
                 section
               else
                 options[:component]
               end
end
configure_s3_client() click to toggle source
# File lib/dpkg/s3/cli.rb, line 604
def configure_s3_client
  error("No value provided for required options '--bucket'") unless options[:bucket]

  settings = {
    region: options[:s3_region],
    http_proxy: options[:proxy_uri],
    force_path_style: options[:force_path_style]
  }
  settings[:endpoint] = options[:endpoint] if options[:endpoint]
  settings.merge!(provider)

  Dpkg::S3::Utils.s3          = Aws::S3::Client.new(settings)
  Dpkg::S3::Utils.bucket      = options[:bucket]
  Dpkg::S3::Utils.signing_key = options[:sign]
  Dpkg::S3::Utils.gpg_options = options[:gpg_options]
  Dpkg::S3::Utils.prefix      = options[:prefix]
  Dpkg::S3::Utils.encryption  = options[:encryption]

  # make sure we have a valid visibility setting
  Dpkg::S3::Utils.access_policy =
    case options[:visibility]
    when 'public'
      'public-read'
    when 'private'
      'private'
    when 'authenticated'
      'authenticated-read'
    when 'bucket_owner'
      'bucket-owner-full-control'
    else
      error('Invalid visibility setting given. Can be public, private, authenticated, or bucket_owner.')
    end
end
error(message) click to toggle source
# File lib/dpkg/s3/cli.rb, line 583
def error(message)
  warn "!! #{message}" unless options[:quiet]
  exit 1
end
log(message) click to toggle source
# File lib/dpkg/s3/cli.rb, line 575
def log(message)
  puts ">> #{message}" unless options[:quiet]
end
provider() click to toggle source
# File lib/dpkg/s3/cli.rb, line 588
def provider
  access_key_id     = options[:access_key_id]
  secret_access_key = options[:secret_access_key]
  session_token     = options[:session_token]

  if access_key_id.nil? ^ secret_access_key.nil?
    error('If you specify one of --access-key-id or --secret-access-key, you must specify the other.')
  end
  static_credentials = {}
  static_credentials[:access_key_id]     = access_key_id     if access_key_id
  static_credentials[:secret_access_key] = secret_access_key if secret_access_key
  static_credentials[:session_token]     = session_token     if session_token

  static_credentials
end
puts(*args) click to toggle source
# File lib/dpkg/s3/cli.rb, line 571
def puts(*args)
  $stdout.puts(*args) unless options[:quiet]
end
sublog(message) click to toggle source
# File lib/dpkg/s3/cli.rb, line 579
def sublog(message)
  puts "   -- #{message}" unless options[:quiet]
end