class Dockly::Docker

Public Instance Methods

add_build_env(image) click to toggle source
# File lib/dockly/docker.rb, line 200
def add_build_env(image)
  return image if build_env.empty?
  info "Setting the following environment variables in the docker image: #{build_env.keys}"
  dockerfile = [
    "FROM #{image.id}",
    *build_env.map { |key, val| "ENV #{key.to_s.shellescape}=#{val.to_s.shellescape}" }
  ].join("\n")
  out_image = ::Docker::Image.build(dockerfile)
  info "Successfully set the environment variables in the dockerfile"
  out_image
end
add_git_archive(image) click to toggle source
# File lib/dockly/docker.rb, line 212
def add_git_archive(image)
  return image if git_archive.nil?
  info "adding the git archive"
  new_image = image.insert_local(
    'localPath' => git_archive_tar,
    'outputPath' => '/'
  )
  info "successfully added the git archive"
  new_image
end
build_env(hash = nil) click to toggle source
# File lib/dockly/docker.rb, line 33
def build_env(hash = nil)
  (@build_env ||= {}).tap { |env| env.merge!(hash) if hash.is_a?(Hash) }
end
build_image(image) click to toggle source
# File lib/dockly/docker.rb, line 223
def build_image(image)
  ensure_present! :name, :build
  info "running custom build steps, starting with id: #{image.id}"
  out_image = ::Docker::Image.build("from #{image.id}\n#{build}")
  info "finished running custom build steps, result id: #{out_image.id}"
  out_image.tap { |img| img.tag(repo: repo, tag: tag, force: true) }
end
cleanup(images) click to toggle source
# File lib/dockly/docker.rb, line 110
def cleanup(images)
  info 'Cleaning up intermediate images'
  images ||= []
  images = images.compact
  ::Docker::Container.all(:all => true).each do |container|
    image_id = container.json['Image']
    if images.any? { |image| image.id.start_with?(image_id) || image_id.start_with?(image.id) }
      container.kill
      container.delete
    end
  end
  images.each { |image| image.remove rescue nil }
  info 'Done cleaning images'
end
copy_from_s3(sha) click to toggle source
# File lib/dockly/docker.rb, line 41
def copy_from_s3(sha)
  return if s3_bucket.nil?
  object = s3_object_for(sha)
  info "Copying s3://#{s3_bucket}/#{object} to #{s3_bucket}/#{s3_object}"
  Dockly.s3.copy_object(
    copy_source: File.join(s3_bucket, object),
    bucket: s3_bucket,
    key: s3_object,
    acl: 'bucket-owner-full-control',
  )
  info "Successfully copied s3://#{s3_bucket}/#{object} to s3://#{s3_bucket}/#{s3_object}"
end
ensure_tar(file_name) click to toggle source
# File lib/dockly/docker.rb, line 143
def ensure_tar(file_name)
  if Dockly::Util::Tar.is_tar?(file_name)
    file_name
  elsif Dockly::Util::Tar.is_gzip?(file_name)
    file_name
  else
    raise "Expected a (possibly gzipped) tar: #{file_name}"
  end
end
exists?() click to toggle source
# File lib/dockly/docker.rb, line 377
def exists?
  return false unless s3_bucket
  debug "#{name}: checking for package: #{s3_url}"
  Dockly.s3.head_object(bucket: s3_bucket, key: s3_object)
  info "#{name}: found package: #{s3_url}"
  true
rescue Aws::S3::Errors::NotFound, Aws::S3::Errors::NoSuchKey
  info "#{name}: could not find package: #{s3_url}"
  false
end
export_filename() click to toggle source
# File lib/dockly/docker.rb, line 125
def export_filename
  "#{name}-image.tgz"
end
export_image(image) click to toggle source
# File lib/dockly/docker.rb, line 246
def export_image(image)
  ensure_present! :name
  if registry.nil?
    ensure_present! :build_dir
    info "Exporting the image with id #{image.id} to file #{File.expand_path(tar_path)}"
    container = image.run('true')
    info "created the container: #{container.id}"

    if s3_bucket.nil?
      output = File.open(tar_path, 'wb')
    else
      output = Dockly::S3Writer.new(Dockly.s3, s3_bucket, s3_object)
    end

    gzip_output = Zlib::GzipWriter.new(output)

    if tar_diff
      export_image_diff(container, gzip_output)
    else
      export_image_whole(container, gzip_output)
    end
  else
    push_to_registry(image)
  end
rescue
  if output && !s3_bucket.nil?
    output.abort_unless_closed
  end
  raise
ensure
  container.tap(&:wait).remove if container
  gzip_output.close if gzip_output
end
export_image_diff(container, output) click to toggle source
# File lib/dockly/docker.rb, line 286
def export_image_diff(container, output)
  rd, wr = IO.pipe(Encoding::ASCII_8BIT)

  rd.binmode
  wr.binmode

  thread = Thread.new do
    begin
      if Dockly::Util::Tar.is_tar?(fetch_import)
        base = File.open(fetch_import, 'rb')
      else
        base = Zlib::GzipReader.new(File.open(fetch_import, 'rb'))
      end
      td = Dockly::TarDiff.new(base, rd, output)
      td.process
      info "done writing the docker tar: #{export_filename}"
    ensure
      base.close if base
      rd.close
    end
  end

  begin
    container.export do |chunk, remaining, total|
      wr.write(chunk)
    end
  ensure
    wr.close
    thread.join
  end
end
export_image_whole(container, output) click to toggle source
# File lib/dockly/docker.rb, line 280
def export_image_whole(container, output)
  container.export do |chunk, remaining, total|
    output.write(chunk)
  end
end
export_only() click to toggle source
# File lib/dockly/docker.rb, line 61
def export_only
  if image = find_image_by_repotag
    info "Found image by repo:tag: #{repo}:#{tag} - #{image.inspect}"
    export_image(image)
  else
    raise "Could not find image"
  end
end
fetch_import() click to toggle source
# File lib/dockly/docker.rb, line 349
def fetch_import
  ensure_present! :import
  path = "/tmp/dockly-docker-import.#{name}.#{File.basename(import)}"

  if File.exist?(path)
    debug "already fetched #{import}"
  else
    debug "fetching #{import}"
    File.open("#{path}.tmp", 'wb') do |file|
      case import
        when /^s3:\/\/(?<bucket>.+?)\/(?<key>.+)$/
          bucket, key = Regexp.last_match[:bucket], Regexp.last_match[:key]
          Dockly.s3.get_object(bucket: bucket, key: key) do |chunk|
            file.write(chunk)
          end
        when /^https?:\/\//
          Excon.get(import, :response_block => lambda { |chunk, remaining, total|
            file.write(chunk)
          })
        else
          raise "You can only import from S3 or a public url"
      end
    end
    FileUtils.mv("#{path}.tmp", path, :force => true)
  end
  path
end
find_image_by_repotag() click to toggle source
# File lib/dockly/docker.rb, line 70
def find_image_by_repotag
  Docker::Image.all.find do |image|
    image.info["RepoTags"].include?("#{repo}:#{tag}")
  end
end
generate!() click to toggle source
# File lib/dockly/docker.rb, line 54
def generate!
  image = generate_build
  export_image(image)
ensure
  cleanup([image]) if cleanup_images
end
generate_build() click to toggle source
# File lib/dockly/docker.rb, line 76
def generate_build
  Docker.options.merge!(:read_timeout => timeout, :write_timeout => timeout)
  Docker.reset_connection!
  images = {}

  if registry_import.nil?
    docker_tar = File.absolute_path(ensure_tar(fetch_import))
    images[:one] = import_base(docker_tar)
  else
    registry.authenticate! unless registry.nil?
    full_name = "#{registry_import[:name]}:#{registry_import[:tag]}"
    info "Pulling #{full_name}"
    images[:one] = ::Docker::Image.create('fromImage' => registry_import[:name], 'tag' => registry_import[:tag])
    info "Successfully pulled #{full_name}"
  end

  images[:two] = add_build_env(images[:one])
  images[:three] = add_git_archive(images[:two])
  images[:four] = run_build_caches(images[:three])
  build_image(images[:four])
ensure
  cleanup(images.values.compact) if cleanup_images
end
git_archive_dir() click to toggle source
# File lib/dockly/docker.rb, line 170
def git_archive_dir
  @git_archive_dir ||= File.join(build_dir, "gitarc")
end
git_archive_path() click to toggle source
# File lib/dockly/docker.rb, line 174
def git_archive_path
  "#{git_archive_dir}/#{name}.tar"
end
git_archive_tar() click to toggle source
# File lib/dockly/docker.rb, line 178
def git_archive_tar
  git_archive && File.absolute_path(make_git_archive)
end
import_base(docker_tar) click to toggle source
# File lib/dockly/docker.rb, line 182
def import_base(docker_tar)
  repo = "#{name}-base"
  tag = "dockly-#{Dockly::VERSION}-#{File.basename(import).split('.').first}"
  info "looking for imported base image with tag: #{tag}"
  image = Docker::Image.all.find do |img|
    img.info['RepoTags'] && img.info['RepoTags'].include?("#{repo}:#{tag}")
  end
  if image
    info "found imported base image: #{image.id}"
    image
  else
    info "could not find image with tag #{tag}, importing the docker image from #{docker_tar}"
    image = ::Docker::Image.import(docker_tar, 'repo' => repo, 'tag' => tag)
    info "imported initial docker image: #{image.id}"
    image
  end
end
make_git_archive() click to toggle source
# File lib/dockly/docker.rb, line 153
def make_git_archive
  ensure_present! :git_archive
  info "initializing"

  prefix = git_archive
  prefix += '/' unless prefix.end_with?('/')

  FileUtils.rm_rf(git_archive_dir)
  FileUtils.mkdir_p(git_archive_dir)
  info "archiving #{Dockly::Util::Git.sha}"
  File.open(git_archive_path, 'wb') do |file|
    Dockly::Util::Git.archive(Dockly::Util::Git.sha, prefix, file)
  end
  info "made the git archive for sha #{Dockly::Util::Git.sha}"
  git_archive_path
end
push_to_registry(image) click to toggle source
# File lib/dockly/docker.rb, line 326
def push_to_registry(image)
  raise "No registry to push to" if registry.nil?

  info "Exporting #{image.id} to Docker registry at #{registry.server_address}"

  registry.authenticate!

  image =
    Docker::Image
      .all(:all => true)
      .find do |img|
        img.id.include?(image.id) || image.id.include?(img.id)
      end

  raise "Could not find image after authentication" if image.nil?

  image.push(registry.to_h, :registry => registry.server_address) do |resp|
    if resp.include?('errorDetail')
      raise "Error pushing to registry: #{resp}"
    end
  end
end
registry() click to toggle source
# File lib/dockly/docker.rb, line 37
def registry
  ecr || docker_registry
end
registry_import(img_name = nil, opts = {}) click to toggle source
# File lib/dockly/docker.rb, line 100
def registry_import(img_name = nil, opts = {})
  if img_name
    @registry_import ||= {}
    @registry_import[:name] = img_name
    @registry_import[:tag] = opts[:tag] || 'latest'
  else
    @registry_import
  end
end
repo() click to toggle source
# File lib/dockly/docker.rb, line 231
def repo
  @repo ||= case
  when registry.nil?
    name
  when registry.default_server_address?
    "#{registry.username}/#{name}"
  else
    "#{registry.server_address}/#{name}"
  end
end
repository(value = nil) click to toggle source
# File lib/dockly/docker.rb, line 388
def repository(value = nil)
  name(value)
end
run_build_caches(image) click to toggle source
# File lib/dockly/docker.rb, line 129
def run_build_caches(image)
  info "starting build caches"
  (build_cache || []).each do |cache|
    cache.image = image
    image = cache.execute!
  end
  info "finished build caches"
  image
end
s3_object() click to toggle source
# File lib/dockly/docker.rb, line 318
def s3_object
  s3_object_for(Dockly::Util::Git.sha)
end
s3_object_for(sha) click to toggle source
# File lib/dockly/docker.rb, line 322
def s3_object_for(sha)
  [s3_object_prefix, sha, '/', export_filename].join
end
s3_url() click to toggle source
# File lib/dockly/docker.rb, line 242
def s3_url
  "s3://#{s3_bucket}/#{s3_object}"
end
tar_path() click to toggle source
# File lib/dockly/docker.rb, line 139
def tar_path
  File.join(build_dir, export_filename)
end