class Chef::CookbookSynchronizer

Synchronizes the locally cached copies of cookbooks with the files on the server.

Constants

CookbookFile

Attributes

remove_obsoleted_files[RW]

Public Class Methods

new(cookbooks_by_name, events) click to toggle source
# File lib/chef/cookbook/synchronizer.rb, line 91
def initialize(cookbooks_by_name, events)
  @cookbooks_by_name, @events = cookbooks_by_name, events

  @cookbook_full_file_paths = {}
  @remove_obsoleted_files = true

  @lazy_files = {}
end

Public Instance Methods

cache() click to toggle source
# File lib/chef/cookbook/synchronizer.rb, line 100
def cache
  Chef::FileCache
end
cached_copy_up_to_date?(local_path, expected_checksum) click to toggle source
# File lib/chef/cookbook/synchronizer.rb, line 321
def cached_copy_up_to_date?(local_path, expected_checksum)
  return true if Chef::Config[:skip_cookbook_sync]

  if cache.key?(local_path)
    current_checksum = CookbookVersion.checksum_cookbook_file(cache.load(local_path, false))
    expected_checksum == current_checksum
  else
    false
  end
end
clear_obsoleted_cookbooks() click to toggle source

Iterates over cached cookbooks’ files, removing files belonging to cookbooks that don’t appear in cookbook_hash

# File lib/chef/cookbook/synchronizer.rb, line 262
def clear_obsoleted_cookbooks
  @events.cookbook_clean_start

  if remove_obsoleted_files
    remove_old_cookbooks
  else
    Chef::Log.info("Skipping removal of obsoleted cookbooks from the cache")
    CookbookCacheCleaner.instance.skip_removal = true
  end

  remove_deleted_files

  @events.cookbook_clean_complete
end
cookbook_count() click to toggle source
# File lib/chef/cookbook/synchronizer.rb, line 112
def cookbook_count
  @cookbooks_by_name.size
end
cookbook_names() click to toggle source
# File lib/chef/cookbook/synchronizer.rb, line 104
def cookbook_names
  @cookbooks_by_name.keys
end
cookbook_segment(cookbook_name, segment) click to toggle source
# File lib/chef/cookbook/synchronizer.rb, line 120
def cookbook_segment(cookbook_name, segment)
  @cookbooks_by_name[cookbook_name].files_for(segment)
end
cookbooks() click to toggle source
# File lib/chef/cookbook/synchronizer.rb, line 108
def cookbooks
  @cookbooks_by_name.values
end
download_file(url, destination) click to toggle source

Unconditionally download the file from the given URL. File will be downloaded to the path destination which is relative to the Chef file cache root.

# File lib/chef/cookbook/synchronizer.rb, line 335
def download_file(url, destination)
  raw_file = server_api.streaming_request(url)

  Chef::Log.info("Storing updated #{destination} in the cache.")
  cache.move_to(raw_file.path, destination)
end
ensure_cookbook_paths() click to toggle source
# File lib/chef/cookbook/synchronizer.rb, line 287
def ensure_cookbook_paths
  cookbook_path = File.join(Chef::Config[:file_cache_path], "cookbooks")
  cookbooks.each do |cookbook|
    cb_dir = File.join(cookbook_path, cookbook.name)
    cookbook.root_paths = Array(cb_dir)
  end
end
files() click to toggle source
# File lib/chef/cookbook/synchronizer.rb, line 124
def files
  lazy = unless Chef::Config[:no_lazy_load]
           %w{ files templates }
         else
           []
         end

  @files ||= cookbooks.inject([]) do |memo, cookbook|
    cookbook.each_file do |manifest_record|
      part = manifest_record[:name].split("/")[0]
      if lazy.include?(part)
        manifest_record[:lazy] = true
        @lazy_files[cookbook] ||= []
        @lazy_files[cookbook] << manifest_record
      else
        memo << CookbookFile.new(cookbook, manifest_record)
      end
    end
    memo
  end
end
files_by_cookbook() click to toggle source
# File lib/chef/cookbook/synchronizer.rb, line 146
def files_by_cookbook
  files.group_by(&:cookbook)
end
files_remaining_by_cookbook() click to toggle source
# File lib/chef/cookbook/synchronizer.rb, line 150
def files_remaining_by_cookbook
  @files_remaining_by_cookbook ||= files_by_cookbook.inject({}) do |memo, (cookbook, files)|
    memo[cookbook] = files.size
    memo
  end
end
have_cookbook?(cookbook_name) click to toggle source
# File lib/chef/cookbook/synchronizer.rb, line 116
def have_cookbook?(cookbook_name)
  @cookbooks_by_name.key?(cookbook_name)
end
mark_cached_file_valid(cache_filename) click to toggle source

Marks the given file as valid (non-stale).

# File lib/chef/cookbook/synchronizer.rb, line 343
def mark_cached_file_valid(cache_filename)
  CookbookCacheCleaner.instance.mark_file_as_valid(cache_filename)
end
mark_file_synced(file) click to toggle source
# File lib/chef/cookbook/synchronizer.rb, line 157
def mark_file_synced(file)
  files_remaining_by_cookbook[file.cookbook] -= 1

  if files_remaining_by_cookbook[file.cookbook] == 0
    @events.synchronized_cookbook(file.cookbook.name, file.cookbook)
  end
end
remove_deleted_files() click to toggle source

remove deleted files in cookbooks that are being used on the node

# File lib/chef/cookbook/synchronizer.rb, line 226
def remove_deleted_files
  cache_file_hash = {}
  @cookbooks_by_name.each_key do |k|
    cache_file_hash[k] = {}
  end

  # First populate files from cache
  cache.find(File.join(%w{cookbooks ** {*,.*}})).each do |cache_file|
    md = cache_file.match(%r{^cookbooks/([^/]+)/([^/]+)/(.*)})
    next unless md

    (cookbook_name, segment, file) = md[1..3]
    if have_cookbook?(cookbook_name)
      cache_file_hash[cookbook_name][segment] ||= {}
      cache_file_hash[cookbook_name][segment]["#{segment}/#{file}"] = cache_file
    end
  end
  # Determine which files don't match manifest
  @cookbooks_by_name.each_key do |cookbook_name|
    cache_file_hash[cookbook_name].each_key do |segment|
      manifest_segment = cookbook_segment(cookbook_name, segment)
      manifest_record_paths = manifest_segment.map { |manifest_record| manifest_record["path"] }
      to_be_removed = cache_file_hash[cookbook_name][segment].keys - manifest_record_paths
      to_be_removed.each do |path|
        cache_file = cache_file_hash[cookbook_name][segment][path]

        Chef::Log.info("Removing #{cache_file} from the cache; its is no longer in the cookbook manifest.")
        cache.delete(cache_file)
        @events.removed_cookbook_file(cache_file)
      end
    end
  end
end
remove_old_cookbooks() click to toggle source

remove cookbooks that are not referenced in the expanded run_list at all (if we have an override run_list we may not want to do this)

# File lib/chef/cookbook/synchronizer.rb, line 214
def remove_old_cookbooks
  cache.find(File.join(%w{cookbooks ** {*,.*}})).each do |cache_file|
    cache_file =~ %r{^cookbooks/([^/]+)/}
    unless have_cookbook?($1)
      Chef::Log.info("Removing #{cache_file} from the cache; its cookbook is no longer needed on this client.")
      cache.delete(cache_file)
      @events.removed_cookbook_file(cache_file)
    end
  end
end
save_full_file_path(file, full_path) click to toggle source

Saves the full_path to the file of the cookbook to be updated in the manifest later

# File lib/chef/cookbook/synchronizer.rb, line 207
def save_full_file_path(file, full_path)
  @cookbook_full_file_paths[file.cookbook] ||= []
  @cookbook_full_file_paths[file.cookbook] << full_path
end
server_api() click to toggle source
# File lib/chef/cookbook/synchronizer.rb, line 347
def server_api
  Thread.current[:server_api] ||= Chef::ServerAPI.new(Chef::Config[:chef_server_url], keepalives: true)
end
sync_cookbooks() click to toggle source

Synchronizes all the cookbooks from the chef-server. )

Returns

true

Always returns true

# File lib/chef/cookbook/synchronizer.rb, line 169
def sync_cookbooks
  Chef::Log.info("Loading cookbooks [#{cookbooks.map { |ckbk| ckbk.name + "@" + ckbk.version }.join(", ")}]")
  Chef::Log.trace("Cookbooks detail: #{cookbooks.inspect}")

  clear_obsoleted_cookbooks

  queue = Chef::Util::ThreadedJobQueue.new

  Chef::Log.warn("skipping cookbook synchronization! DO NOT LEAVE THIS ENABLED IN PRODUCTION!!!") if Chef::Config[:skip_cookbook_sync]
  files.each do |file|
    queue << lambda do |lock|
      full_file_path = sync_file(file)

      lock.synchronize do
        # Save the full_path of the downloaded file to be restored in the manifest later
        save_full_file_path(file, full_file_path)
        mark_file_synced(file)
      end
    end
  end

  @events.cookbook_sync_start(cookbook_count)
  queue.process(Chef::Config[:cookbook_sync_threads])
  # Ensure that cookbooks know where they're rooted at, for manifest purposes.
  ensure_cookbook_paths
  # Update the full file paths in the manifest
  update_cookbook_filenames

rescue Exception => e
  @events.cookbook_sync_failed(cookbooks, e)
  raise
else
  @events.cookbook_sync_complete
  true
end
sync_file(file) click to toggle source

Sync an individual file if needed. If there is an up to date copy locally, nothing is done. Updates file‘s manifest with the full path to the cached file.

Arguments

file<CookbookFile>

Returns

Full path to the cached file as a String

# File lib/chef/cookbook/synchronizer.rb, line 303
def sync_file(file)
  cache_filename = File.join("cookbooks", file.cookbook.name, file.manifest_record["path"])
  mark_cached_file_valid(cache_filename)

  # If the checksums are different between on-disk (current) and on-server
  # (remote, per manifest), do the update. This will also execute if there
  # is no current checksum.
  if !cached_copy_up_to_date?(cache_filename, file.manifest_record["checksum"])
    download_file(file.manifest_record["url"], cache_filename)
    @events.updated_cookbook_file(file.cookbook.name, cache_filename)
  else
    Chef::Log.trace("Not storing #{cache_filename}, as the cache is up to date.")
  end

  # Load the file in the cache and return the full file path to the loaded file
  cache.load(cache_filename, false)
end
update_cookbook_filenames() click to toggle source
# File lib/chef/cookbook/synchronizer.rb, line 277
def update_cookbook_filenames
  @cookbook_full_file_paths.each do |cookbook, full_paths|
    cookbook.all_files = full_paths
  end

  @lazy_files.each do |cookbook, lazy_files|
    cookbook.cookbook_manifest.add_files_to_manifest(lazy_files)
  end
end