class Bosh::Director::DeploymentPlan::Steps::PackageCompileStep

Attributes

compilations_performed[R]

Public Class Methods

new(jobs_to_compile, compilation_config, compilation_instance_pool, logger, director_job) click to toggle source
# File lib/bosh/director/deployment_plan/steps/package_compile_step.rb, line 12
def initialize(jobs_to_compile, compilation_config, compilation_instance_pool, logger, director_job)
  @event_log_stage = nil
  @logger = logger
  @director_job = director_job

  @tasks_mutex = Mutex.new
  @counter_mutex = Mutex.new

  @compilation_instance_pool = compilation_instance_pool
  @compile_tasks = {}
  @ready_tasks = []
  @compilations_performed = 0
  @jobs_to_compile = jobs_to_compile
  @compilation_config = compilation_config
end

Public Instance Methods

compile_package(task) click to toggle source
# File lib/bosh/director/deployment_plan/steps/package_compile_step.rb, line 55
def compile_package(task)
  package = task.package
  stemcell = task.stemcell

  with_compile_lock(package.id, stemcell.id) do
    # Check if the package was compiled in a parallel deployment
    compiled_package = task.find_compiled_package(@logger, @event_log_stage)
    if compiled_package.nil?
      build = Models::CompiledPackage.generate_build_number(package, stemcell.model.operating_system, stemcell.model.version)
      task_result = nil

      prepare_vm(stemcell) do |instance|
        vm_metadata_updater.update(instance.model, :compiling => package.name)
        agent_task =
          instance.agent_client.compile_package(
            package.blobstore_id,
            package.sha1,
            package.name,
            "#{package.version}.#{build}",
            task.dependency_spec
          )

        task_result = agent_task['result']
      end

      compiled_package = Models::CompiledPackage.create do |p|
        p.package = package
        p.stemcell_os = stemcell.os
        p.stemcell_version = stemcell.version
        p.sha1 = task_result['sha1']
        p.build = build
        p.blobstore_id = task_result['blobstore_id']
        p.dependency_key = task.dependency_key
      end

      if Config.use_compiled_package_cache?
        if BlobUtil.exists_in_global_cache?(package, task.cache_key)
          @logger.info('Already exists in global package cache, skipping upload')
        else
          @logger.info('Uploading to global package cache')
          BlobUtil.save_to_global_cache(compiled_package, task.cache_key)
        end
      else
        @logger.info('Global blobstore not configured, skipping upload')
      end

      @counter_mutex.synchronize { @compilations_performed += 1 }
    end

    task.use_compiled_package(compiled_package)
  end
end
compile_tasks_count() click to toggle source
# File lib/bosh/director/deployment_plan/steps/package_compile_step.rb, line 47
def compile_tasks_count
  @compile_tasks.size
end
perform() click to toggle source
# File lib/bosh/director/deployment_plan/steps/package_compile_step.rb, line 28
def perform
  @logger.info('Generating a list of compile tasks')
  prepare_tasks

  @compile_tasks.each_value do |task|
    if task.ready_to_compile?
      @logger.info("Package '#{task.package.desc}' is ready to be compiled for stemcell '#{task.stemcell.desc}'")
      @ready_tasks << task
    end
  end

  if @ready_tasks.empty?
    @logger.info('All packages are already compiled')
  else
    compile_packages
    director_job_checkpoint
  end
end
prepare_vm(stemcell) click to toggle source

This method will create a VM for each stemcell in the stemcells array passed in. The VMs are yielded and their destruction is ensured. @param [Models::Stemcell] stemcell The stemcells that need to have

compilation VMs created.

@yield [DeploymentPlan::Instance] Yields an instance that should be used for compilation. This may be a reused VM or a

# File lib/bosh/director/deployment_plan/steps/package_compile_step.rb, line 113
def prepare_vm(stemcell)
  if @compilation_config.reuse_compilation_vms
    @compilation_instance_pool.with_reused_vm(stemcell, &Proc.new)
  else
    @compilation_instance_pool.with_single_use_vm(stemcell, &Proc.new)
  end
end
ready_tasks_count() click to toggle source
# File lib/bosh/director/deployment_plan/steps/package_compile_step.rb, line 51
def ready_tasks_count
  @tasks_mutex.synchronize { @ready_tasks.size }
end

Private Instance Methods

compilation_count() click to toggle source
# File lib/bosh/director/deployment_plan/steps/package_compile_step.rb, line 220
def compilation_count
  counter = 0
  @compile_tasks.each_value do |task|
    counter += 1 unless task.compiled?
  end
  counter
end
compile_packages() click to toggle source
# File lib/bosh/director/deployment_plan/steps/package_compile_step.rb, line 150
def compile_packages
  @event_log_stage = Config.event_log.begin_stage('Compiling packages', compilation_count)

  begin
    ThreadPool.new(:max_threads => @compilation_config.workers).wrap do |pool|
      loop do
        # process as many tasks without waiting
        loop do
          break if director_job_cancelled?
          task = @tasks_mutex.synchronize { @ready_tasks.pop }
          break if task.nil?

          pool.process { process_task(task) }
        end

        break if !pool.working? && (director_job_cancelled? || @ready_tasks.empty?)
        sleep(0.1)
      end
    end
  ensure
    # Delete all of the VMs if we were reusing compilation VMs. This can't
    # happen until everything was done compiling.
    if @compilation_config.reuse_compilation_vms
      # Using a new ThreadPool instead of reusing the previous one,
      # as if there's a failed compilation, the thread pool will stop
      # processing any new thread.
      @compilation_instance_pool.delete_instances(@compilation_config.workers)
    end
  end
end
director_job_cancelled?() click to toggle source
# File lib/bosh/director/deployment_plan/steps/package_compile_step.rb, line 212
def director_job_cancelled?
  @director_job && @director_job.task_cancelled?
end
director_job_checkpoint() click to toggle source
# File lib/bosh/director/deployment_plan/steps/package_compile_step.rb, line 216
def director_job_checkpoint
  @director_job.task_checkpoint if @director_job
end
enqueue_unblocked_tasks(task) click to toggle source
# File lib/bosh/director/deployment_plan/steps/package_compile_step.rb, line 181
def enqueue_unblocked_tasks(task)
  @tasks_mutex.synchronize do
    @logger.info("Unblocking dependents of '#{task.package.desc}' for '#{task.stemcell.desc}'")
    task.dependent_tasks.each do |dep_task|
      if dep_task.ready_to_compile?
        @logger.info("Package '#{dep_task.package.desc}' now ready to be compiled for '#{dep_task.stemcell.desc}'")
        @ready_tasks << dep_task
      end
    end
  end
end
prepare_tasks() click to toggle source
# File lib/bosh/director/deployment_plan/steps/package_compile_step.rb, line 123
def prepare_tasks
  @event_log_stage = Config.event_log.begin_stage('Preparing package compilation', 1)
  @compile_task_generator = CompileTaskGenerator.new(@logger, @event_log_stage)

  @event_log_stage.advance_and_track('Finding packages to compile') do
    @jobs_to_compile.each do |job|
      stemcell = job.stemcell

      template_descs = job.templates.map do |t|
        # we purposefully did NOT inline those because
        # when instance_double blows up,
        # it's obscure which double is at fault
        release_name = t.release.name
        template_name = t.name
        "'#{release_name}/#{template_name}'"
      end
      @logger.info("Job templates #{template_descs.join(', ')} need to run on stemcell '#{stemcell.desc}'")

      job.templates.each do |template|
        template.package_models.each do |package|
          @compile_task_generator.generate!(@compile_tasks, job, template, package, stemcell)
        end
      end
    end
  end
end
process_task(task) click to toggle source
# File lib/bosh/director/deployment_plan/steps/package_compile_step.rb, line 193
def process_task(task)
  package_desc = task.package.desc
  stemcell_desc = task.stemcell.desc
  task_desc = "package '#{package_desc}' for stemcell '#{stemcell_desc}'"

  with_thread_name("compile_package(#{package_desc}, #{stemcell_desc})") do
    if director_job_cancelled?
      @logger.info("Cancelled compiling #{task_desc}")
    else
      @event_log_stage.advance_and_track(package_desc) do
        @logger.info("Compiling #{task_desc}")
        compile_package(task)
        @logger.info("Finished compiling #{task_desc}")
        enqueue_unblocked_tasks(task)
      end
    end
  end
end
vm_metadata_updater() click to toggle source
# File lib/bosh/director/deployment_plan/steps/package_compile_step.rb, line 228
def vm_metadata_updater
  @vm_metadata_updater ||= VmMetadataUpdater.build
end