class Autoproj::Ops::Import

Constants

VALID_OSDEP_AVAILABILITY

Attributes

auto_exclude[W]
ws[R]

Public Class Methods

new(ws, report_path: nil) click to toggle source
# File lib/autoproj/ops/import.rb, line 15
def initialize(ws, report_path: nil)
    @ws = ws
    @auto_exclude = false
    @report_path = report_path
end

Public Instance Methods

auto_exclude?() click to toggle source

Whether packages are added to exclusions if they error during the import process

This is mostly meant for CI operations

# File lib/autoproj/ops/import.rb, line 10
def auto_exclude?
    @auto_exclude
end
create_report(package_list) click to toggle source
# File lib/autoproj/ops/import.rb, line 530
def create_report(package_list)
    FileUtils.mkdir_p File.dirname(@report_path)

    packages = package_list.each_with_object({}) do |pkg_name, h|
        pkg = @ws.manifest.find_autobuild_package(pkg_name)

        h[pkg.name] = {
            invoked: !!pkg.import_invoked?,
            success: !!pkg.imported?
        }
    end

    report = JSON.pretty_generate(
        {
            import_report: {
                timestamp: Time.now,
                packages: packages
            }
        }
    )
    File.write(@report_path, report)
end
finalize_package_load(processed_packages, ignore_optional_dependencies: false, auto_exclude: auto_exclude?) click to toggle source
# File lib/autoproj/ops/import.rb, line 389
def finalize_package_load(processed_packages,
    ignore_optional_dependencies: false,
    auto_exclude: auto_exclude?)
    manifest = ws.manifest

    all = Set.new
    package_queue =
        manifest.all_layout_packages(false)
                .each_source_package_name.to_a +
        processed_packages.map(&:name).to_a

    until package_queue.empty?
        pkg_name = package_queue.shift
        next if all.include?(pkg_name)

        all << pkg_name
        next if manifest.ignored?(pkg_name) || manifest.excluded?(pkg_name)

        pkg_definition = manifest.find_package_definition(pkg_name)
        pkg = pkg_definition.autobuild
        if !processed_packages.include?(pkg_definition) && pkg.checked_out?
            begin
                manifest.load_package_manifest(pkg.name)
                process_post_import_blocks(pkg)
            rescue Exception => e
                raise unless auto_exclude

                manifest.exclude_package(
                    pkg.name, "#{pkg.name} had an error when "\
                              "being loaded (#{e.message}) and "\
                              "auto_exclude is true"
                )
                next
            end
        end

        packages, osdeps = pkg.partition_optional_dependencies
        (packages + osdeps).each do |dep_pkg_name|
            next if manifest.ignored?(dep_pkg_name)
            next if manifest.excluded?(dep_pkg_name)

            pkg.depends_on dep_pkg_name
        end

        if File.directory?(pkg.srcdir)
            pkg.prepare
            Rake::Task["#{pkg.name}-prepare"]
                .instance_variable_set(:@already_invoked, true)
        end
        pkg.update_environment
        package_queue.concat(pkg.dependencies)
    end
    all
end
import_next_step(pkg, reverse_dependencies) click to toggle source
# File lib/autoproj/ops/import.rb, line 45
def import_next_step(pkg, reverse_dependencies)
    new_packages = []
    pkg.dependencies.each do |dep_name|
        reverse_dependencies[dep_name] << pkg.name
        new_packages << ws.manifest.find_package_definition(dep_name)
    end
    pkg_opt_deps, pkg_opt_os_deps = pkg.partition_optional_dependencies
    pkg_opt_deps.each do |dep_name|
        new_packages << ws.manifest.find_package_definition(dep_name)
    end

    # Handle OS dependencies, excluding the package if some
    # dependencies are not available
    pkg.os_packages.each do |dep_name|
        reverse_dependencies[dep_name] << pkg.name
    end
    (pkg.os_packages | pkg_opt_os_deps).each do |dep_name|
        if ws.manifest.excluded?(dep_name)
            mark_exclusion_along_revdeps(dep_name, reverse_dependencies)
        end
    end

    new_packages.delete_if do |new_pkg|
        if ws.manifest.excluded?(new_pkg.name)
            mark_exclusion_along_revdeps(new_pkg.name, reverse_dependencies)
            true
        elsif ws.manifest.ignored?(new_pkg.name)
            true
        end
    end
    new_packages
end
import_packages(selection, non_imported_packages: :checkout, warn_about_ignored_packages: true, warn_about_excluded_packages: true, recursive: true, keep_going: false, install_vcs_packages: Hash.new, auto_exclude: auto_exclude?, filter: ->(pkg) { true } click to toggle source
# File lib/autoproj/ops/import.rb, line 444
def import_packages(selection,
    non_imported_packages: :checkout,
    warn_about_ignored_packages: true,
    warn_about_excluded_packages: true,
    recursive: true,
    keep_going: false,
    install_vcs_packages: Hash.new,
    auto_exclude: auto_exclude?,
    filter: ->(pkg) { true },
    **import_options)

    manifest = ws.manifest

    all_processed_packages, failures = import_selected_packages(
        selection,
        non_imported_packages: non_imported_packages,
        keep_going: keep_going,
        recursive: recursive,
        install_vcs_packages: install_vcs_packages,
        auto_exclude: auto_exclude,
        filter: filter,
        **import_options
    )

    raise failures.first if !keep_going && !failures.empty?

    install_internal_dependencies_for(
        *all_processed_packages, install_only: import_options[:checkout_only]
    )
    finalize_package_load(all_processed_packages, auto_exclude: auto_exclude)

    all_enabled_osdeps = selection.each_osdep_package_name.to_set
    all_enabled_sources = all_processed_packages.map(&:name)
    if recursive
        all_processed_packages.each do |pkg|
            all_enabled_osdeps.merge(pkg.autobuild.os_packages)
        end
    end

    if warn_about_excluded_packages
        selection.exclusions.each do |sel, pkg_names|
            pkg_names.sort.each do |pkg_name|
                Autoproj.warn "#{pkg_name}, which was selected "\
                              "for #{sel}, cannot be built: "\
                              "#{manifest.exclusion_reason(pkg_name)}", :bold
            end
        end
    end
    if warn_about_ignored_packages
        selection.ignores.each do |sel, pkg_names|
            pkg_names.sort.each do |pkg_name|
                Autoproj.warn "#{pkg_name}, which was selected for #{sel}, "\
                              "is ignored", :bold
            end
        end
    end

    if failures.empty?
        [all_enabled_sources, all_enabled_osdeps]
    else
        raise PackageImportFailed.new(
            failures, source_packages: all_enabled_sources,
                      osdep_packages: all_enabled_osdeps
        )
    end
ensure
    create_report(all_processed_packages || []) if @report_path

    update_log = ws.config.import_log_enabled? &&
                 Autoproj::Ops::Snapshot.update_log_available?(manifest)
    if update_log
        update_log_for_processed_packages(
            all_processed_packages || Array.new, $!
        )
    end
end
import_selected_packages(selection, parallel: ws.config.parallel_import_level, recursive: true, retry_count: nil, keep_going: false, install_vcs_packages: Hash.new, non_imported_packages: :checkout, auto_exclude: auto_exclude?, filter: ->(package) { true } click to toggle source

Import all packages from the given selection, and their dependencies

# File lib/autoproj/ops/import.rb, line 196
def import_selected_packages(selection,
    parallel: ws.config.parallel_import_level,
    recursive: true,
    retry_count: nil,
    keep_going: false,
    install_vcs_packages: Hash.new,
    non_imported_packages: :checkout,
    auto_exclude: auto_exclude?,
    filter: ->(package) { true },
    **import_options)

    unless %i[checkout ignore return].include?(non_imported_packages)
        raise ArgumentError, "invalid value for 'non_imported_packages'. "\
                             "Expected one of :checkout, :ignore or :return "\
                             "but got #{non_imported_packages}"
    end

    # This is used in the ensure block, initialize as early as
    # possible
    executor = Concurrent::FixedThreadPool.new(parallel)
    manifest = ws.manifest

    selected_packages = selection.each_source_package_name.map do |pkg_name|
        manifest.find_package_definition(pkg_name)
    end.to_set

    # The reverse dependencies for the package tree. It is discovered as
    # we go on with the import
    #
    # It only contains strong dependencies. Optional dependencies are
    # not included, as we will use this only to take into account
    # package exclusion (and that does not affect optional dependencies)
    reverse_dependencies = Hash.new { |h, k| h[k] = Set.new }

    completion_queue = Queue.new
    pending_packages = Set.new
    # The set of all packages that are currently selected by +selection+
    all_processed_packages = Set.new
    main_thread_imports = Array.new
    package_queue = selected_packages.to_a.sort_by(&:name)

    failures = Array.new
    missing_vcs = Array.new
    installed_vcs_packages = Set["none", "local"]
    while failures.empty? || keep_going
        # Allow 'filter' to parallelize as well
        if filter.respond_to?(:lookahead)
            package_queue.each { |pkg| filter.lookahead(pkg) }
        end

        # Queue work for all packages in the queue
        package_queue.each do |pkg|
            # Remove packages that have already been processed
            next if all_processed_packages.include?(pkg)

            vcs_installed = installed_vcs_packages.include?(pkg.vcs.type)
            if (non_imported_packages != :checkout) && !pkg.checked_out?
                all_processed_packages << pkg
                if non_imported_packages == :return
                    completion_queue << [pkg, Time.now, false, nil]
                else
                    ws.manifest.ignore_package(pkg.name)
                end
                next
            elsif install_vcs_packages && !vcs_installed
                missing_vcs << pkg
                next
            end
            all_processed_packages << pkg

            unless filter.call(pkg)
                completion_queue << [pkg, Time.now]
                next
            end

            importer = pkg.autobuild.importer
            if !pre_package_import(selection, manifest, pkg.autobuild,
                                   reverse_dependencies)
                next
            elsif !importer
                # The validity of this is checked in
                # pre_package_import
                completion_queue << [pkg, Time.now, false, nil]
                next
            elsif importer.interactive?
                main_thread_imports << pkg
                next
            elsif pkg.checked_out? && import_options[:checkout_only]
                main_thread_imports << pkg
                next
            end

            begin
                pending_packages << pkg
                queue_import_work(
                    executor, completion_queue, pkg,
                    retry_count: retry_count,
                    **import_options.merge(allow_interactive: false)
                )
            rescue Exception
                pending_packages.delete(pkg)
                raise
            end
        end
        package_queue.clear

        if completion_queue.empty? && pending_packages.empty?
            unless missing_vcs.empty?
                installed_vcs_packages.merge(
                    install_vcs_packages_for(
                        *missing_vcs,
                        install_only: import_options[:checkout_only],
                        **install_vcs_packages
                    )
                )
                package_queue.concat(missing_vcs)
                missing_vcs.clear
                next
            end

            # We've nothing to process anymore ... process
            # interactive imports if there are some. Otherwise,
            # we're done
            if main_thread_imports.empty?
                break
            else
                main_thread_imports.delete_if do |pkg|
                    begin
                        if retry_count
                            pkg.autobuild.importer.retry_count = retry_count
                        end
                        result = pkg.autobuild.import(
                            **import_options.merge(allow_interactive: true)
                        )
                    rescue StandardError => e
                    end
                    completion_queue << [pkg,
                                         Time.now, result, e]
                end
            end
        end

        # And wait for one to finish
        pkg, _time, _result, reason = completion_queue.pop
        pending_packages.delete(pkg)
        if reason
            if reason.kind_of?(Autobuild::InteractionRequired)
                main_thread_imports << pkg
            elsif auto_exclude
                manifest.add_exclusion(
                    pkg.name, "#{pkg.name} failed to import with "\
                              "#{reason} and auto_exclude was true"
                )
                selection.filter_excluded_and_ignored_packages(manifest)
            else
                # One importer failed... terminate
                Autoproj.error "import of #{pkg.name} failed"
                Autoproj.error reason.to_s unless reason.kind_of?(Interrupt)
                failures << reason
            end
        else
            new_packages = post_package_import(
                selection, manifest, pkg, reverse_dependencies,
                auto_exclude: auto_exclude
            )
            if new_packages
                # Excluded dependencies might have caused the package to be
                # excluded as well ... do not add any dependency to the
                # processing queue if it is the case
                if manifest.excluded?(pkg.name)
                    selection.filter_excluded_and_ignored_packages(manifest)
                elsif recursive
                    package_queue = new_packages.sort_by(&:name)
                end
            end
        end
    end

    all_processed_packages.delete_if do |processed_pkg|
        ws.manifest.excluded?(processed_pkg.name) ||
            ws.manifest.ignored?(processed_pkg.name)
    end
    [all_processed_packages, failures]
ensure
    if failures && !failures.empty? && !keep_going
        Autoproj.error "waiting for pending import jobs to finish"
    end
    if executor
        executor.shutdown
        executor.wait_for_termination
    end
end
install_internal_dependencies_for(*packages, **osdeps_options) click to toggle source

Install the internal dependencies for the given packages

@param [Hash] osdeps_options the options that will be passed to

{Workspace#install_os_packages}

@return [Set] the set of installed OS packages

# File lib/autoproj/ops/import.rb, line 151
def install_internal_dependencies_for(*packages, **osdeps_options)
    packages_to_install = packages.map do |pkg|
        pkg.autobuild.internal_dependencies
    end.flatten.uniq
    return if packages_to_install.empty?

    # This assumes that the internal dependencies do not depend on a
    # 'strict' package mangers such as e.g. BundlerManager and that
    # the package manager itself does not have any dependencies
    ws.install_os_packages(packages_to_install, all: nil, **osdeps_options)
    packages_to_install
end
install_vcs_packages_for(*packages, **osdeps_options) click to toggle source

Install the VCS osdep for the given packages

@param [Hash] osdeps_options the options that will be passed to

{Workspace#install_os_packages}

@return [Set] the set of installed OS packages

# File lib/autoproj/ops/import.rb, line 138
def install_vcs_packages_for(*packages, **osdeps_options)
    vcs_to_install = packages.map { |pkg| pkg.vcs.type }.uniq
    # This assumes that the VCS packages do not depend on a
    # 'strict' package mangers such as e.g. BundlerManager
    ws.install_os_packages(vcs_to_install, all: nil, **osdeps_options)
    vcs_to_install
end
mark_exclusion_along_revdeps(pkg_name, revdeps, chain = [], reason = nil) click to toggle source
# File lib/autoproj/ops/import.rb, line 21
def mark_exclusion_along_revdeps(pkg_name, revdeps, chain = [], reason = nil)
    root = !reason
    chain.unshift pkg_name
    if root
        reason = ws.manifest.exclusion_reason(pkg_name)
    elsif chain.size == 1
        ws.manifest.exclude_package(pkg_name, "its dependency #{reason}")
    else
        ws.manifest.exclude_package(pkg_name, "#{reason} (dependency chain: "\
                                              "#{chain.join('>')})")
    end

    return unless revdeps.key?(pkg_name)

    revdeps[pkg_name].each do |dep_name|
        unless ws.manifest.excluded?(dep_name)
            mark_exclusion_along_revdeps(dep_name, revdeps, chain.dup, reason)
        end
    end
end
post_package_import(selection, manifest, pkg, reverse_dependencies, auto_exclude: auto_exclude?) click to toggle source
# File lib/autoproj/ops/import.rb, line 96
def post_package_import(selection, manifest, pkg, reverse_dependencies,
    auto_exclude: auto_exclude?)
    Rake::Task["#{pkg.name}-import"]
        .instance_variable_set(:@already_invoked, true)
    if pkg.checked_out?
        begin
            manifest.load_package_manifest(pkg.name)
        rescue StandardError => e
            raise unless auto_exclude

            manifest.add_exclusion(
                pkg.name, "#{pkg.name} failed to import "\
                          "with #{e} and auto_exclude was true"
            )
        end
    end

    if !manifest.excluded?(pkg.name) && !manifest.ignored?(pkg.name)
        process_post_import_blocks(pkg) if pkg.checked_out?
    end

    # The package setup mechanisms might have added an exclusion
    # on this package. Handle this.
    if manifest.excluded?(pkg.name)
        mark_exclusion_along_revdeps(pkg.name, reverse_dependencies)
        # Run a filter now, to have errors as early as possible
        selection.filter_excluded_and_ignored_packages(manifest)
        # Delete this package from the current_packages set
        false
    elsif manifest.ignored?(pkg.name)
        false
    else
        pkg.apply_dependencies_from_manifest
        import_next_step(pkg.autobuild, reverse_dependencies)
    end
end
pre_package_import(selection, manifest, pkg, reverse_dependencies) click to toggle source
# File lib/autoproj/ops/import.rb, line 78
def pre_package_import(selection, manifest, pkg, reverse_dependencies)
    # Try to auto-exclude the package early. If the autobuild file
    # contained some information that allows us to exclude it now,
    # then let's just do it
    import_next_step(pkg, reverse_dependencies)
    if manifest.excluded?(pkg.name)
        selection.filter_excluded_and_ignored_packages(manifest)
        false
    elsif manifest.ignored?(pkg.name)
        false
    elsif !pkg.importer && !File.directory?(pkg.srcdir)
        raise ConfigError.new, "#{pkg.name} has no VCS, but is not "\
                               "checked out in #{pkg.srcdir}"
    else
        true
    end
end
process_post_import_blocks(pkg) click to toggle source

Call post-import blcoks registered for the given package

@param [PackageDefinition] pkg

# File lib/autoproj/ops/import.rb, line 524
def process_post_import_blocks(pkg)
    Autoproj.each_post_import_block(pkg) do |block|
        block.call(pkg)
    end
end
queue_import_work(executor, completion_queue, pkg, retry_count: nil, **import_options) click to toggle source

@api private

Queue the work necessary to import the given package, making sure that the execution results end up in a given queue

@param executor the future executor @param [Queue] completion_queue the queue where the completion

results should be pushed, as a (package, time, result,
error_reason) tuple

@param [Integer] retry_count the number of retries that are

allowed. Set to zero for no retry

@param [Hash] import_options options passed to {Autobuild::Importer#import}

# File lib/autoproj/ops/import.rb, line 176
def queue_import_work(executor, completion_queue, pkg,
    retry_count: nil, **import_options)
    import_future =
        Concurrent::Promises.future_on(executor) do
            ## COMPLETELY BYPASS RAKE HERE
            # The reason is that the ordering of import/prepare between
            # packages is not important BUT the ordering of import vs.
            # prepare in one package IS important: prepare is the method
            # that takes into account dependencies.
            pkg.autobuild.importer.retry_count = retry_count if retry_count
            pkg.autobuild.import(**import_options)
        end

    import_future.on_resolution! do |time, result, reason|
        completion_queue << [pkg, time, result, reason]
    end
end
update_log_for_processed_packages(all_processed_packages, exception) click to toggle source
# File lib/autoproj/ops/import.rb, line 553
def update_log_for_processed_packages(all_processed_packages, exception)
    all_updated_packages = all_processed_packages.find_all do |processed_pkg|
        processed_pkg.autobuild.updated?
    end

    unless all_updated_packages.empty?
        failure_message =
            (" (#{exception.message.split("\n").first})" if exception)
        ops = Ops::Snapshot.new(ws.manifest, keep_going: true)
        ops.update_package_import_state(
            "#{$0} #{ARGV.join(' ')}#{failure_message}",
            all_updated_packages.map(&:name)
        )
    end
end