class Autoproj::CLI::CI
Actual implementation of the functionality for the `autoproj ci` subcommand
Autoproj
internally splits the CLI
definition (Thor subclass) and the underlying functionality of each CLI
subcommand. `autoproj-ci` follows the same pattern, and registers its subcommand in {MainCI} while implementing the functionality in this class
Constants
- PHASES
Public Instance Methods
# File lib/autoproj/cli/ci.rb, line 41 def cache_pull(dir, ignore: []) packages = resolve_packages memo = {} results = packages.each_with_object({}) do |pkg, h| if ignore.include?(pkg.name) pkg.message '%s: ignored by command line' fingerprint = pkg.fingerprint(memo: memo) h[pkg.name] = { 'cached' => false, 'fingerprint' => fingerprint } next end state, fingerprint, metadata = pull_package_from_cache(dir, pkg, memo: memo) if state pkg.message "%s: pulled #{fingerprint}", :green else pkg.message "%s: #{fingerprint} not in cache, "\ 'or not pulled from cache' end h[pkg.name] = metadata.merge( 'cached' => state, 'fingerprint' => fingerprint ) end hit = results.count { |_, info| info['cached'] } Autoproj.message "#{hit} hits, #{results.size - hit} misses" results end
# File lib/autoproj/cli/ci.rb, line 77 def cache_push(dir) packages = resolve_packages metadata = consolidated_report['packages'] memo = {} results = packages.each_with_object({}) do |pkg, h| if !(pkg_metadata = metadata[pkg.name]) pkg.message '%s: no metadata in build report', :magenta next elsif !(build_info = pkg_metadata['build']) pkg.message '%s: no build info in build report', :magenta next elsif build_info['cached'] pkg.message '%s: was pulled from cache, not pushing' next elsif !build_info['success'] pkg.message '%s: build failed, not pushing', :magenta next end # Remove cached flags before saving pkg_metadata = pkg_metadata.dup PHASES.each do |phase_name| pkg_metadata[phase_name]&.delete('cached') end state, fingerprint = push_package_to_cache( dir, pkg, pkg_metadata, force: true, memo: memo ) if state pkg.message "%s: pushed #{fingerprint}", :green else pkg.message "%s: #{fingerprint} already in cache" end h[pkg.name] = { 'updated' => state, 'fingerprint' => fingerprint } end hit = results.count { |_, info| info['updated'] } Autoproj.message "#{hit} updated packages, #{results.size - hit} "\ 'reused entries' results end
# File lib/autoproj/cli/ci.rb, line 27 def cache_state(dir, ignore: []) packages = resolve_packages memo = {} packages.each_with_object({}) do |pkg, h| state = package_cache_state(dir, pkg, memo: memo) if ignore.include?(pkg.name) state = state.merge('cached' => false, 'metadata' => false) end h[pkg.name] = state end end
# File lib/autoproj/cli/ci.rb, line 274 def cleanup_build_cache(dir, size_limit) all_files = Find.enum_for(:find, dir).map do |path| next unless File.file?(path) && File.file?("#{path}.json") [path, File.stat(path)] end.compact total_size = all_files.map { |_, s| s.size }.sum lru = all_files.sort_by { |_, s| s.mtime } while total_size > size_limit path, stat = lru.shift Autoproj.message "removing #{path} (size=#{stat.size}, mtime=#{stat.mtime})" FileUtils.rm_f path FileUtils.rm_f "#{path}.json" total_size -= stat.size end Autoproj.message format("current build cache size: %.1f GB", Float(total_size) / 1_000_000_000) total_size end
# File lib/autoproj/cli/ci.rb, line 314 def consolidated_report # NOTE: keys must match PHASES new_reports = { 'import' => @ws.import_report_path, 'build' => @ws.build_report_path, 'test' => @ws.utility_report_path('test') } # We start with the cached info (if any) and override with # information from the other phase reports cache_report_path = File.join(@ws.root_dir, 'cache-pull.json') result = load_report(cache_report_path, 'cache_pull_report')['packages'] result.delete_if do |_name, info| next true unless info.delete('cached') PHASES.each do |phase_name| if (phase_info = info[phase_name]) phase_info['cached'] = true end end false end new_reports.each do |phase_name, path| report = load_report(path, "#{phase_name}_report") report['packages'].each do |pkg_name, pkg_info| result[pkg_name] ||= {} if pkg_info['invoked'] result[pkg_name][phase_name] = pkg_info.merge( 'cached' => false, 'timestamp' => report['timestamp'] ) end end end { 'packages' => result } end
Build a report in a given directory
The method itself will not archive the directory, only gather the information in a consistent way
# File lib/autoproj/cli/ci.rb, line 176 def create_report(dir) initialize_and_load finalize_setup([], non_imported_packages: :ignore) report = consolidated_report FileUtils.mkdir_p(dir) File.open(File.join(dir, 'report.json'), 'w') do |io| JSON.dump(report, io) end installation_manifest = InstallationManifest .from_workspace_root(@ws.root_dir) logs = File.join(dir, 'logs') # Pre-create the logs, or cp_r will have a different behavior # if the directory exists or not FileUtils.mkdir_p(logs) installation_manifest.each_package do |pkg| glob = Dir.glob(File.join(pkg.logdir, '*')) FileUtils.cp_r(glob, logs) if File.directory?(pkg.logdir) end end
# File lib/autoproj/cli/ci.rb, line 297 def load_built_flags path = @ws.build_report_path return {} unless File.file?(path) report = JSON.parse(File.read(path)) report['build_report']['packages'] .each_with_object({}) do |pkg_report, h| h[pkg_report['name']] = pkg_report['built'] end end
# File lib/autoproj/cli/ci.rb, line 308 def load_report(path, root_name, default: { 'packages' => {} }) return default unless File.file?(path) JSON.parse(File.read(path)).fetch(root_name) end
Checks if a package's test results should be processed with xunit-viewer
@param [String] results_dir the directory where the @param [String] xunit_output path to the xunit-viewer output. An
existing file is re-generated only if force is true
@param [Boolean] force re-generation of the xunit-viewer output
# File lib/autoproj/cli/ci.rb, line 131 def need_xunit_processing?(results_dir, xunit_output, force: false) # We don't re-generate if the xunit-processed files were cached return if !force && File.file?(xunit_output) # We only check whether there are xml files in the # package's test dir. That's the only check we do ... if # the XML files are not JUnit, we'll finish with an empty # xunit html file Dir.enum_for(:glob, File.join(results_dir, '*.xml')) .first end
# File lib/autoproj/cli/ci.rb, line 199 def package_cache_path(dir, pkg, fingerprint: nil, memo: {}) fingerprint ||= pkg.fingerprint(memo: memo) File.join(dir, pkg.name, fingerprint) end
# File lib/autoproj/cli/ci.rb, line 204 def package_cache_state(dir, pkg, memo: {}) fingerprint = pkg.fingerprint(memo: memo) path = package_cache_path(dir, pkg, fingerprint: fingerprint, memo: memo) { 'path' => path, 'cached' => File.file?(path), 'metadata' => File.file?("#{path}.json"), 'fingerprint' => fingerprint } end
Post-processing of test results
# File lib/autoproj/cli/ci.rb, line 168 def process_test_results(force: false, xunit_viewer: 'xunit-viewer') process_test_results_xunit(force: force, xunit_viewer: xunit_viewer) end
Process the package's test results with xunit-viewer
@param [String] xunit_viewer path to xunit-viewer @param [Boolean] force re-generation of the xunit-viewer output. If
false, packages that already have a xunit-viewer output will be skipped
# File lib/autoproj/cli/ci.rb, line 148 def process_test_results_xunit(force: false, xunit_viewer: 'xunit-viewer') consolidated_report['packages'].each_value do |info| next unless info['test'] next unless (results_dir = info['test']['target_dir']) xunit_output = "#{results_dir}.html" next unless need_xunit_processing?(results_dir, xunit_output, force: force) success = system(xunit_viewer, "--results=#{results_dir}", "--output=#{xunit_output}") unless success Autoproj.warn 'xunit-viewer conversion failed '\ "for '#{results_dir}'" end end end
# File lib/autoproj/cli/ci.rb, line 219 def pull_package_from_cache(dir, pkg, memo: {}) fingerprint = pkg.fingerprint(memo: memo) path = package_cache_path(dir, pkg, fingerprint: fingerprint, memo: memo) return [false, fingerprint, {}] unless File.file?(path) metadata_path = "#{path}.json" metadata = if File.file?(metadata_path) JSON.parse(File.read(metadata_path)) else {} end # Do not pull packages for which we should run tests tests_enabled = pkg.test_utility.enabled? tests_invoked = metadata['test'] && metadata['test']['invoked'] if tests_enabled && !tests_invoked pkg.message '%s: has tests that have never '\ 'been invoked, not pulling from cache' return [false, fingerprint, {}] end FileUtils.mkdir_p(pkg.prefix) unless system('tar', 'xzf', path, chdir: pkg.prefix, out: '/dev/null') raise PullError, "tar failed when pulling cache file for #{pkg.name}" end [true, fingerprint, metadata] end
# File lib/autoproj/cli/ci.rb, line 249 def push_package_to_cache(dir, pkg, metadata, force: false, memo: {}) fingerprint = pkg.fingerprint(memo: memo) path = package_cache_path(dir, pkg, fingerprint: fingerprint, memo: memo) temppath = "#{path}.#{Process.pid}.#{rand(256)}" FileUtils.mkdir_p(File.dirname(path)) if force || !File.file?("#{path}.json") File.open(temppath, 'w') { |io| JSON.dump(metadata, io) } FileUtils.mv(temppath, "#{path}.json") end if !force && File.file?(path) # Update modification time for the cleanup process FileUtils.touch(path) return [false, fingerprint] end result = system('tar', 'czf', temppath, '.', chdir: pkg.prefix, out: '/dev/null') raise "tar failed when pushing cache file for #{pkg.name}" unless result FileUtils.mv(temppath, path) [true, fingerprint] end
# File lib/autoproj/cli/ci.rb, line 17 def resolve_packages initialize_and_load source_packages, * = finalize_setup( [], non_imported_packages: :ignore ) source_packages.map do |pkg_name| ws.manifest.find_autobuild_package(pkg_name) end end