class MesaTestSubmitter
Constants
- DEFAULT_URI
Attributes
Public Class Methods
many defaults are set in body
# File lib/mesa_test.rb, line 125 def initialize( computer_name: nil, user_name: nil, email: nil, github_protocol: nil, mesa_mirror: nil, platform: nil, platform_version: nil, processor: nil, config_file: nil, base_uri: nil, logs_token: nil ) @computer_name = computer_name || Socket.gethostname.scan(/^[^\.]+\.?/)[0] @computer_name.chomp!('.') if @computer_name @user_name = user_name || (ENV['USER'] || ENV['USERNAME']) @email = email || '' @password = password || '' @github_protocol = github_protocol || :ssh @mesa_mirror = mesa_mirror || File.join(ENV['HOME'], '.mesa_test', 'mirror') @mesa_work = mesa_work || File.join(ENV['HOME'], '.mesa_test', 'work') @platform = platform if @platform.nil? @platform = if OS.osx? 'macOS' elsif OS.linux? 'Linux' else '' end end @platform_version = platform_version || '' @processor = processor || '' @config_file = config_file || File.join(ENV['HOME'], '.mesa_test', 'config.yml') @base_uri = base_uri @logs_token = logs_token || ENV['MESA_LOGS_TOKEN'] # set up thor-proof way to get responses from user. Thor hijacks the # gets command, so we have to use its built-in "ask" method, which is # actually more useful @shell = Thor::Shell::Color.new yield self if block_given? end
# File lib/mesa_test.rb, line 102 def self.new_from_config( config_file: File.join(ENV['HOME'], '.mesa_test', 'config.yml'), force_setup: false, base_uri: DEFAULT_URI ) new_submitter = new(config_file: config_file, base_uri: base_uri) if force_setup new_submitter.setup elsif not File.exist? config_file puts "No such config file #{config_file}. Starting setup wizard." new_submitter.setup end new_submitter.load_computer_data return new_submitter end
Public Instance Methods
Parameters for reporting a failed compilation to the logs server
# File lib/mesa_test.rb, line 288 def build_log_params(mesa) { 'computer_name' => computer_name, 'commit' => mesa.sha, 'build.log' => mesa.build_log_64 } end
Parameters to be submitted in JSON format for reporting information about the overall commit being tested; used even if only submitting an entire test. This also determines if the submission is for an entire commit (compilation information and every test), an empty commit (just compilation information), or a non-empty, but also non-entire submission (results for a single test without compilation information)
# File lib/mesa_test.rb, line 244 def commit_params(mesa, entire: true, empty: false) # the compiler data should be able to be used as-is, but right now the # names don't match with what the database expects, so we do some renaming # shenanigans. # #################################### # THIS SHOULD GO BEFORE PRODUCTION # { sha: mesa.sha, compiled: mesa.installed?, entire: entire, empty: empty, }.merge(mesa.compiler_hash) end
Phone home to testhub and confirm that computer and user are valid. Useful for confirming that submissions will be accepted before wasting time on a test later.
# File lib/mesa_test.rb, line 319 def confirm_computer uri = URI.parse(base_uri + '/check_computer.json') https = Net::HTTP.new(uri.hostname, uri.port) https.use_ssl = base_uri.include? 'https' request = Net::HTTP::Post.new( uri, initheader = { 'Accept' => 'application/json', 'Content-Type' => 'application/json' } ) request.body = { email: email, password: password, computer_name: computer_name }.to_json JSON.parse(https.request(request).body).to_hash end
# File lib/mesa_test.rb, line 171 def confirm_computer_data puts 'Ready to submit the following data:' puts '-------------------------------------------------------' puts "Computer Name #{computer_name}" puts "User email #{email}" puts 'Password ***********' puts "logs API token #{logs_token}" puts "GitHub Protocol #{github_protocol}" puts "MESA Mirror Location #{mesa_mirror}" puts "MESA Work Location #{mesa_work}" puts "Platform #{platform} #{platform_version}" # puts "Config location #{config_file}" puts '-------------------------------------------------------' puts '' response = shell.ask 'Is this correct? (y/Y = Yes, anything else = No):' response.strip.casecmp('y').zero? end
Given a valid Mesa
object, create an array of hashes that describe the test cases and the test results. These will be encoded as an array of JSON objects.
# File lib/mesa_test.rb, line 262 def instance_params(mesa) has_errors = [] res = [] mesa.test_case_names.each do |mod, names| names.each do |test_name| begin test_case = mesa.test_cases[mod][test_name] res << test_case.results_hash rescue TestCaseDirError # shell.say "It appears that #{test_case.test_name} has not been "\ # 'run yet. Unable to submit data for this test.', :red has_errors << test_case end end end unless has_errors.empty? shell.say "The following test cases could NOT be read for submission:", :red has_errors.each do |test_case| shell.say "- #{test_case.test_name}", :red end end res end
# File lib/mesa_test.rb, line 214 def load_computer_data data_hash = YAML.load(File.read(config_file)) @computer_name = data_hash['computer_name'] @email = data_hash['email'] @password = data_hash['password'] @logs_token = data_hash['logs_token'] @github_protocol = data_hash['github_protocol'].to_sym @mesa_mirror = data_hash['mesa_mirror'] @mesa_work = data_hash['mesa_work'] @platform = data_hash['platform'] @platform_version = data_hash['platform_version'] end
For one “computer” on the web server, and for [subjective] consistency reasons, the platform, processor, and RAM cannot be changed! If you change platforms (i.e. switch from mac to linux, or change between linux flavors), you should create a new computer account. Similarly, create new computer accounts if you change your RAM or processor. You do not need to change computers if you upgrade your platform (macOS 10.12 -> 10.13
# File lib/mesa_test.rb, line 195 def save_computer_data data_hash = { 'computer_name' => computer_name, 'email' => email, 'password' => password, 'logs_token' => logs_token, 'github_protocol' => github_protocol, 'mesa_mirror' => mesa_mirror, 'mesa_work' => mesa_work, 'platform' => platform, 'platform_version' => platform_version, } # make sure there's a directory to write to unless dir_or_symlink_exists? File.dirname(config_file) FileUtils.mkdir_p File.dirname(config_file) end File.open(config_file, 'w') { |f| f.write(YAML.dump(data_hash)) } end
set up config file for computer
# File lib/mesa_test.rb, line 24 def setup update do |s| shell.say 'This wizard will guide you through setting up a computer profile and default data for test case submissions to MESATestHub. You will be able to confirm entries at the end. Default/current values are always shown in parentheses at the end of a prompt. Pressing enter will accept the default values. To submit to MESATestHub, a valid computer name, email address, and password are all required. To actually run a test, you need to specify a location for your base MESA git repository. All other data are useful, but optional. Any data transferred to MESATestHub will be encrypted via HTTPS, but be warned that your e-mail and password will be stored in plain text.' # Get computer name response = shell.ask('What is the name of this computer (required)? ' \ "(#{s.computer_name}):", :blue) s.computer_name = response unless response.empty? # Get user e-mail response = shell.ask 'What is the email you can be reached ' \ "at (required)? (#{s.email}):", :blue s.email = response unless response.empty? # Get user password response = shell.ask 'What is the password associated with the email ' \ "#{s.email} (required)? (#{s.password})", :blue s.password = response unless response.empty? # Get API key for submitting failure logs response = shell.ask 'What is the logs submission API token associated '\ "with the email #{s.email} (required; contact Josiah Schwab if you "\ "need a key)? (#{s.logs_token})", :blue s.logs_token = response unless response.empty? # Determine if we'll use ssh or https to access github response = shell.ask 'When accessing GitHub, which protocol do you '\ 'want to use?', :blue, limited_to: %w[ssh https] s.github_protocol = response.strip.downcase.to_sym # Get location of source MESA repo (the mirror) response = shell.ask "Where is/should your mirrored MESA repository " \ "located? This is where a mirror will be stored from which test " \ "repos will be generated. You won't touch this in regular operation. " \ "(#{s.mesa_mirror}):", :blue s.mesa_mirror = response unless response.empty? # Get location of source MESA work (where testing happens) response = shell.ask "Where is/should your working directory for "\ "testing be located? This is where testing actually occurs, but all "\ "files it uses are cached in the mirror repo to save time later. " \ "(#{s.mesa_work}):", :blue s.mesa_work = response unless response.empty? # Get platform information response = shell.ask 'What is the platform of this computer (eg. ' \ "macOS, Ubuntu)? (#{s.platform}):", :blue s.platform = response unless response.empty? response = shell.ask 'What is the version of the platform (eg. 10.15.5, ' \ "Ubuntu 16.04)? (#{s.platform_version}):", :blue s.platform_version = response unless response.empty? # we are powerless to do change the location for now, so stop asking # about it # # Confirm save location # response = shell.ask "This will be saved in #{s.config_file}. Press " \ # 'enter to accept or enter a new location:', :blue, path: true # s.config_file = response unless response.empty? end # Confirm data. If not confirmed, restart whole wizard. if confirm_computer_data save_computer_data else shell.say "Restarting wizard.\n" setup end end
Parameters for a single test case. mesa
is an instance of Mesa
, and test_case
is an instance of MesaTestCase
representing the test case to be submitted
# File lib/mesa_test.rb, line 312 def single_instance_params(test_case) [test_case.results_hash] end
send build log to the logs server
# File lib/mesa_test.rb, line 459 def submit_build_log(mesa) # intercept and don't send if mesa was properly installed return true if mesa.installed? # don't even try unless we have a logs token set unless logs_token shell.say 'Cannot submit to logs server; need to set mesa_logs_token '\ 'in the mesa_test config file.' return false end # do submission res = submit_logs(build_log_params(mesa)) # report out results if !res.is_a? Net::HTTPOK shell.say "\nFailed to submit build.log to the LOGS server for commit "\ "#{mesa.sha}.", :red false else shell.say "\nSuccessfully submitted build.log to the LOGS server for "\ "#{mesa.sha}.", :green true end end
submit entire commit's worth of test cases, OR submit compilation status and NO test cases
# File lib/mesa_test.rb, line 337 def submit_commit(mesa, empty: false, force_logs: false) unless mesa.install_attempted? raise MesaDirError, 'No testhub.yml file found in installation; '\ 'must attempt to install before subitting.' end uri = URI.parse(base_uri + '/submissions/create.json') https = Net::HTTP.new(uri.hostname, uri.port) https.use_ssl = true if base_uri.include? 'https' request = Net::HTTP::Post.new( uri, initheader = { 'Accept' => 'application/json', 'Content-Type' => 'application/json' } ) # create the request body for submission to the submissions API # # if we have an empty submission, then it is necessarily not entire. # Similarly, a non-empty submission is necessarily entire (otherwise one # would use +submit_instance+). Also, make a "nonempty" submission be # empty if there was an overall build error empty ||= !mesa.installed? request_data = {submitter: submitter_params, commit: commit_params(mesa, empty: empty, entire: !empty)} # don't need test instances if it's an empty submission or if compilation # failed if !empty && request_data[:commit][:compiled] request_data[:instances] = instance_params(mesa) end request.body = request_data.to_json # actually do the submission response = https.request request if !response.is_a? Net::HTTPCreated shell.say "\nFailed to submit some or all test case instances and/or "\ 'commit data.', :red false else shell.say "\nSuccessfully submitted commit #{mesa.sha}.", :green # commit submitted to testhub, now submit build log if compilation failed # and exit unless mesa.installed? return submit_build_log(mesa) end # compilation succeded, so submit any logs for failing tests res = true unless empty mesa.test_cases.each do |mod, test_case_hash| test_case_hash.each do |tc_name, test_case| # get at each individual test case, see if it failed, and if it # did, submit its log files if !test_case.passed? || force_logs res &&= submit_test_log(test_case, skip_passing: !force_logs) end end end end # a true return value means that any and all log submission were # successful res end end
submit results for a single test case instance. Does not report overall compilation status to testhub. Use an empty commit submission for that
# File lib/mesa_test.rb, line 404 def submit_instance(mesa, test_case, force_logs: false) unless mesa.install_attempted? raise MesaDirError, 'No testhub.yml file found in installation; '\ 'must attempt to install before subitting.' end uri = URI.parse(base_uri + '/submissions/create.json') https = Net::HTTP.new(uri.hostname, uri.port) https.use_ssl = true if base_uri.include? 'https' request = Net::HTTP::Post.new( uri, initheader = { 'Accept' => 'application/json', 'Content-Type' => 'application/json' } ) # create the request body for submission to the submissions API # # submission is not empty (there is one test case), and it is also not # entire (... there is only test case) request_data = {submitter: submitter_params, commit: commit_params(mesa, empty: false, entire: false), instances: single_instance_params(test_case)} request.body = request_data.to_json # actually do the submission response = https.request request if !response.is_a? Net::HTTPCreated shell.say "\nFailed to submit #{test_case.test_name} for commit "\ "#{mesa.sha}", :red false else shell.say "\nSuccessfully submitted instance of #{test_case.test_name} "\ "for commit #{mesa.sha}.", :green # submit logs if test failed if !test_case.passed? || force_logs return submit_test_log(test_case, skip_passing: !force_logs) end true end end
make generic request to LOGS server params
is a hash of data to be encoded as JSON and sent off
# File lib/mesa_test.rb, line 448 def submit_logs(params) uri = URI('https://logs.mesastar.org/uploads') https = Net::HTTP.new(uri.host, uri.port) https.use_ssl = true req = Net::HTTP::Post.new(uri.path, 'Content-Type' => 'application/json', 'X-Api-Key' => logs_token) req.body = params.to_json https.request(req) end
send build log to the logs server
# File lib/mesa_test.rb, line 486 def submit_test_log(test_case, skip_passing: true) # skip submission if mesa was never installed or if the test passed if !test_case.mesa.installed? || (test_case.passed? && skip_passing) return true end # don't even try unless we have a logs token set unless logs_token shell.say 'Cannot submit to logs server; need to set mesa_logs_token '\ 'in the mesa_test config file..' return false end # do submission res = submit_logs(test_log_params(test_case)) # report out results if !res.is_a? Net::HTTPOK shell.say "Failed to submit logs for test case #{test_case.test_name} "\ "in commit #{test_case.mesa.sha}.", :red false else shell.say "Successfully submitted logs for test case "\ "#{test_case.test_name} in commit #{test_case.mesa.sha}.", :green true end end
Parameters to be submitted in JSON format for reporting information about the submitting user and computer
# File lib/mesa_test.rb, line 229 def submitter_params { email: email, password: password, computer: computer_name, platform_version: platform_version } end
Parameters for reporting a failed test to the logs server
# File lib/mesa_test.rb, line 297 def test_log_params(test_case) res = { 'computer_name' => computer_name, 'commit' => test_case.mesa.sha, 'test_case' => test_case.test_name } res['mk.txt'] = test_case.mk_64 unless test_case.mk_64.empty? res['out.txt'] = test_case.out_64 unless test_case.out_64.empty? res['err.txt'] = test_case.err_64 unless test_case.err_64.empty? res end
ease setup of a blank/default submitter
# File lib/mesa_test.rb, line 167 def update yield self if block_given? end