class MesaTestSubmitter

Constants

DEFAULT_URI

Attributes

base_uri[RW]
computer_name[RW]
config_file[RW]
email[RW]
github_protocol[RW]
logs_token[RW]
mesa_mirror[RW]
mesa_work[RW]
password[RW]
platform[RW]
platform_version[RW]
processor[RW]
shell[R]
user_name[RW]

Public Class Methods

new( computer_name: nil, user_name: nil, email: nil, github_protocol: nil, mesa_mirror: nil, platform: nil, platform_version: nil, processor: nil, config_file: nil, base_uri: nil, logs_token: nil ) { |self| ... } click to toggle source

many defaults are set in body

# File lib/mesa_test.rb, line 125
def initialize(
    computer_name: nil, user_name: nil, email: nil, github_protocol: nil,
    mesa_mirror: nil, platform: nil, platform_version: nil, processor: nil,
    config_file: nil, base_uri: nil, logs_token: nil
)
  @computer_name = computer_name || Socket.gethostname.scan(/^[^\.]+\.?/)[0]
  @computer_name.chomp!('.') if @computer_name
  @user_name = user_name || (ENV['USER'] || ENV['USERNAME'])
  @email = email || ''
  @password = password || ''
  @github_protocol = github_protocol || :ssh
  @mesa_mirror = mesa_mirror ||
    File.join(ENV['HOME'], '.mesa_test', 'mirror')
  @mesa_work = mesa_work ||
    File.join(ENV['HOME'], '.mesa_test', 'work')
  @platform = platform
  if @platform.nil?
    @platform =
      if OS.osx?
        'macOS'
      elsif OS.linux?
        'Linux'
      else
        ''
      end
  end
  @platform_version = platform_version || ''
  @processor = processor || ''
  @config_file = config_file || File.join(ENV['HOME'], '.mesa_test',
                                          'config.yml')
  @base_uri = base_uri
  @logs_token = logs_token || ENV['MESA_LOGS_TOKEN']

  # set up thor-proof way to get responses from user. Thor hijacks the
  # gets command, so we have to use its built-in "ask" method, which is
  # actually more useful
  @shell = Thor::Shell::Color.new

  yield self if block_given?
end
new_from_config( config_file: File.join(ENV['HOME'], '.mesa_test', 'config.yml'), force_setup: false, base_uri: DEFAULT_URI ) click to toggle source
# File lib/mesa_test.rb, line 102
def self.new_from_config(
  config_file: File.join(ENV['HOME'], '.mesa_test', 'config.yml'),
  force_setup: false,
  base_uri: DEFAULT_URI
)
  new_submitter = new(config_file: config_file, base_uri: base_uri)
  if force_setup
    new_submitter.setup
  elsif not File.exist? config_file
    puts "No such config file #{config_file}. Starting setup wizard."
    new_submitter.setup
  end
  new_submitter.load_computer_data
  return new_submitter
end

Public Instance Methods

build_log_params(mesa) click to toggle source

Parameters for reporting a failed compilation to the logs server

# File lib/mesa_test.rb, line 288
def build_log_params(mesa)
  {
    'computer_name' => computer_name,
    'commit' => mesa.sha,
    'build.log' => mesa.build_log_64
  }
end
commit_params(mesa, entire: true, empty: false) click to toggle source

Parameters to be submitted in JSON format for reporting information about the overall commit being tested; used even if only submitting an entire test. This also determines if the submission is for an entire commit (compilation information and every test), an empty commit (just compilation information), or a non-empty, but also non-entire submission (results for a single test without compilation information)

# File lib/mesa_test.rb, line 244
def commit_params(mesa, entire: true, empty: false)
  # the compiler data should be able to be used as-is, but right now the
  # names don't match with what the database expects, so we do some renaming
  # shenanigans.
  #
  ####################################
  # THIS SHOULD GO BEFORE PRODUCTION #
  {
    sha: mesa.sha,
    compiled: mesa.installed?,
    entire: entire,
    empty: empty,
  }.merge(mesa.compiler_hash)
end
confirm_computer() click to toggle source

Phone home to testhub and confirm that computer and user are valid. Useful for confirming that submissions will be accepted before wasting time on a test later.

# File lib/mesa_test.rb, line 319
def confirm_computer
  uri = URI.parse(base_uri + '/check_computer.json')
  https = Net::HTTP.new(uri.hostname, uri.port)
  https.use_ssl = base_uri.include? 'https'

  request = Net::HTTP::Post.new(
    uri, initheader = { 'Accept' => 'application/json', 'Content-Type' => 'application/json' }
  )
  request.body = {
    email: email,
    password: password,
    computer_name: computer_name
  }.to_json
  JSON.parse(https.request(request).body).to_hash
end
confirm_computer_data() click to toggle source
# File lib/mesa_test.rb, line 171
def confirm_computer_data
  puts 'Ready to submit the following data:'
  puts '-------------------------------------------------------'
  puts "Computer Name           #{computer_name}"
  puts "User email              #{email}"
  puts 'Password                ***********'
  puts "logs API token          #{logs_token}"
  puts "GitHub Protocol         #{github_protocol}"
  puts "MESA Mirror Location    #{mesa_mirror}"
  puts "MESA Work Location      #{mesa_work}"
  puts "Platform                #{platform} #{platform_version}"
  # puts "Config location         #{config_file}"
  puts '-------------------------------------------------------'
  puts ''
  response = shell.ask 'Is this correct? (y/Y = Yes, anything else = No):'
  response.strip.casecmp('y').zero?
end
instance_params(mesa) click to toggle source

Given a valid Mesa object, create an array of hashes that describe the test cases and the test results. These will be encoded as an array of JSON objects.

# File lib/mesa_test.rb, line 262
def instance_params(mesa)
  has_errors = []
  res = []
  mesa.test_case_names.each do |mod, names|
    names.each do |test_name|
      begin
        test_case = mesa.test_cases[mod][test_name]
        res << test_case.results_hash
      rescue TestCaseDirError
        # shell.say "It appears that #{test_case.test_name} has not been "\
        #           'run yet. Unable to submit data for this test.', :red
        has_errors << test_case
      end
    end
  end
  unless has_errors.empty?
    shell.say "The following test cases could NOT be read for submission:",
              :red
    has_errors.each do |test_case|
      shell.say "- #{test_case.test_name}", :red
    end
  end
  res
end
load_computer_data() click to toggle source
# File lib/mesa_test.rb, line 214
def load_computer_data
  data_hash = YAML.load(File.read(config_file))
  @computer_name = data_hash['computer_name']
  @email = data_hash['email']
  @password = data_hash['password']
  @logs_token = data_hash['logs_token']
  @github_protocol = data_hash['github_protocol'].to_sym
  @mesa_mirror = data_hash['mesa_mirror']
  @mesa_work = data_hash['mesa_work']
  @platform = data_hash['platform']
  @platform_version = data_hash['platform_version']
end
save_computer_data() click to toggle source

For one “computer” on the web server, and for [subjective] consistency reasons, the platform, processor, and RAM cannot be changed! If you change platforms (i.e. switch from mac to linux, or change between linux flavors), you should create a new computer account. Similarly, create new computer accounts if you change your RAM or processor. You do not need to change computers if you upgrade your platform (macOS 10.12 -> 10.13

# File lib/mesa_test.rb, line 195
def save_computer_data
  data_hash = {
    'computer_name' => computer_name,
    'email' => email,
    'password' => password,
    'logs_token' => logs_token,
    'github_protocol' => github_protocol,
    'mesa_mirror' => mesa_mirror,
    'mesa_work' => mesa_work,
    'platform' => platform,
    'platform_version' => platform_version,
  }
  # make sure there's a directory to write to
  unless dir_or_symlink_exists? File.dirname(config_file)
    FileUtils.mkdir_p File.dirname(config_file)
  end
  File.open(config_file, 'w') { |f| f.write(YAML.dump(data_hash)) }
end
setup() click to toggle source

set up config file for computer

# File lib/mesa_test.rb, line 24
  def setup
    update do |s|
      shell.say 'This wizard will guide you through setting up a computer
profile and default data for test case submissions to MESATestHub. You
will be able to confirm entries at the end. Default/current values are always
shown in parentheses at the end of a prompt. Pressing enter will accept the
default values.

To submit to MESATestHub, a valid computer name, email address, and password
are all required. To actually run a test, you need to specify a location for
your base MESA git repository. All other data are useful, but optional. Any data
transferred to MESATestHub will be encrypted via HTTPS, but be warned that your
e-mail and password will be stored in plain text.'
      # Get computer name
      response = shell.ask('What is the name of this computer (required)? ' \
        "(#{s.computer_name}):", :blue)
      s.computer_name = response unless response.empty?

      # Get user e-mail
      response = shell.ask 'What is the email you can be reached ' \
        "at (required)? (#{s.email}):", :blue
      s.email = response unless response.empty?

      # Get user password
      response = shell.ask 'What is the password associated with the email ' \
        "#{s.email} (required)? (#{s.password})", :blue
      s.password = response unless response.empty?

      # Get API key for submitting failure logs
      response = shell.ask 'What is the logs submission API token associated '\
        "with the email #{s.email} (required; contact Josiah Schwab if you "\
        "need a key)? (#{s.logs_token})", :blue
      s.logs_token = response unless response.empty?

      # Determine if we'll use ssh or https to access github
      response = shell.ask 'When accessing GitHub, which protocol do you '\
        'want to use?', :blue, limited_to: %w[ssh https]
      s.github_protocol = response.strip.downcase.to_sym

      # Get location of source MESA repo (the mirror)
      response = shell.ask "Where is/should your mirrored MESA repository " \
      "located? This is where a mirror will be stored from which test " \
      "repos will be generated. You won't touch this in regular operation. " \
      "(#{s.mesa_mirror}):", :blue
      s.mesa_mirror = response unless response.empty?

      # Get location of source MESA work (where testing happens)
      response = shell.ask "Where is/should your working directory for "\
      "testing be located? This is where testing actually occurs, but all "\
      "files it uses are cached in the mirror repo to save time later. " \
      "(#{s.mesa_work}):", :blue
      s.mesa_work = response unless response.empty?

      # Get platform information
      response = shell.ask 'What is the platform of this computer (eg. ' \
        "macOS, Ubuntu)? (#{s.platform}):", :blue
      s.platform = response unless response.empty?
      response = shell.ask 'What is the version of the platform (eg. 10.15.5, ' \
        "Ubuntu 16.04)? (#{s.platform_version}):", :blue
      s.platform_version = response unless response.empty?

      # we are powerless to do change the location for now, so stop asking
      # about it
      # # Confirm save location
      # response = shell.ask "This will be saved in #{s.config_file}. Press " \
      #   'enter to accept or enter a new location:', :blue, path: true
      # s.config_file = response unless response.empty?
    end

    # Confirm data. If not confirmed, restart whole wizard.
    if confirm_computer_data
      save_computer_data
    else
      shell.say "Restarting wizard.\n"
      setup
    end
  end
single_instance_params(test_case) click to toggle source

Parameters for a single test case. mesa is an instance of Mesa, and test_case is an instance of MesaTestCase representing the test case to be submitted

# File lib/mesa_test.rb, line 312
def single_instance_params(test_case)
  [test_case.results_hash]
end
submit_build_log(mesa) click to toggle source

send build log to the logs server

# File lib/mesa_test.rb, line 459
def submit_build_log(mesa)
  # intercept and don't send if mesa was properly installed
  return true if mesa.installed?

  # don't even try unless we have a logs token set
  unless logs_token
    shell.say 'Cannot submit to logs server; need to set mesa_logs_token '\
              'in the mesa_test config file.'
    return false
  end

  # do submission
  res = submit_logs(build_log_params(mesa))

  # report out results
  if !res.is_a? Net::HTTPOK
    shell.say "\nFailed to submit build.log to the LOGS server for commit "\
              "#{mesa.sha}.", :red
    false
  else
    shell.say "\nSuccessfully submitted build.log to the LOGS server for "\
              "#{mesa.sha}.", :green
    true
  end
end
submit_commit(mesa, empty: false, force_logs: false) click to toggle source

submit entire commit's worth of test cases, OR submit compilation status and NO test cases

# File lib/mesa_test.rb, line 337
def submit_commit(mesa, empty: false, force_logs: false)
  unless mesa.install_attempted?
    raise MesaDirError, 'No testhub.yml file found in installation; '\
                        'must attempt to install before subitting.'
  end
  uri = URI.parse(base_uri + '/submissions/create.json')
  https = Net::HTTP.new(uri.hostname, uri.port)
  https.use_ssl = true if base_uri.include? 'https'

  request = Net::HTTP::Post.new(
    uri,
    initheader = { 'Accept' => 'application/json', 'Content-Type' => 'application/json' }
  )

  # create the request body for submission to the submissions API
  #
  # if we have an empty submission, then it is necessarily not entire.
  # Similarly, a non-empty submission is necessarily entire (otherwise one
  # would use +submit_instance+). Also, make a "nonempty" submission be
  # empty if there was an overall build error
  empty ||= !mesa.installed?
  request_data = {submitter: submitter_params,
                  commit: commit_params(mesa, empty: empty, entire: !empty)}
  # don't need test instances if it's an empty submission or if compilation
  # failed
  if !empty && request_data[:commit][:compiled]
    request_data[:instances] = instance_params(mesa)
  end
  request.body = request_data.to_json

  # actually do the submission
  response = https.request request

  if !response.is_a? Net::HTTPCreated
    shell.say "\nFailed to submit some or all test case instances and/or "\
              'commit data.', :red
    false
  else
    shell.say "\nSuccessfully submitted commit #{mesa.sha}.", :green
    # commit submitted to testhub, now submit build log if compilation failed
    # and exit
    unless mesa.installed?
      return submit_build_log(mesa)
    end

    # compilation succeded, so submit any logs for failing tests
    res = true
    unless empty
      mesa.test_cases.each do |mod, test_case_hash|
        test_case_hash.each do |tc_name, test_case|
          # get at each individual test case, see if it failed, and if it
          # did, submit its log files
          if !test_case.passed? || force_logs
            res &&= submit_test_log(test_case, skip_passing: !force_logs)
          end
        end
      end
    end

    # a true return value means that any and all log submission were
    # successful
    res
  end
end
submit_instance(mesa, test_case, force_logs: false) click to toggle source

submit results for a single test case instance. Does not report overall compilation status to testhub. Use an empty commit submission for that

# File lib/mesa_test.rb, line 404
def submit_instance(mesa, test_case, force_logs: false)
  unless mesa.install_attempted?
    raise MesaDirError, 'No testhub.yml file found in installation; '\
                        'must attempt to install before subitting.'
  end

  uri = URI.parse(base_uri + '/submissions/create.json')
  https = Net::HTTP.new(uri.hostname, uri.port)
  https.use_ssl = true if base_uri.include? 'https'

  request = Net::HTTP::Post.new(
    uri,
    initheader = { 'Accept' => 'application/json', 'Content-Type' => 'application/json' }
  )

  # create the request body for submission to the submissions API
  #
  # submission is not empty (there is one test case), and it is also not
  # entire (... there is only test case)
  request_data = {submitter: submitter_params,
                  commit: commit_params(mesa, empty: false, entire: false),
                  instances: single_instance_params(test_case)}
  request.body = request_data.to_json

  # actually do the submission
  response = https.request request

  if !response.is_a? Net::HTTPCreated
    shell.say "\nFailed to submit #{test_case.test_name} for commit "\
              "#{mesa.sha}", :red
    false
  else
    shell.say "\nSuccessfully submitted instance of #{test_case.test_name} "\
              "for commit #{mesa.sha}.", :green
    # submit logs if test failed
    if !test_case.passed? || force_logs
      return submit_test_log(test_case, skip_passing: !force_logs)
    end
    true
  end
end
submit_logs(params) click to toggle source

make generic request to LOGS server params is a hash of data to be encoded as JSON and sent off

# File lib/mesa_test.rb, line 448
def submit_logs(params)
  uri = URI('https://logs.mesastar.org/uploads')
  https = Net::HTTP.new(uri.host, uri.port)
  https.use_ssl = true
  req = Net::HTTP::Post.new(uri.path, 'Content-Type' => 'application/json',
                            'X-Api-Key' => logs_token)
  req.body = params.to_json
  https.request(req)
end
submit_test_log(test_case, skip_passing: true) click to toggle source

send build log to the logs server

# File lib/mesa_test.rb, line 486
def submit_test_log(test_case, skip_passing: true)
  # skip submission if mesa was never installed or if the test passed
  if !test_case.mesa.installed? || (test_case.passed? && skip_passing)
    return true
  end

  # don't even try unless we have a logs token set
  unless logs_token
    shell.say 'Cannot submit to logs server; need to set mesa_logs_token '\
              'in the mesa_test config file..'
    return false
  end

  # do submission
  res = submit_logs(test_log_params(test_case))

  # report out results
  if !res.is_a? Net::HTTPOK
    shell.say "Failed to submit logs for test case #{test_case.test_name} "\
              "in commit #{test_case.mesa.sha}.", :red
    false
  else
    shell.say "Successfully submitted logs for test case "\
              "#{test_case.test_name} in commit #{test_case.mesa.sha}.",
              :green
    true
  end
end
submitter_params() click to toggle source

Parameters to be submitted in JSON format for reporting information about the submitting user and computer

# File lib/mesa_test.rb, line 229
def submitter_params
  {
    email: email,
    password: password,
    computer: computer_name,
    platform_version: platform_version
  }
end
test_log_params(test_case) click to toggle source

Parameters for reporting a failed test to the logs server

# File lib/mesa_test.rb, line 297
def test_log_params(test_case)
  res = {
    'computer_name' => computer_name,
    'commit' => test_case.mesa.sha, 
    'test_case' => test_case.test_name     
  }
  res['mk.txt'] = test_case.mk_64 unless test_case.mk_64.empty?
  res['out.txt'] = test_case.out_64 unless test_case.out_64.empty?
  res['err.txt'] = test_case.err_64 unless test_case.err_64.empty?
  res
end
update() { |self| ... } click to toggle source

ease setup of a blank/default submitter

# File lib/mesa_test.rb, line 167
def update
  yield self if block_given?
end