class Flaky::Run
Attributes
result_dir[R]
result_file[R]
tests[R]
Public Class Methods
new(result_dir_postfix='')
click to toggle source
# File lib/flaky/run.rb, line 41 def initialize result_dir_postfix='' @tests = {} @start_time = Time.now result_dir = '/tmp/flaky/' + result_dir_postfix # rm -rf result_dir FileUtils.rm_rf result_dir FileUtils.mkdir_p result_dir @result_dir = result_dir @result_file = File.join result_dir, 'result.txt' @fail_file = File.join result_dir, 'fail.txt' end
Public Instance Methods
_execute(run_cmd, test_name, runs, appium, sauce)
click to toggle source
# File lib/flaky/run.rb, line 124 def _execute run_cmd, test_name, runs, appium, sauce # must capture exit code or log is an array. result = /\d+ runs, \d+ assertions, \d+ failures, \d+ errors, \d+ skips/ success = /0 failures, 0 errors, 0 skips/ passed = true exit_code = -1 timedout = false rake_pid = -1 # we need the minitest log in memory to scan for results. log = '' tmp_ruby_log = '/tmp/flaky/ruby_log_tmp.txt' File.delete(tmp_ruby_log) if File.exists? tmp_ruby_log flaky_logs_txt = '/tmp/flaky_logs.txt' File.delete flaky_logs_txt if File.exist? flaky_logs_txt tail_cmd = "tail -f -n1 /Users/#{ENV['USER']}/Library/Logs/iOS\\ Simulator/7.0.3/system.log > #{flaky_logs_txt}" tail_cmd = "adb logcat > #{flaky_logs_txt}" if !sauce && !appium.ios tail_system_log = Flaky::Cmd.new tail_cmd begin ten_minutes = 10 * 60 timeout ten_minutes do rake = Flaky::Cmd.new run_cmd rake_pid = rake.pid while process_exists rake_pid begin # readpartial throws end of file reached error new_out = rake.out.readpartial 999_999 # blocks on 0 data log += new_out File.open(tmp_ruby_log, 'a') do |f| f.write new_out end rescue end end # must write rake.err. it's not included in rake.out begin new_err = rake.err.read_nonblock 999_999 log += new_err if new_err File.open(tmp_ruby_log, 'a') do |f| f.write new_err end if new_err rescue end end rescue Exception => e timedout = true passed = false # after_run in run.rb is triggered by sigint Process.kill :SIGINT, rake_pid begin two_minutes = 2 * 60 timeout two_minutes do Process::waitpid rake_pid end rescue # if the process still isn't done after sigint, use sigkill Process.kill :SIGKILL, rake_pid end end # waitpid may throw if the pid doesn't exist by the time we're ready to wait. begin tail_system_log_pid = tail_system_log.pid Process.kill :SIGKILL, tail_system_log_pid Process::waitpid tail_system_log_pid rescue end unless timedout found_results = log.scan result # all result instances must match success found_results.each do |result| # runs must be >= 1. 0 runs mean no tests were run. r_count = result.match /(\d+) runs/ runs_not_zero = r_count && r_count[1] && r_count[1].to_i > 0 ? true : false unless result.match(success) && runs_not_zero passed = false break end end # no results found. passed = false if found_results.length <= 0 end pass_str = passed ? 'pass' : 'fail' test = @tests[test_name] # save log if passed pass = test[:pass] += 1 postfix = "pass_#{pass}" else fail = test[:fail] += 1 postfix = "fail_#{fail}" test[:timedout] = true if timedout end postfix = "#{runs}_#{test_name}_" + postfix postfix = '0' + postfix if runs <= 9 log_file = LogArtifact.new result_dir: result_dir, pass_str: pass_str, test_name: test_name # File.open 'w' will not create folders. Use mkdir_p before. test_file_path = log_file.name("#{postfix}.txt") FileUtils.mkdir_p File.dirname(test_file_path) # html Ruby test log File.open(test_file_path, 'w') do |f| f.write log end unless sauce movie_path = log_file.name("#{postfix}.mov") FileUtils.mkdir_p File.dirname(movie_path) movie_src = '/tmp/video.mov' if File.exists?(movie_src) unless Flaky.no_video # save movie on failure FileUtils.copy movie_src, movie_path if !passed end # always clean up movie File.delete movie_src if File.exists? movie_src end # save .TIMED_OUT.txt in timeout fails if timedout timeout_path = log_file.name("#{postfix}.TIMED_OUT.txt") FileUtils.mkdir_p File.dirname(timeout_path) File.open(timeout_path, 'w') {} end src_system_log = '/tmp/flaky_logs.txt' if File.exists? src_system_log # postfix is required! or the log will be saved to an incorrect path system_log_path = log_file.name("#{postfix}.system.txt") FileUtils.mkdir_p File.dirname(system_log_path) FileUtils.copy_file src_system_log, system_log_path File.delete src_system_log if File.exists? src_system_log end # appium server log appium_server_path = log_file.name("#{postfix}.appium.txt") FileUtils.mkdir_p File.dirname(appium_server_path) tmp_file = appium.flush_buffer if File.exists?(tmp_file) && !tmp_file.nil? && !tmp_file.empty? FileUtils.copy_file tmp_file, appium_server_path end File.delete tmp_file if File.exists? tmp_file # also delete the temp ruby log File.delete tmp_ruby_log if File.exists? tmp_ruby_log # copy app logs app_logs = '/tmp/flaky_tmp_log_folder' dest_dir = File.dirname(appium_server_path) if File.exists? app_logs Dir.glob(File.join(app_logs, '*')).each { |f| FileUtils.cp f, dest_dir } end FileUtils.rm_rf app_logs # copy android coverage coverage_folder = '/tmp/flaky/coverage' FileUtils.mkdir_p coverage_folder coverage_file = '/tmp/coverage.ec' if File.exists? '/tmp/coverage.ec' FileUtils.cp coverage_file, File.join(coverage_folder, "#{Time.now.to_i}.ec") File.delete coverage_file end end passed end
collect_crashes(array)
click to toggle source
# File lib/flaky/run.rb, line 302 def collect_crashes array Dir.glob(File.join(Dir.home, '/Library/Logs/DiagnosticReports/*.crash')) do |crash| array << crash end array end
execute(opts={})
click to toggle source
# File lib/flaky/run.rb, line 309 def execute opts={} run_cmd = opts[:run_cmd] test_name = opts[:test_name] appium = opts[:appium] sauce = opts[:sauce] old_crash_files = [] # appium is nil when on sauce if !sauce && appium collect_crashes old_crash_files end raise 'must pass :run_cmd' unless run_cmd raise 'must pass :test_name' unless test_name # local appium is not required when running on Sauce raise 'must pass :appium' unless appium || sauce test = @tests[test_name] ||= {runs: 0, pass: 0, fail: 0, timedout: false} runs = test[:runs] += 1 passed = _execute run_cmd, test_name, runs, appium, sauce unless sauce print cyan("\n #{test_name} ") if @last_test.nil? || @last_test != test_name print passed ? green(' ✓') : red(' ✖') else print cyan("\n #{test_name} ") print passed ? green(' ✓') : red(' ✖') print " https://saucelabs.com/tests/#{File.read('/tmp/appium_lib_session').chomp}\n" end # androids adb may crash also and it ends up in the same location as iOS. # appium is nil when running on Sauce if !sauce && appium new_crash_files = [] collect_crashes new_crash_files new_crash_files = new_crash_files - old_crash_files if new_crash_files.length > 0 File.open('/tmp/flaky/crashes.txt', 'a') do |f| f.puts '--' f.puts "Test: #{test_name} crashed on #{appium.ios ? 'ios' : 'android'}:" new_crash_files.each { |crash| f.puts crash } f.puts '--' end end end @last_test = test_name passed end
process_exists(pid)
click to toggle source
# File lib/flaky/run.rb, line 115 def process_exists pid begin Process.waitpid(pid, Process::WNOHANG) true rescue false end end
report(opts={})
click to toggle source
# File lib/flaky/run.rb, line 55 def report opts={} save_file = opts.fetch :save_file, true puts "\n" * 2 success = '' failure = '' failure_name_only = '' total_success = 0 total_failure = 0 @tests.each do |name, stats| runs = stats[:runs] pass = stats[:pass] fail = stats[:fail] timedout = '' timedout = '-- TIMED OUT' if stats[:timedout] == true line = "#{name}, runs: #{runs}, pass: #{pass}," + " fail: #{fail} #{timedout}\n" if fail > 0 && pass <= 0 failure_name_only += "#{File.basename(name)}\n" failure += line total_failure += 1 else success += line total_success += 1 end end total_tests = total_success + total_failure out = "#{total_tests} Tests\n\n" out += "Failure (#{total_failure}):\n#{failure}\n" unless failure.empty? out += "Success (#{total_success}):\n#{success}" unless success.empty? time_now = Time.now duration = time_now - @start_time duration = ChronicDuration.output(duration.round) || '0s' out += "\nFinished in #{duration}\n" time_format = '%b %d %l:%M %P' time_format2 = '%l:%M %P' out += "#{@start_time.strftime(time_format)} - #{time_now.strftime(time_format2)}" month_day_year = Time.now.strftime '%-m/%-d/%Y' success_percent = (total_success.to_f/total_tests.to_f*100).round(2) success_percent = 100 if total_failure <= 0 google_docs_line = [month_day_year, total_tests, total_failure, total_success, success_percent].join("\t") out += "\n#{google_docs_line}" out += "\n--\n" if save_file File.open(@fail_file, 'w') do |f| f.puts failure_name_only end # overwrite file File.open(@result_file, 'w') do |f| f.puts out end end puts out end