class Pwrake::Report

Constants

HTML_HEAD

Attributes

base[R]
cmd_elap[R]
cmd_file[R]
cmd_stat[R]
elap[R]
html_file[R]
id[R]
ncore[R]
sh_table[R]
task_table[R]

Public Class Methods

new(option,pattern) click to toggle source
# File lib/pwrake/report/report.rb, line 35
def initialize(option,pattern)
  @dir = option['REPORT_DIR']
  if !File.directory?(@dir)
    raise ArgumentError,"Could not find log directory: #{@dir}"
  end
  @pattern = pattern

  @@id = @@id.succ
  @id = @@id
  @base = @dir

  @img_fmt = option['REPORT_IMAGE'] || 'png'

  @cmd_file = File.join(@dir,option['COMMAND_CSV_FILE'])
  @task_file = File.join(@dir,option['TASK_CSV_FILE'])
  @html_file = File.join(@dir,'report.html')

  begin
    @sh_table = CSV.read(@cmd_file,:headers=>true,:skip_lines=>/\A#/)
  rescue
    $stderr.puts "error in reading "+@cmd_file
    $stderr.puts $!, $@
    exit
  end

  h = {}
  @elap_sum = 0
  @elap_core_sum = 0
  @sh_table.each do |row|
    if host = row['host']
      h[host] = true
    end
    t = row['elap_time'].to_f
    @elap_sum += t
    @elap_core_sum += t * row['ncore'].to_f
  end
  @hosts = h.keys.sort
  @start_time = Time.parse(@sh_table[0]["start_time"])
  @end_time = Time.parse(@sh_table[-1]["start_time"])
  @elap = @end_time - @start_time
  read_elap_each_cmd
  make_cmd_stat

  @stat = TaskStat.new(@task_file,@sh_table)
  @ncore = @stat.ncore
end

Public Instance Methods

find_single_file(pattern) click to toggle source
# File lib/pwrake/report/report.rb, line 88
def find_single_file(pattern)
  g = Dir.glob(File.join(@dir,pattern))
  case g.size
  when 0
    raise ArgumentError, "Could not find any file with '#{pattern}' in #{@dir}"
  when 1
  else
    raise ArgumentError, "Found multiple files '#{pattern}' in #{@dir}"
  end
  g[0]
end
format_comma(x) click to toggle source
# File lib/pwrake/report/report.rb, line 191
def format_comma(x)
  x.to_s.gsub(/(?<=\d)(?=(?:\d\d\d)+(?!\d))/, ',')
end
get_command(s) click to toggle source
# File lib/pwrake/report/report.rb, line 140
def get_command(s)
  case s
  when /^\s*\((.*)$/
    get_command($1)
  when /^\s*\w+=\S+\s+(.*)$/
    get_command($1)
  when /^\s*([\w.,~^\/=+-]+)(.*)$/
    cmd, rest = $1, $2
    case cmd
    when "cd"
      if /[^;]*;(.*)$/ =~ rest
        return get_command($1)
      end
    when /^ruby[\d.]*$/
      case rest
      when /([\w.,~^\/=+-]+\.rb)\b/
        return "#{cmd} #{$1}"
      when /\s+-e\s+("[^"]*"|'[^']*'|\S+)/
        return "#{cmd} -e #{$1}"
      end
    when /^python[\d.]*$/
      case rest
      when /([\w.,~^\/=+-]+\.py)\b/
        return "#{cmd} #{$1}"
      when /\s+-c\s+("[^"]*"|'[^']*'|\S+)/
        return "python -c #{$1}"
      end
    when /^perl[\d.]*$/
      case rest
      when /([\w.,~^\/=+-]+\.pl)\b/
        return "#{cmd} #{$1}"
      when /\s+-e\s+("[^"]*"|'[^']*'|\S+)/
        return "#{cmd} -e #{$1}"
      end
    end
    cmd
  else
    s[0..15]
  end
end
histogram_plot() click to toggle source
# File lib/pwrake/report/report.rb, line 310
    def histogram_plot
      command_list = []
      @cmd_stat.each do |cmd,stat|
        if stat.n > 2
          command_list << cmd
        end
      end
      hist_image = @base+"/hist."+@img_fmt
      if system("which gnuplot >/dev/null 2>&1")
      IO.popen("gnuplot","r+") do |f|
        f.puts "
set terminal #{@img_fmt} # size 480,360
set output '#{hist_image}'
set ylabel 'histogram'
set xlabel 'Execution time (sec)'
set logscale x
set title 'histogram of elapsed time'"
        a = []

        command_list.each_with_index do |cmd,i|
          a << "'-' w histeps ls #{i+1} title ''"
          a << "'-' w lines ls #{i+1} title #{cmd.inspect}"
        end
        f.puts "plot "+ a.join(',')

        command_list.each do |cmd|
          stat = @cmd_stat[cmd]
          2.times do
            stat.hist_each do |x1,x2,y|
              x = Math.sqrt(x1*x2)
              f.printf "%f %d\n", x, y
            end
            f.puts "e"
          end
        end
      end
      end
      hist_image
    end
id_str() click to toggle source
# File lib/pwrake/report/report.rb, line 100
def id_str
  if @@id_fmt.nil?
    id_len = Math.log10(@@id).floor + 1
    @@id_fmt = "#%0#{id_len}d"
  end
  @@id_fmt % @id
end
make_cmd_stat() click to toggle source
# File lib/pwrake/report/report.rb, line 181
def make_cmd_stat
  @cmd_stat = {}
  @cmd_elap.each do |cmd,elap|
    @cmd_stat[cmd] = s = Stat.new(elap)
    if elap.size > 1
      s.make_logx_histogram(1.0/8)
    end
  end
end
read_elap_each_cmd() click to toggle source
# File lib/pwrake/report/report.rb, line 108
def read_elap_each_cmd
  @cmd_elap = {}
  @sh_table.each do |row|
    command = row['command']
    elap = row['elap_time']
    if command && elap
      elap = elap.to_f
      found = nil
      @pattern.each do |cmd,regex|
        if regex =~ command
          if a = @cmd_elap[cmd]
            a << elap
          else
            @cmd_elap[cmd] = [elap]
          end
          found = true
        end
      end
      if !found
        if cmd = get_command( command )
          if a = @cmd_elap[cmd]
            a << elap
          else
            @cmd_elap[cmd] = [elap]
          end
        end
      end
    end
  end
  @cmd_elap
end
report_html() click to toggle source
# File lib/pwrake/report/report.rb, line 211
def report_html
  html = HTML_HEAD + "<body><h1>Pwrake Statistics</h1>\n"
  html << "<h2>Workflow</h2>\n"
  html << "<table>\n"
  html << "<tr><th>log directory</th><td>#{@base}</td><tr>\n"
  html << "<tr><th>ncore</th><td>#{@ncore}</td><tr>\n"
  html << "<tr><th>elapsed time</th><td>%.3f sec</td><tr>\n"%[@elap]
  html << "<tr><th>accumulated process time</th><td>%.3f sec</td><tr>\n"%[@elap_sum]
  html << "<tr><th>occupancy</th><td>%.3f %%</td><tr>\n"%[@elap_core_sum/@elap/@ncore*100]
  html << "<tr><th>start time</th><td>#{@start_time}</td><tr>\n"
  html << "<tr><th>end time</th><td>#{@end_time}</td><tr>\n"
  html << "</table><br/>\n"
  html << "<table>\n"
  html << "<tr><th colspan=5>#{@hosts.size} hosts &times; #{@ncore.fdiv @hosts.size} cores/host</th><tr>\n"
  @hosts.each_slice(5) do |a|
    html << "<tr>"+a.map{|h|"<td>#{h}</td>"}.join("")+"<tr>\n"
  end
  html << "</table>\n"
  html << "<h2>Parallelism</h2>\n"
  fimg = Parallelism.plot_parallelism(@sh_table,@base,@img_fmt)
  html << "<img src='./#{File.basename(fimg)}' align='top'/></br>\n"

  html << "<h2>Parallelism by command</h2>\n"
  fimg3 = Parallelism.plot_parallelism_by_pattern(@sh_table,@base,@pattern,@img_fmt)
  html << "<img src='./#{File.basename(fimg3)}' align='top'/></br>\n"

  html << "<h2>Parallelism by host</h2>\n"
  fimg2 = Parallelism.plot_parallelism_by_host(@sh_table,@base,@img_fmt)
  html << "<img src='./#{File.basename(fimg2)}' align='top'/></br>\n"

  html << "<h2>Command time statistics</h2>\n"
  html << "<table>\n"
  html << Stat.html_th
  @cmd_stat.each do |cmd,stat|
    html << "<tr><td>#{cmd}</td>"
    html << stat.html_td
    html << "</tr>\n"
  end
  html << "<table>\n"
  html << "<img src='./#{File.basename(histogram_plot)}' align='top'/></br>\n"

  html << "<h2>Locality statistics</h2>\n"
  html << "<table>\n"

  html << "<tr><th></th><th rowspan=3>gross elapsed time (sec)</th><th></th>"
  html << "<th colspan=6>read</th>"
  html << "<th></th>"
  html << "<th colspan=6>write</th>"
  html << "</tr>\n"


  html << "<tr><th></th><th></th>"
  html << "<th colspan=3>count</th><th colspan=3>file size (bytes)</th>"
  html << "<th></th>"
  html << "<th colspan=3>count</th><th colspan=3>file size (bytes)</th>"
  html << "</tr>\n"

  html << "<tr><th>host</th><th></th>"
  html << "<th>local</th><th>remote</th><th>total</th>"
  html << "<th>local</th><th>remote</th><th>total</th>"
  html << "<th></th>"
  html << "<th>local</th><th>remote</th><th>total</th>"
  html << "<th>local</th><th>remote</th><th>total</th>"
  html << "</tr>\n"
  @stat.exec_hosts.each do |h|
    if h.to_s!="" || @stat[h,nil,:elap]!=0
      html << "<tr><td>#{h}</td>"
      html << "<td align='right'>%.3f</td>" % @stat[h,nil,:elap]
      html << "<td></td>"
      html << tr_count(@stat[h,true,:in_num],@stat[h,false,:in_num])
      html << tr_count(@stat[h,true,:in_size],@stat[h,false,:in_size])
      html << "<td></td>"
      html << tr_count(@stat[h,true,:out_num],@stat[h,false,:out_num])
      html << tr_count(@stat[h,true,:out_size],@stat[h,false,:out_size])
      html << "</tr>\n"
    end
  end
  html << "<tr><td>total</td>"
  html << "<td align='right'>%.3f</td>" % @stat.total(nil,:elap)
  html << "<td></td>"
  html << tr_count(@stat.total(true,:in_num),@stat.total(false,:in_num))
  html << tr_count(@stat.total(true,:in_size),@stat.total(false,:in_size))
  html << "<td></td>"
  html << tr_count(@stat.total(true,:out_num),@stat.total(false,:out_num))
  html << tr_count(@stat.total(true,:out_size),@stat.total(false,:out_size))

  html << "</tr>\n"
  html << "<table>\n"

  html << "</body></html>\n"
  File.open(@html_file,"w") do |f|
    f.puts html
  end
  #puts "generate "+@html_file

  printf "%s,%d,%d,%d,%d\n",@html_file, @stat.total(true,:in_num),@stat.total(false,:in_num),@stat.total(true,:in_size),@stat.total(false,:in_size)
end
tr_count(x,y) click to toggle source
# File lib/pwrake/report/report.rb, line 195
def tr_count(x,y)
  sum = x+y
  if sum==0
    xp = "--%"
    yp = "--%"
  else
    xp = "%.2f%%"%(x*100.0/sum)
    yp = "%.2f%%"%(y*100.0/sum)
  end
  td = "<td align='right' valign='top'>"
  return \
    td + '%s<br/>%s</td>' % [format_comma(x),xp] +
    td + '%s<br/>%s</td>' % [format_comma(y),yp] +
    td + "%s</td>" % format_comma(sum)
end