module Pwrake::Parallelism

Public Instance Methods

count_start_end_by_pattern(csvtable, pattern) click to toggle source
# File lib/pwrake/report/parallelism.rb, line 249
def count_start_end_by_pattern(csvtable, pattern)
  h = Hash.new
  start_time = []

  csvtable.each do |row|
    cmd = get_command_key(row['command'], pattern)
    push_time_by_key(h, row, cmd, start_time)
  end

  h.each do |k,a|
    a.sort!{|x,y| x[0]<=>y[0]}
  end
  h
end
count_start_end_from_csv(file) click to toggle source
# File lib/pwrake/report/parallelism.rb, line 22
def count_start_end_from_csv(file)
  a = []
  start_time = []

  CSV.foreach(file,:headers=>true) do |row|
    push_time_event(a, row, start_time)
  end

  a.sort{|x,y| x[0]<=>y[0]}
end
count_start_end_from_csv_table(csvtable) click to toggle source
# File lib/pwrake/report/parallelism.rb, line 33
def count_start_end_from_csv_table(csvtable)
  a = []
  start_time = []

  csvtable.each do |row|
    push_time_event(a, row, start_time)
  end

  a.sort{|x,y| x[0]<=>y[0]}
end
exec_density(a) click to toggle source
# File lib/pwrake/report/parallelism.rb, line 44
def exec_density(a)
  reso = 0.1
  delta = 1/reso
  t_end = a.last[0]
  n = (t_end/reso).round + 1
  d = (n+1).times.map{|i| [reso*i,0]}
  i = 0
  a.each do |x|
    while d[i+1][0] <= x[0]
      i += 1
    end
    if x[1] > 0
      d[i][1] += delta
    end
  end
  d
end
get_command_key(s, pattern=[]) click to toggle source
# File lib/pwrake/report/parallelism.rb, line 226
def get_command_key(s, pattern=[])
  pattern.each do |cmd,regex|
    if regex =~ s
      return cmd
    end
  end
  case s
  when /^\s*\((.*)$/
    get_command_key($1)
  when /^\s*\w+=\S+\s+(.*)$/
    get_command_key($1)
  when /^\s*([\w.,~^\/=+-]+)(.*)$/
    cmd, rest = $1, $2
    if cmd == "cd" && /[^;]*;(.*)$/ =~ rest
      return get_command_key($1)
    else
      cmd
    end
  else
    s[0..15]
  end
end
plot_parallelism(csvtable, base, fmt) click to toggle source
# File lib/pwrake/report/parallelism.rb, line 63
    def plot_parallelism(csvtable, base, fmt)
      fimg = base+'/parallelism.'+fmt
      a = count_start_end_from_csv_table(csvtable)
      return fimg if a.size < 4

      #density = exec_density(a)

      n = a.size
      i = 0
      y = 0
      y_max = 0

      para = []
      begin
        t = 0
        y_pre = 0
        while i < n
          if a[i][0]-t > 0.001
            para.push "%.3f %d" % [t, y_pre]
            t = a[i][0]
            para.push "%.3f %d" % [t, y]
          end
          y += a[i][1]
          y_pre = y
          y_max = y if y > y_max
          i += 1
        end
      rescue
        p a[i]
      end

      t_end = (a.last)[0]

      begin
      if system("which gnuplot >/dev/null 2>&1")
        IO.popen("gnuplot","r+") do |f|
          f.print "
set terminal #{fmt}
set output '#{fimg}'
set xlabel 'time (sec)'
set ylabel '# of cores'
set ytics nomirror

set arrow 1 from #{t_end},#{y_max*0.5} to #{t_end},0 linecolor rgb 'blue'
set label 1 \"#{t_end}\\nsec\" at first #{t_end},#{y_max*0.5} right front textcolor rgb 'blue'

plot '-' w l notitle
"
          para.each do |x|
            f.puts x
          end
        end
      end
      rescue => exc
        $stderr.puts exc
        $stderr.puts exc.backtrace.join("\n")
      end

      #puts "Parallelism plot: #{fimg}"
      fimg
    end
plot_parallelism2(csvtable, base, fmt) click to toggle source
# File lib/pwrake/report/parallelism.rb, line 126
    def plot_parallelism2(csvtable, base, fmt)
      fimg = base+'/parallelism.'+fmt
      a = count_start_end_from_csv_table(csvtable)
      return fimg if a.size < 4

      density = exec_density(a)

      n = a.size
      i = 0
      y = 0
      y_max = 0

      para = []
      begin
        t = 0
        y_pre = 0
        while i < n
          if a[i][0]-t > 0.001
            para.push "%.3f %d" % [t, y_pre]
            t = a[i][0]
            para.push "%.3f %d" % [t, y]
          end
          y += a[i][1]
          y_pre = y
          y_max = y if y > y_max
          i += 1
        end
      rescue
        p a[i]
      end

      t_end = (a.last)[0]

      begin
      if system("which gnuplot >/dev/null 2>&1")
      IO.popen("gnuplot","r+") do |f|
        f.print "
set terminal #{fmt}
set output '#{fimg}'
set xlabel 'time (sec)'
set ylabel '# of cores'
set y2tics
set ytics nomirror
set y2label 'exec/sec'

set arrow 1 from #{t_end},#{y_max*0.5} to #{t_end},0 linecolor rgb 'blue'
set label 1 \"#{t_end}\\nsec\" at first #{t_end},#{y_max*0.5} right front textcolor rgb 'blue'

plot '-' w l axis x1y1 title 'parallelism', '-' w l axis x1y2 title 'exec/sec'
"
        para.each do |x|
          f.puts x
        end
        f.puts "e"

        density.each do |t,d|
          f.puts "#{t} #{d}"
        end
      end
      end
      rescue => exc
        $stderr.puts exc
        $stderr.puts exc.backtrace.join("\n")
      end

      #puts "Parallelism plot: #{fimg}"
      fimg
    end
plot_parallelism_by_host(csvtable,base,fmt) click to toggle source
# File lib/pwrake/report/parallelism.rb, line 342
    def plot_parallelism_by_host(csvtable,base,fmt)
      fimg = base+"/para_host."+fmt
      data = read_time_by_host_from_csv(csvtable)
      return fimg if data.size == 0

      grid = []
      hosts = data.keys.sort
      hosts.each do |h|
        a = timeline_to_grid(data[h])
        grid << a
      end

      begin
      if system("which gnuplot >/dev/null 2>&1")
      IO.popen("gnuplot","r+") do |f|
        f.puts "
set terminal #{fmt}
set output '#{fimg}'
#set rmargin 7
set lmargin 16
set pm3d map
set pm3d corners2color c1
set xlabel 'time (sec)'
set ytics nomirror
set ticslevel 0
set format y ''
"
        hosts.each_with_index do |h,i|
          if /^([^.]+)\./ =~ h
            h = $1
          end
          f.puts "set ytics add ('#{h}' #{i+0.5})"
        end
        f.puts "splot '-' using 2:1:3 with pm3d title ''"

        grid.each_with_index do |a,j|
          a.each do |x|
            f.printf "%g %g %d\n", j, x[0], x[1]
          end
          f.printf "\n"
        end
        j = grid.size
        grid.last.each do |x|
          f.printf "%g %g %d\n", j, x[0], x[1]
        end
        f.printf "e\n"
      end
      end
      rescue => exc
        $stderr.puts exc
        $stderr.puts exc.backtrace.join("\n")
      end
      fimg
    end
plot_parallelism_by_pattern(csvtable, base, pattern, fmt) click to toggle source
# File lib/pwrake/report/parallelism.rb, line 264
    def plot_parallelism_by_pattern(csvtable, base, pattern, fmt)
      y_max = 0
      t_end = 0
      para = {}
      t_pat = count_start_end_by_pattern(csvtable, pattern)
      t_pat.each do |cmd,a|
        n = a.size
        i = 0
        y = 0
        para[cmd] = dat = []
        begin
          t = 0
          y_pre = 0
          while i < n
            if a[i][0]-t > 0.001
              dat.push "%.3f %d" % [t, y_pre]
              t = a[i][0]
              dat.push "%.3f %d" % [t, y]
            end
            y += a[i][1]
            y_pre = y
            y_max = y if y > y_max
            i += 1
          end
        rescue
          p a[i]
        end
        if (a.last)[0] > t_end
          t_end = (a.last)[0]
        end
      end

      fimg = base+'/para_cmd.'+fmt

      if system("which gnuplot >/dev/null 2>&1")
      IO.popen("gnuplot","r+") do |f|
        #begin f = $stdout
        f.print "
set terminal #{fmt}
set output '#{fimg}'
set xlabel 'time (sec)'
set ylabel '# of cores'
"
        f.print "plot "
        f.puts para.map{|cmd,re| "'-' w l title #{cmd.inspect}"}.join(",")
        para.each do |cmd,dat|
          dat.each do |x|
            f.puts x
          end
          f.puts "e"
        end
      end
      end

      #puts "Parallelism plot: #{fimg}"
      fimg
    end
push_time_by_key(h, row, key, start_time) click to toggle source
# File lib/pwrake/report/parallelism.rb, line 196
    def push_time_by_key(h, row, key, start_time)
      command = row['command']
      if command == 'pwrake_profile_start'
        start_time[0] = Time.parse(row['start_time'])
      elsif command == 'pwrake_profile_end'
        t = Time.parse(row['start_time']) - start_time[0]
        h.each_value do |v|
          v << [t,0]
        end
      elsif start_time[0]
        a = (h[key] ||= [])
        n = begin Integer(row['ncore']) rescue 1 end
        t = Time.parse(row['start_time']) - start_time[0]
        a << [t,+n]
        t = Time.parse(row['end_time']) - start_time[0]
        a << [t,-n]
      end
    end

    def read_time_by_host_from_csv(csvtable)
      a = {}
      start_time = []

      csvtable.each do |row|
        push_time_by_key(a, row, row['host'], start_time)
      end
      a
    end


    def get_command_key(s, pattern=[])
      pattern.each do |cmd,regex|
        if regex =~ s
          return cmd
        end
      end
      case s
      when /^\s*\((.*)$/
        get_command_key($1)
      when /^\s*\w+=\S+\s+(.*)$/
        get_command_key($1)
      when /^\s*([\w.,~^\/=+-]+)(.*)$/
        cmd, rest = $1, $2
        if cmd == "cd" && /[^;]*;(.*)$/ =~ rest
          return get_command_key($1)
        else
          cmd
        end
      else
        s[0..15]
      end
    end

    def count_start_end_by_pattern(csvtable, pattern)
      h = Hash.new
      start_time = []

      csvtable.each do |row|
        cmd = get_command_key(row['command'], pattern)
        push_time_by_key(h, row, cmd, start_time)
      end

      h.each do |k,a|
        a.sort!{|x,y| x[0]<=>y[0]}
      end
      h
    end

    def plot_parallelism_by_pattern(csvtable, base, pattern, fmt)
      y_max = 0
      t_end = 0
      para = {}
      t_pat = count_start_end_by_pattern(csvtable, pattern)
      t_pat.each do |cmd,a|
        n = a.size
        i = 0
        y = 0
        para[cmd] = dat = []
        begin
          t = 0
          y_pre = 0
          while i < n
            if a[i][0]-t > 0.001
              dat.push "%.3f %d" % [t, y_pre]
              t = a[i][0]
              dat.push "%.3f %d" % [t, y]
            end
            y += a[i][1]
            y_pre = y
            y_max = y if y > y_max
            i += 1
          end
        rescue
          p a[i]
        end
        if (a.last)[0] > t_end
          t_end = (a.last)[0]
        end
      end

      fimg = base+'/para_cmd.'+fmt

      if system("which gnuplot >/dev/null 2>&1")
      IO.popen("gnuplot","r+") do |f|
        #begin f = $stdout
        f.print "
set terminal #{fmt}
set output '#{fimg}'
set xlabel 'time (sec)'
set ylabel '# of cores'
"
        f.print "plot "
        f.puts para.map{|cmd,re| "'-' w l title #{cmd.inspect}"}.join(",")
        para.each do |cmd,dat|
          dat.each do |x|
            f.puts x
          end
          f.puts "e"
        end
      end
      end

      #puts "Parallelism plot: #{fimg}"
      fimg
    end

    def timeline_to_grid(a)
      resolution = Rational(1,10)

      a = a.sort{|x,y| x[0]<=>y[0]}
      #t_end = (a.last)[0]
      #ngrid = (t_end/resolution).floor
      grid = [[0,0]]

      j = 0
      a.each do |x|
        i = (x[0]/resolution).floor
        while j < i
          grid[j+1] = [j*resolution,grid[j][1]]
          j += 1
        end
        grid[i][1] += x[1]
      end
      return grid
    end

    def plot_parallelism_by_host(csvtable,base,fmt)
      fimg = base+"/para_host."+fmt
      data = read_time_by_host_from_csv(csvtable)
      return fimg if data.size == 0

      grid = []
      hosts = data.keys.sort
      hosts.each do |h|
        a = timeline_to_grid(data[h])
        grid << a
      end

      begin
      if system("which gnuplot >/dev/null 2>&1")
      IO.popen("gnuplot","r+") do |f|
        f.puts "
set terminal #{fmt}
set output '#{fimg}'
#set rmargin 7
set lmargin 16
set pm3d map
set pm3d corners2color c1
set xlabel 'time (sec)'
set ytics nomirror
set ticslevel 0
set format y ''
"
        hosts.each_with_index do |h,i|
          if /^([^.]+)\./ =~ h
            h = $1
          end
          f.puts "set ytics add ('#{h}' #{i+0.5})"
        end
        f.puts "splot '-' using 2:1:3 with pm3d title ''"

        grid.each_with_index do |a,j|
          a.each do |x|
            f.printf "%g %g %d\n", j, x[0], x[1]
          end
          f.printf "\n"
        end
        j = grid.size
        grid.last.each do |x|
          f.printf "%g %g %d\n", j, x[0], x[1]
        end
        f.printf "e\n"
      end
      end
      rescue => exc
        $stderr.puts exc
        $stderr.puts exc.backtrace.join("\n")
      end
      fimg
    end

  end
push_time_event(a, row, start_time) click to toggle source
# File lib/pwrake/report/parallelism.rb, line 6
    def push_time_event(a, row, start_time)
      command = row['command']
      if command == 'pwrake_profile_start'
        start_time[0] = Time.parse(row['start_time'])
      elsif command == 'pwrake_profile_end'
        t = Time.parse(row['start_time']) - start_time[0]
        a << [t,0]
      elsif start_time[0]
        n = begin Integer(row['ncore']) rescue 1 end
        t = Time.parse(row['start_time']) - start_time[0]
        a << [t,+n]
        t = Time.parse(row['end_time']) - start_time[0]
        a << [t,-n]
      end
    end

    def count_start_end_from_csv(file)
      a = []
      start_time = []

      CSV.foreach(file,:headers=>true) do |row|
        push_time_event(a, row, start_time)
      end

      a.sort{|x,y| x[0]<=>y[0]}
    end

    def count_start_end_from_csv_table(csvtable)
      a = []
      start_time = []

      csvtable.each do |row|
        push_time_event(a, row, start_time)
      end

      a.sort{|x,y| x[0]<=>y[0]}
    end

    def exec_density(a)
      reso = 0.1
      delta = 1/reso
      t_end = a.last[0]
      n = (t_end/reso).round + 1
      d = (n+1).times.map{|i| [reso*i,0]}
      i = 0
      a.each do |x|
        while d[i+1][0] <= x[0]
          i += 1
        end
        if x[1] > 0
          d[i][1] += delta
        end
      end
      d
    end


    def plot_parallelism(csvtable, base, fmt)
      fimg = base+'/parallelism.'+fmt
      a = count_start_end_from_csv_table(csvtable)
      return fimg if a.size < 4

      #density = exec_density(a)

      n = a.size
      i = 0
      y = 0
      y_max = 0

      para = []
      begin
        t = 0
        y_pre = 0
        while i < n
          if a[i][0]-t > 0.001
            para.push "%.3f %d" % [t, y_pre]
            t = a[i][0]
            para.push "%.3f %d" % [t, y]
          end
          y += a[i][1]
          y_pre = y
          y_max = y if y > y_max
          i += 1
        end
      rescue
        p a[i]
      end

      t_end = (a.last)[0]

      begin
      if system("which gnuplot >/dev/null 2>&1")
        IO.popen("gnuplot","r+") do |f|
          f.print "
set terminal #{fmt}
set output '#{fimg}'
set xlabel 'time (sec)'
set ylabel '# of cores'
set ytics nomirror

set arrow 1 from #{t_end},#{y_max*0.5} to #{t_end},0 linecolor rgb 'blue'
set label 1 \"#{t_end}\\nsec\" at first #{t_end},#{y_max*0.5} right front textcolor rgb 'blue'

plot '-' w l notitle
"
          para.each do |x|
            f.puts x
          end
        end
      end
      rescue => exc
        $stderr.puts exc
        $stderr.puts exc.backtrace.join("\n")
      end

      #puts "Parallelism plot: #{fimg}"
      fimg
    end


    def plot_parallelism2(csvtable, base, fmt)
      fimg = base+'/parallelism.'+fmt
      a = count_start_end_from_csv_table(csvtable)
      return fimg if a.size < 4

      density = exec_density(a)

      n = a.size
      i = 0
      y = 0
      y_max = 0

      para = []
      begin
        t = 0
        y_pre = 0
        while i < n
          if a[i][0]-t > 0.001
            para.push "%.3f %d" % [t, y_pre]
            t = a[i][0]
            para.push "%.3f %d" % [t, y]
          end
          y += a[i][1]
          y_pre = y
          y_max = y if y > y_max
          i += 1
        end
      rescue
        p a[i]
      end

      t_end = (a.last)[0]

      begin
      if system("which gnuplot >/dev/null 2>&1")
      IO.popen("gnuplot","r+") do |f|
        f.print "
set terminal #{fmt}
set output '#{fimg}'
set xlabel 'time (sec)'
set ylabel '# of cores'
set y2tics
set ytics nomirror
set y2label 'exec/sec'

set arrow 1 from #{t_end},#{y_max*0.5} to #{t_end},0 linecolor rgb 'blue'
set label 1 \"#{t_end}\\nsec\" at first #{t_end},#{y_max*0.5} right front textcolor rgb 'blue'

plot '-' w l axis x1y1 title 'parallelism', '-' w l axis x1y2 title 'exec/sec'
"
        para.each do |x|
          f.puts x
        end
        f.puts "e"

        density.each do |t,d|
          f.puts "#{t} #{d}"
        end
      end
      end
      rescue => exc
        $stderr.puts exc
        $stderr.puts exc.backtrace.join("\n")
      end

      #puts "Parallelism plot: #{fimg}"
      fimg
    end


    def push_time_by_key(h, row, key, start_time)
      command = row['command']
      if command == 'pwrake_profile_start'
        start_time[0] = Time.parse(row['start_time'])
      elsif command == 'pwrake_profile_end'
        t = Time.parse(row['start_time']) - start_time[0]
        h.each_value do |v|
          v << [t,0]
        end
      elsif start_time[0]
        a = (h[key] ||= [])
        n = begin Integer(row['ncore']) rescue 1 end
        t = Time.parse(row['start_time']) - start_time[0]
        a << [t,+n]
        t = Time.parse(row['end_time']) - start_time[0]
        a << [t,-n]
      end
    end

    def read_time_by_host_from_csv(csvtable)
      a = {}
      start_time = []

      csvtable.each do |row|
        push_time_by_key(a, row, row['host'], start_time)
      end
      a
    end


    def get_command_key(s, pattern=[])
      pattern.each do |cmd,regex|
        if regex =~ s
          return cmd
        end
      end
      case s
      when /^\s*\((.*)$/
        get_command_key($1)
      when /^\s*\w+=\S+\s+(.*)$/
        get_command_key($1)
      when /^\s*([\w.,~^\/=+-]+)(.*)$/
        cmd, rest = $1, $2
        if cmd == "cd" && /[^;]*;(.*)$/ =~ rest
          return get_command_key($1)
        else
          cmd
        end
      else
        s[0..15]
      end
    end

    def count_start_end_by_pattern(csvtable, pattern)
      h = Hash.new
      start_time = []

      csvtable.each do |row|
        cmd = get_command_key(row['command'], pattern)
        push_time_by_key(h, row, cmd, start_time)
      end

      h.each do |k,a|
        a.sort!{|x,y| x[0]<=>y[0]}
      end
      h
    end

    def plot_parallelism_by_pattern(csvtable, base, pattern, fmt)
      y_max = 0
      t_end = 0
      para = {}
      t_pat = count_start_end_by_pattern(csvtable, pattern)
      t_pat.each do |cmd,a|
        n = a.size
        i = 0
        y = 0
        para[cmd] = dat = []
        begin
          t = 0
          y_pre = 0
          while i < n
            if a[i][0]-t > 0.001
              dat.push "%.3f %d" % [t, y_pre]
              t = a[i][0]
              dat.push "%.3f %d" % [t, y]
            end
            y += a[i][1]
            y_pre = y
            y_max = y if y > y_max
            i += 1
          end
        rescue
          p a[i]
        end
        if (a.last)[0] > t_end
          t_end = (a.last)[0]
        end
      end

      fimg = base+'/para_cmd.'+fmt

      if system("which gnuplot >/dev/null 2>&1")
      IO.popen("gnuplot","r+") do |f|
        #begin f = $stdout
        f.print "
set terminal #{fmt}
set output '#{fimg}'
set xlabel 'time (sec)'
set ylabel '# of cores'
"
        f.print "plot "
        f.puts para.map{|cmd,re| "'-' w l title #{cmd.inspect}"}.join(",")
        para.each do |cmd,dat|
          dat.each do |x|
            f.puts x
          end
          f.puts "e"
        end
      end
      end

      #puts "Parallelism plot: #{fimg}"
      fimg
    end

    def timeline_to_grid(a)
      resolution = Rational(1,10)

      a = a.sort{|x,y| x[0]<=>y[0]}
      #t_end = (a.last)[0]
      #ngrid = (t_end/resolution).floor
      grid = [[0,0]]

      j = 0
      a.each do |x|
        i = (x[0]/resolution).floor
        while j < i
          grid[j+1] = [j*resolution,grid[j][1]]
          j += 1
        end
        grid[i][1] += x[1]
      end
      return grid
    end

    def plot_parallelism_by_host(csvtable,base,fmt)
      fimg = base+"/para_host."+fmt
      data = read_time_by_host_from_csv(csvtable)
      return fimg if data.size == 0

      grid = []
      hosts = data.keys.sort
      hosts.each do |h|
        a = timeline_to_grid(data[h])
        grid << a
      end

      begin
      if system("which gnuplot >/dev/null 2>&1")
      IO.popen("gnuplot","r+") do |f|
        f.puts "
set terminal #{fmt}
set output '#{fimg}'
#set rmargin 7
set lmargin 16
set pm3d map
set pm3d corners2color c1
set xlabel 'time (sec)'
set ytics nomirror
set ticslevel 0
set format y ''
"
        hosts.each_with_index do |h,i|
          if /^([^.]+)\./ =~ h
            h = $1
          end
          f.puts "set ytics add ('#{h}' #{i+0.5})"
        end
        f.puts "splot '-' using 2:1:3 with pm3d title ''"

        grid.each_with_index do |a,j|
          a.each do |x|
            f.printf "%g %g %d\n", j, x[0], x[1]
          end
          f.printf "\n"
        end
        j = grid.size
        grid.last.each do |x|
          f.printf "%g %g %d\n", j, x[0], x[1]
        end
        f.printf "e\n"
      end
      end
      rescue => exc
        $stderr.puts exc
        $stderr.puts exc.backtrace.join("\n")
      end
      fimg
    end

  end
end
read_time_by_host_from_csv(csvtable) click to toggle source
# File lib/pwrake/report/parallelism.rb, line 215
def read_time_by_host_from_csv(csvtable)
  a = {}
  start_time = []

  csvtable.each do |row|
    push_time_by_key(a, row, row['host'], start_time)
  end
  a
end
timeline_to_grid(a) click to toggle source
# File lib/pwrake/report/parallelism.rb, line 322
def timeline_to_grid(a)
  resolution = Rational(1,10)

  a = a.sort{|x,y| x[0]<=>y[0]}
  #t_end = (a.last)[0]
  #ngrid = (t_end/resolution).floor
  grid = [[0,0]]

  j = 0
  a.each do |x|
    i = (x[0]/resolution).floor
    while j < i
      grid[j+1] = [j*resolution,grid[j][1]]
      j += 1
    end
    grid[i][1] += x[1]
  end
  return grid
end