class Stack

Attributes

exec_folder[RW]
jobs[RW]
persist_variables[RW]

Public Class Methods

new(exec_folder, options) click to toggle source

PARSE TEMPLATE

# File lib/autoflow/stack.rb, line 12
def initialize(exec_folder, options)
        Batch.set_general_attrib({
                :cpu => options[:cpus], 
                :mem => options[:memory], 
                :time => options[:time],
                :node => options[:node_type],
                :multinode => options[:use_multinode],
                :ntask => options[:use_ntasks],
                :additional_job_options => options[:additional_job_options]
        })
        @@folder_name = :program_name
        @@folder_name = :job_name if options[:key_name]
        @commands = {}        
        @variables = {}
        @persist_variables = {}
        @@all_jobs_relations = {}
        @exec_folder = exec_folder #TODO move this to queue_manager
        @do_retry = options[:retry]
        @options = options
        @workflow = options[:workflow]
        @external_variables= options[:Variables] 
        @jobs = {}
end

Public Instance Methods

asign_folder(job) click to toggle source
# File lib/autoflow/stack.rb, line 180
def asign_folder(job) #TODO move this to queue_manager
        folder = nil
        if job.attrib[:folder]
                if @@folder_name == :program_name
                        program = File.join(job.attrib[:exec_folder], job.parameters.split(' ', 2).first)                   
                        count = 0
                        folder = program + "_#{"%04d" % count}"
                        while @@all_jobs_relations.values.include?(folder)
                                folder = program + "_#{"%04d" % count}"
                                count += 1
                        end
                elsif @@folder_name == :job_name
                        folder = File.join(job.attrib[:exec_folder], job.name)
                end
        else
                folder = job.attrib[:exec_folder]
        end
        @@all_jobs_relations[job.name.gsub(')','')] = folder
        job.attrib[:exec_folder] = folder 
end
comment_main_command() click to toggle source
# File lib/autoflow/stack.rb, line 228
def comment_main_command
        @jobs.each do |name, job|
                job.parameters = "##{job.parameters}"
        end
end
create_ids(nodes) click to toggle source
# File lib/autoflow/stack.rb, line 165
def create_ids(nodes)
        nodes.each_with_index do |node, i|
                node << i
        end
        return nodes
end
draw(name, name_type) click to toggle source
# File lib/autoflow/stack.rb, line 243
def draw(name, name_type)
        representation_type = '_structural'
        representation_type = '_semantic' if name_type.include?('t')
        if name_type.include?('b')
                representation_type << '_simplified'
                set = @commands
        else
                set = @jobs
        end
        name.gsub!(/\.\S+/,'')
        file = File.open(name+representation_type+'.dot','w')
        file.puts 'digraph G {', 'node[shape=box]'
        all_dependencies = []
        all_tag = []
        set.each do |id, tag|
                if name_type.include?('b')
                        tag_name = tag.main_command.split(' ').first+"_#{tag.id}"
                else 
                        tag_name = File.basename(tag.attrib[:exec_folder])
                end
                tag_name = id if name_type.include?('t')
                tag_name = tag_name + '(*)' if name_type.include?('b') && tag.iterator.length > 1
                
                all_tag << tag_name
                if tag.dependencies.length > 0
                        tag.dependencies.each do |dependencie, type, string|
                                if name_type.include?('b')
                                        dependencie_name = set[dependencie].main_command.split(' ').first+"_#{set[dependencie].id}"
                                else 
                                        dependencie_name = File.basename(set[dependencie].attrib[:exec_folder])                           
                                end
                                dependencie_name = dependencie if name_type.include?('t')

                                dependencie_name = dependencie_name + '(*)' if name_type.include?('b') && set[dependencie].iterator.length > 1
                                all_dependencies << dependencie_name

                                file.puts "\"#{dependencie_name}\"-> \"#{tag_name}\""
                        end
                else
                        file.puts "\"#{tag_name}\"[color=black, peripheries=2, style=filled, fillcolor=yellow]"
                end
        end
        all_tag.keep_if{|tag| !all_dependencies.include?(tag)}
        all_tag.each do |tag|
                if name_type.include?('b')
                        if !name_type.include?('f')
                                tag = tag + '(*)'  if !tag.include?('(*)') && set[tag].iterator.length > 1
                        else
                                id = tag.reverse.split('_',2).first.reverse.to_i
                                batch = nil
                                set.each do |id,tag|
                                        batch = tag if tag.id = id 
                                end
                                tag = tag + '(*)'  if batch.iterator.length > 1
                        end
                end
                file.puts "\"#{tag}\"[fontcolor=white, color=black, style=filled]"
        end
        file.puts '}'
        file.close
        system('dot -Tpdf '+name+representation_type+'.dot -o '+name+representation_type+'.pdf')
end
get_jobs() click to toggle source
# File lib/autoflow/stack.rb, line 201
def get_jobs
        jobs =[]
        @commands.each do |name, batch|
                next if batch.has_jobs? #parent batch (intermediates)
                batch.get_jobs.each do |j|
                        folder = asign_folder(j) #TODO move this to queue_manager
                        jobs << [j.name, j]
                end

        end
        jobs.each do |j_name, job| #TODO move this to queue_manager
                set_dependencies_path(job)
                j_name.gsub!(')','') #Clean function characters on name
                job.name.gsub!(')','')
        end
        return jobs
end
get_jobs_relations() click to toggle source
# File lib/autoflow/stack.rb, line 220
def get_jobs_relations
        hash = {}
        get_jobs.each do |name, job|
                hash[name] = job
        end
        return hash
end
inspect() click to toggle source

WORKFLOW REPRESENTATION

# File lib/autoflow/stack.rb, line 237
def inspect
        @jobs.each do |id, job|
                puts "#{id} > #{job.inspect}\t#{job.attrib[:done]}\n\t\e[32m#{job.dependencies.join("\n\t")}\e[0m"
        end
end
load_variables(variables_lines, variable_type) click to toggle source
# File lib/autoflow/stack.rb, line 73
def load_variables(variables_lines, variable_type)
        if !variables_lines.nil?
                variables_lines.each do |line|
                        line.chomp!
                        line.gsub!(/\s/,'')
                        pairs = line.split(',')
                        pairs.each do |pair|
                                #pair =~ /(.+)=(.+)/
                                #variable_type[$1] = $2
                                var, value = pair.split('=', 2)
                                variable_type[var] = value
                        end
                end
        end
end
parse!() click to toggle source
# File lib/autoflow/stack.rb, line 36
def parse!
        #Clean template
        @workflow.gsub!(/\#.+$/,'')   #Delete comments
        @workflow.gsub!("\t",'')              #Drop tabs
        @workflow.gsub!(/\n+/,"\n")   #Drop empty lines
        @workflow.gsub!(/^\s*/,'')

        #Parse template
        variables_lines = []
        persist_variables_lines = []
        node_lines = []

        node_beg = false              
        @workflow.each_line do |line|
                node_beg = true if line.include?('{')        # This check the context of a variable
                if line.include?('}')                                        # if a variable is within a node,
                        if node_beg                                                 # we consider tha is a bash variable not a static autoflow variable
                                node_beg = false
                        else
                                node_beg = true
                        end
                end
                if line =~ /^\$/ && !node_beg
                        variables_lines << line
                elsif line =~ /^\@/
                        persist_variables_lines << line.gsub('@','')
                else
                        node_lines << line
                end
        end
        load_variables(variables_lines, @variables)
        load_variables(@external_variables, @variables)
        load_variables(persist_variables_lines, @persist_variables)
        parse_nodes(node_lines)
        @jobs = get_jobs_relations
end
parse_nodes(execution_lines) click to toggle source
# File lib/autoflow/stack.rb, line 143
def parse_nodes(execution_lines)
        dinamic_variables = []
        nodes = scan_nodes(execution_lines)
        nodes = create_ids(nodes)
        
        #nodes.each do |tag, init, command, index|
        #     puts "#{tag.colorize(:red)}\t#{index}\n#{('-'*tag.length).colorize(:red)}\n#{init.chomp.colorize(:blue)}\n#{command.to_s.colorize(:green)}"
        #end

        nodes.each do |tag, init, command, index| #Takes the info of each node of workflow to create the job
                # Set batch
                new_batch = Batch.new(tag, init, command, index, @exec_folder)
                dinamic_variables.concat(new_batch.handle_dependencies(dinamic_variables))
                @commands[new_batch.name] = new_batch
        end

        # link each parent batch to a child batch
        @commands.each do |name, batch|
                batch.asign_child_batch
        end
end
replace_variables(string) click to toggle source
# File lib/autoflow/stack.rb, line 137
def replace_variables(string)
        @variables.each do |name, value|
                string.gsub!(name, value)
        end
end
scan_nodes(execution_lines) click to toggle source
# File lib/autoflow/stack.rb, line 89
        def scan_nodes(execution_lines)
                template_executions = execution_lines.join('')
                replace_variables(template_executions)
                # $1 => tag, $2 => initialize, $3 => main command
                #executions = template_executions.scan(/(^.+\))\s{0,}\{\s{0,}([^\?]{0,})\s{0,}\?\s([^\}]{1,})\s{0,}\}/)
#=begin
                executions = [] # tag, initialize, main_command
                states = {} #name => [state, id(position)]
                                        #t => tag, i => initialize , c => command
                open_nodes = []
                template_executions.each_line do |line|
                        line.strip! #Clean al whitespaces at beginning and the end of string
                        node = states[open_nodes.last] if !open_nodes.empty?
                        if line.empty?
                                next
                        # Create nodes and asign nodes states
                        #----------------------------------------
                        elsif line =~ /(\S*\)){$/ #Check tag and create node
                                name = $1
                                executions << [name, '', ''] # create node
                                states[name] = [:i, executions.length - 1]
                                open_nodes << name
                        elsif line == '?' #Check command
                                node[0] = :c
                        elsif line == '}' #Close node
                                finished_node = open_nodes.pop
                                if !open_nodes.empty?
                                        parent_node = states[open_nodes.last].last #position
                                        child_node = states[finished_node].last
                                        parent_execution = executions[parent_node]
                                        if parent_execution[2].class == String   
                                                parent_execution[2] = [child_node]
                                        else
                                                parent_execution[2] << child_node
                                        end
                                end
                        # Add lines to nodes
                        #----------------------
                        elsif states[open_nodes.last].first == :i #Add initialize line
                                executions[node.last][1] << line +"\n"
                        elsif states[open_nodes.last].first == :c #Add command line
                                executions[node.last][2] << line +"\n"                      
                        end
                end
#=end
                return executions
        end
set_dependencies_path(job) click to toggle source
# File lib/autoflow/stack.rb, line 172
def set_dependencies_path(job) #TODO move this to queue_manager
        job.dependencies.sort{|d1, d2| d2.length <=> d1.length}.each do |dep|
                path =  @@all_jobs_relations[dep]
                job.initialization.gsub!(dep+')', path) if !job.initialization.nil?
                job.parameters.gsub!(dep+')', path)
        end
end