class Batch
Attributes
attrib[RW]
dependencies[RW]
id[RW]
init[RW]
iterator[RW]
jobs[RW]
main_command[RW]
name[RW]
parent[RW]
Public Class Methods
new(tag, init, main_command, id, exec_folder)
click to toggle source
# File lib/autoflow/batch.rb, line 23 def initialize(tag, init, main_command, id, exec_folder) @regex_deps = nil replace_regexp(tag, init, main_command) @name = nil @id = id @iterator = [nil] @parent = nil @dependencies = [] # [batch_name, dependency_type, keyword2replace] # nil => There isn't dependecies, # 'simple' => One job needs a previous job, # '1to1' => A job in a batch needs another job in other batch, # '*to1' => A job need a previous full batch of jobs # 'local' => A simple job needs one job of a batch @initialization = init @main_command = main_command @attrib = { :done => false, :folder => true, :buffer => false, :exec_folder => exec_folder, :cpu_asign => nil # number, list or mono }.merge(@@general_computation_attrib) get_name_and_iterators_and_modifiers(tag) set_execution_attrib set_cpu @jobs = [] @@all_batch[@name] = self end
set_general_attrib(attrib_hash)
click to toggle source
# File lib/autoflow/batch.rb, line 18 def self.set_general_attrib(attrib_hash) @@general_computation_attrib = attrib_hash end
Public Instance Methods
add_nested_iteration_relation(tmp_j, new_job)
click to toggle source
# File lib/autoflow/batch.rb, line 296 def add_nested_iteration_relation(tmp_j, new_job) query = @@nested_iteration_relations[tmp_j.name] if query.nil? @@nested_iteration_relations[tmp_j.name] = [new_job.name] else query << new_job.name end end
asign_child_batch()
click to toggle source
# File lib/autoflow/batch.rb, line 113 def asign_child_batch batches = [] if @main_command.class.to_s == 'Array' @main_command.each do |id| batch = get_batch(id) batch.parent = @name batches << batch end @main_command = batches end end
check_dependencies(tmp_j, iter, jobs)
click to toggle source
tmp_j => job to set dependencies in iteration iter => sufix of current iteration jobs => array of jobs which has the job dependency
# File lib/autoflow/batch.rb, line 354 def check_dependencies(tmp_j, iter, jobs) jobs_names = jobs.map{|job| job.name} deps = {} tmp_j.dependencies.each_with_index do |dep, i| deps[dep] = i if jobs_names.include?(dep) end deps.each do |name, index| dep = name+'_'+iter tmp_j.initialization.gsub!(name+')', dep+')') tmp_j.parameters.gsub!(name+')', dep+')') tmp_j.dependencies[index] = dep end end
check_dependencies_with_DinVar(command, dinamic_variables)
click to toggle source
# File lib/autoflow/batch.rb, line 459 def check_dependencies_with_DinVar(command, dinamic_variables) dep = [] dinamic_variables.each do |var, name| dep << [name, 'DinVar'] if command.include?(var) end return dep end
check_execution_modifiers(name, iter_type = false)
click to toggle source
# File lib/autoflow/batch.rb, line 159 def check_execution_modifiers(name, iter_type = false) #The last paremeter iused to indicate tha name is a iterator not an orignal node name done = false folder = true buffer = false done = true if name.include?('%') folder = false if name.include?('!') buffer = true if name.include?('&') if !iter_type name.gsub!(/&|\!|\%|\)/,'')# Delete function characters else name.gsub!(/&|\!|\%/,'')# Delete function characters end return name, done, folder, buffer end
check_regex_dependencies()
click to toggle source
# File lib/autoflow/batch.rb, line 305 def check_regex_dependencies if @regex_deps == 'tag' new_job_names = [] @iterator.each do |iter| new_names = find_job_names(iter.gsub(')', '')) new_job_names.concat(new_names) end @iterator = new_job_names.map{|nj| nj + ')'} if !new_job_names.empty? elsif @regex_deps == 'command' [@initialization, @main_command].each do |command| patterns = command.scan(/([^\s)]+)\)([^\s]*)/) if !patterns.empty? patterns.each do |putative_job, sufix| job_names = find_job_names(putative_job) if !job_names.empty? new_string = job_names.map{|jn| "#{jn})#{sufix}"}.join(' ') old_string = "#{putative_job})#{sufix}" command.gsub!(old_string, new_string) end end end end end end
collect_dinamic_variables(command)
click to toggle source
# File lib/autoflow/batch.rb, line 443 def collect_dinamic_variables(command) dinamic_variables = [] if !command.nil? && command.include?('env_manager') command =~ /env_manager "([^"]+)/ command =~ /env_manager '([^']+)/ if $1.nil? if !$1.nil? $1.split(';').each do |variable| name, value = variable.split('=') name.gsub!(' ', '') #Remove spaces dinamic_variables << [name, @name] end end end return dinamic_variables end
delete_jobs(jobs2delete, job_array)
click to toggle source
# File lib/autoflow/batch.rb, line 242 def delete_jobs(jobs2delete, job_array) jobs2delete.uniq! jobs2delete.sort{|s1, s2| s2 <=> s1}.each do |index| job_array.delete_at(index) end return job_array end
duplicate_job(tmp_j, sufix_name = '')
click to toggle source
# File lib/autoflow/batch.rb, line 232 def duplicate_job(tmp_j, sufix_name = '') new_job = tmp_j.clone new_job.name = tmp_j.name+'_'+sufix_name new_job.attrib = tmp_j.attrib.clone new_job.dependencies = tmp_j.dependencies.clone new_job.initialization = tmp_j.initialization.clone new_job.parameters = tmp_j.parameters.clone return new_job end
find_job_names(name)
click to toggle source
# File lib/autoflow/batch.rb, line 330 def find_job_names(name) final_names = [] intermediary_names = @@nested_iteration_relations[name] if !intermediary_names.nil? while !intermediary_names.empty? final_names = intermediary_names i_names = [] intermediary_names.each do |i_n| query = @@nested_iteration_relations[i_n] i_names.concat(query) if !query.nil? end if !i_names.empty? intermediary_names = i_names else break end end end return final_names end
get_batch(id)
click to toggle source
# File lib/autoflow/batch.rb, line 125 def get_batch(id) selected_batch = nil @@all_batch.each do |name, batch| if batch.id == id selected_batch = batch break end end return selected_batch end
get_dependencies_by_regexp(batch_pattern, iterator_pattern)
click to toggle source
# File lib/autoflow/batch.rb, line 85 def get_dependencies_by_regexp(batch_pattern, iterator_pattern) selected_batches = @@all_batch.keys.select{|cmd_name| cmd_name =~ /#{batch_pattern}/} job_names = [] selected_batches.each do |batch_name| iterators = @@all_batch[batch_name].iterator iterators = iterators.map{|it| it.gsub(/&|\!|\%|\)/,'')} if !iterators.first.nil? if iterator_pattern != '-' next if iterators.first.nil? iterators = iterators.select{|iter| iter =~ /#{iterator_pattern}/} end if !iterators.empty? if iterators.first.nil? job_names << batch_name else iterators.each do |iter| job_names << batch_name+iter end end end end return job_names end
get_jobs()
click to toggle source
# File lib/autoflow/batch.rb, line 250 def get_jobs jobs = [] #@@batch_iterator_relations[@name] = @iterator #?????? if @main_command.class.to_s == 'Array' # There are nested batchs temp_jobs = [] @main_command.each do |batch| temp_jobs.concat(batch.get_jobs) end jobs2delete = [] @iterator.each_with_index do |iter, i| temp_jobs.each_with_index do |tmp_j, tmp_i| new_job = duplicate_job(tmp_j, iter) check_dependencies(new_job, iter, temp_jobs) parse_iter(iter, @name, new_job) add_nested_iteration_relation(tmp_j, new_job) @@jobs_names << new_job.name jobs << new_job @jobs << new_job jobs2delete << tmp_i end end temp_jobs = delete_jobs(jobs2delete, temp_jobs) #Remove temporal jobs else check_regex_dependencies @iterator.each_with_index do |iter, num| job_attrib = @attrib.dup if !iter.nil? iter, done, job_attrib[:folder], job_attrib[:buffer] = check_execution_modifiers(iter, true) job_attrib[:done] = done if !@attrib[:done] # To keep attrib priority in batch on job end name = "#{@name}#{iter}" job_dependencies = [] batch_deps = @dependencies.length initialization = replace_dependencies(@initialization, job_dependencies, iter, num) parameters = replace_dependencies(@main_command, job_dependencies, iter, num) @dependencies.pop(@dependencies.length - batch_deps) # Clean temporal dependencies by regexp job = Program.new(name, initialization, parameters, job_dependencies, job_attrib) job.batch = @name @@jobs_names << job.name jobs << job @jobs << job end end return jobs end
get_jobs_by_batch_name(batch_name)
click to toggle source
# File lib/autoflow/batch.rb, line 536 def get_jobs_by_batch_name(batch_name) jobs = @jobs.select{|j| j.batch == batch_name} return jobs end
get_name_and_iterators_and_modifiers(tag)
click to toggle source
# File lib/autoflow/batch.rb, line 136 def get_name_and_iterators_and_modifiers(tag) tag =~ /(^.+)\[([^\]]+)\]\)/ # iterative node name = $1 if $1.nil? # Non iterative node (simple node) tag =~ /(^.+)\)/ name = $1 end @name , @attrib[:done], @attrib[:folder], @attrib[:buffer] = check_execution_modifiers(name) if !$2.nil? @iterator = [] #$2.split(';').map{|iter| iter.gsub(')','')}.each do |interval| $2.split(';').each do |interval| if interval.include?('-') limits = interval.split('-') @iterator.concat((limits.first..limits.last).to_a.map{|n| n.to_s}) else @iterator << interval end end end @@batch_iterator_relations[@name] = @iterator end
get_root(batch_name)
click to toggle source
# File lib/autoflow/batch.rb, line 530 def get_root(batch_name) root_batch = @@all_batch[batch_name] root_batch = root_batch.get_root(root_batch.parent) if !root_batch.nil? && !root_batch.parent.nil? return root_batch end
get_string_position(substr, string)
click to toggle source
# File lib/autoflow/batch.rb, line 419 def get_string_position(substr, string) start = string.index(substr) ending = start + substr.length - 1 range = [start, ending] return range end
handle_dependencies(dinamic_variables)
click to toggle source
# File lib/autoflow/batch.rb, line 378 def handle_dependencies(dinamic_variables) [@initialization, @main_command].each do |instructions| if instructions.class.to_s == 'String' #scan_dependencies(instructions) # NOT NECESSARY? REMOVED BY COLLISION CON REGEX SYSTEM. THE DINAMYC VARIABLES ARE NO USED dinamic_variables.concat(collect_dinamic_variables(instructions)) @dependencies.concat(check_dependencies_with_DinVar(instructions, dinamic_variables)) end end return dinamic_variables end
has_jobs?()
click to toggle source
# File lib/autoflow/batch.rb, line 108 def has_jobs? res = !@jobs.empty? return res end
parse_iter(iter, name, job)
click to toggle source
# File lib/autoflow/batch.rb, line 368 def parse_iter(iter, name, job) job.parameters = set_iter(name, iter, job.parameters) job.initialization = set_iter(name, iter, job.initialization) end
replace_dependencies(command, job_dependencies, iter, num)
click to toggle source
# File lib/autoflow/batch.rb, line 467 def replace_dependencies(command, job_dependencies, iter, num) if !command.nil? command = command.gsub('(*)', "#{iter}") if command.class.to_s == 'String' scan_dependencies(command) @dependencies.each do |batch_name, dep_type, dep_keyword2replace, dep_info| if dep_type == 'simple' if @@all_batch[batch_name].parent.nil? new_string = batch_name + ')' end job_dependencies << batch_name elsif dep_type == '1to1' if @@all_batch[batch_name].parent.nil? || !@parent.nil? dep_name = "#{batch_name}#{@@batch_iterator_relations[batch_name][num]}" else root_batch = get_root(batch_name) selected_jobs = root_batch.get_jobs_by_batch_name(batch_name) dep_name = selected_jobs[num].name end job_dependencies << dep_name new_string = dep_name + ')' elsif dep_type == '*to1' if @@all_batch[batch_name].parent.nil? || !@parent.nil? new_string = @@batch_iterator_relations[batch_name].map{|iter| dep_name = batch_name + iter job_dependencies << dep_name "#{dep_name})#{dep_info}" }.join(' ') else root_batch = get_root(batch_name) selected_jobs = root_batch.get_jobs_by_batch_name(batch_name) new_string = selected_jobs.map{|j| job_dependencies << j.name "#{j.name + ')'}#{dep_info}" }.join(' ') end dep_keyword2replace = "#{dep_keyword2replace}#{dep_info}" elsif dep_type == 'local' if @@all_batch[batch_name].parent.nil? || !@parent.nil? #@@batch_iterator_relations.each do |key,val| # puts "#{key}\t#{val.inspect}" #end if @@batch_iterator_relations[batch_name].map{|iter| iter.gsub(')','')}.include?(dep_info) #This avoids cross dependencies by similar names, map used for regexp deps dep_name = batch_name + dep_info job_dependencies << dep_name new_string = dep_name + ')' end else dep_name = dep_keyword2replace.gsub(')','') #This avoids cross dependencies by similar names #if !@@jobs_names.include?(dep_name) job_dependencies << dep_name new_string = dep_name + ')' #end end elsif dep_type == 'DinVar' job_dependencies << batch_name if batch_name != @name # This condition avoids autodependencies end job_dependencies.uniq! command = command.gsub(dep_keyword2replace, new_string) if dep_type != 'DinVar' && !dep_keyword2replace.nil? && !new_string.nil? end end return command end
replace_regexp(tag, init, main_command)
click to toggle source
# File lib/autoflow/batch.rb, line 52 def replace_regexp(tag, init, main_command) scan_JobRegExp_tag(tag) if tag.include?('JobRegExp:') [init, main_command].each do |intructions| if intructions.class.to_s == 'String' while intructions.include?('!JobRegExp:') scan_JobRegExp(intructions) end end end #puts main_command.inspect end
scan_JobRegExp(command)
click to toggle source
# File lib/autoflow/batch.rb, line 64 def scan_JobRegExp(command) data = /!JobRegExp:([^ \n]+):([^ \n]+)!([^ \n]+)/.match(command) # *to1 with regexp #data[0] => reference string (command), data[1] => batch_pattern, data[2] => iterator_pattern, data[3] => adyacent string to regexp as regexp/file_name job_names = get_dependencies_by_regexp(data[1], data[2]) @regex_deps = 'command' if job_names.length > 0 new_string = job_names.map{|jn| jn + ')' + data[3] }.join(' ') command.gsub!(data[0], new_string) #puts command.inspect end
scan_JobRegExp_tag(tag)
click to toggle source
# File lib/autoflow/batch.rb, line 74 def scan_JobRegExp_tag(tag) data = /JobRegExp:([^ \n]+):([^;\] \n]+)/.match(tag) # 1to1 with regexp #data[0] => reference string (command), data[1] => batch_pattern, data[2] => iterator_pattern job_names = get_dependencies_by_regexp(data[1], data[2]) if job_names.length > 0 @regex_deps = 'tag' end new_string = job_names.map{|jn| jn + ')'}.join(';') tag.gsub!(data[0], new_string) end
scan_cpu(command)
click to toggle source
# File lib/autoflow/batch.rb, line 218 def scan_cpu(command) if command.include?('[cpu]') command.gsub!('[cpu]', @attrib[:cpu].to_s) @attrib[:cpu_asign] = 'number' elsif command.include?('[lcpu]') command.gsub!('[lcpu]', 'workers') @attrib[:cpu_asign] = 'list' elsif @attrib[:cpu_asign].nil? @attrib[:cpu_asign] = 'mono' end return command end
scan_dependencies(command)
click to toggle source
# File lib/autoflow/batch.rb, line 389 def scan_dependencies(command) if !command.nil?# When command is the initialize, sometimes can be undefined matched_regions = [] batches = [] @@all_batch.each do |k , val| #sorting is used to match last jobs first, and avoid small matches of first nodes batches << [k , val] end batches.reverse.each do |name, batch| if command.include?(name+')') && !string_overlap(matched_regions, name+')', command) @dependencies << [name, 'simple', name+')'] end if command.include?("!#{name}*!") && !string_overlap(matched_regions, "!#{name}*!", command) @dependencies << [name, '1to1', "!#{name}*!"] end if command.include?("!#{name}!") && !string_overlap(matched_regions, "!#{name}!", command) #command =~ /!#{name}!([^ \n]+)/ command.scan(/!#{name}!([^ \n]+)/).each do |string_match| @dependencies << [name, '*to1', "!#{name}!", string_match.first] end end local_dependencies = command.scan(/#{name}([^\( \n]+)\)/) local_dependencies.each do |local_dependency| if !string_overlap(matched_regions, "#{name}#{local_dependency.first}"+')', command) @dependencies << [name, 'local', "#{name}#{local_dependency.first}"+')', local_dependency.first] end end end end end
scan_resources(command)
click to toggle source
# File lib/autoflow/batch.rb, line 179 def scan_resources(command) resources_line = nil command.each_line do |line| if line.include?('resources:') line = line.chomp resources_line = line fields = line.split(' ') fields.each_with_index do |field, index| if field == '-c' @attrib[:cpu] = fields[index+1].to_i elsif field == '-m' @attrib[:mem] = fields[index+1] elsif field == '-n' @attrib[:node] = fields[index+1] elsif field == '-t' @attrib[:time] = fields[index+1] elsif field == '-u' @attrib[:multinode] = fields[index+1].to_i elsif field == '-A' @attrib[:additional_job_options] = fields[index+1].split(':') end end if fields.include?('-s') @attrib[:ntask] = true else @attrib[:ntask] = false end end end command.gsub!(resources_line, '') if !resources_line.nil? return command end
set_cpu()
click to toggle source
# File lib/autoflow/batch.rb, line 212 def set_cpu @initialization = scan_cpu(@initialization) if !@initialization.nil? @main_command = scan_cpu(@main_command) if @main_command.class.to_s == 'String' @attrib[:cpu] = 1 if @attrib[:cpu_asign] == 'mono' end
set_execution_attrib()
click to toggle source
# File lib/autoflow/batch.rb, line 174 def set_execution_attrib @initialization = scan_resources(@initialization) if !@initialization.nil? @main_command = scan_resources(@main_command) if @main_command.class.to_s == 'String' end
set_iter(name, iter, string)
click to toggle source
# File lib/autoflow/batch.rb, line 373 def set_iter(name, iter, string) string = string.gsub(name+'(+)', iter) return string end
string_overlap(matched_regions, substr, string)
click to toggle source
# File lib/autoflow/batch.rb, line 426 def string_overlap(matched_regions, substr, string) match = false range = get_string_position(substr, string) if !range.empty? matched_regions.each do |start, ending| if (range.first >= start && range.first <= ending) || (range.last >= start && range.last <= ending) || (range.first <= start && range.last >= ending) match = true break end end matched_regions << range end return match end