class Object

Constants

BACKLOG_ELEMENT
CONTAINER_ELEMENT
IE_ELEMENT
NAME_ELEMENT
PATH_ELEMENT
RSA_PRIVATE

RSA_PRIVATE = OpenSSL::PKey::RSA.generate 2048 RSA_PUBLIC = RSA_PRIVATE.public_key

File.open(File.expand_path('../key.priv.pem', __FILE__), 'w') { |f| f.puts(RSA_PRIVATE.to_pem)} File.open(File.expand_path('../key.pem', __FILE__), 'w') { |f| f.puts(RSA_PUBLIC.to_pem)}

RSA_PUBLIC
STUK_ELEMENT
THUMBNAIL_ELEMENT

Public Class Methods

const_unset(const) click to toggle source
# File lib/libis/ingester/server/api/representer/base.rb, line 8
def self.const_unset(const)
  self.instance_eval { remove_const(const) }
end

Public Instance Methods

action_menu(process) click to toggle source
# File lib/libis/ingester/console/process_lib.rb, line 43
def action_menu(process)
  menu = {}
  menu[:halt] = Proc.new { quiet_sidekiq process; false } unless process.stopping?
  menu[:threads] = Proc.new { list_threads process; true }
  loop do
    break unless selection_menu('action', [], hidden: menu, prompt: nil, parent: process['tag'], layout: :one_line)
  end
end
base_opts(opts) click to toggle source
# File lib/libis/ingester/console/menu.rb, line 4
def base_opts(opts)
  opts.on('-h', '--help', 'Prints this help') do
    puts opts
    exit
  end
end
clean_string(s) click to toggle source
# File lib/libis/ingester/console/submit_lib.rb, line 60
def clean_string(s)
  s.split(/[^\w\/-]+/).reject { |x| x.empty? }.join('_').split(/[\/-]+/).reject{|x|x.empty?}.join('-')
end
close_report() click to toggle source
# File lib/libis/ingester/console/reorg_lib.rb, line 125
def close_report
  return unless @report
  if @report_type == :xml
    @report.puts '</report>'
  end
  @report.close
end
common_opts(opts) click to toggle source
# File lib/libis/ingester/console/include.rb, line 71
def common_opts(opts)
  opts.on('-c', '--config CONFIG', 'Config file') do |v|
    @options[:config] = v
  end
  opts.on('--version', 'Show version information') do
    puts "Libis::Tools ................ #{Libis::Tools::VERSION}"
    puts "Libis::Format ............... #{Libis::Format::VERSION}"
    puts "Libis::Workflow ............. #{Libis::Workflow::VERSION}"
    puts "Libis::Workflow::Mongoid .... #{Libis::Workflow::Mongoid::VERSION}"
    puts "Libis::Services ............. #{Libis::Services::VERSION}"
    puts "Libis::Ingester ............. #{Libis::Ingester::VERSION}"
    exit
  end
  base_opts(opts)
end
compare_entry(src, tgt) click to toggle source
# File lib/libis/ingester/console/reorg_lib.rb, line 195
def compare_entry(src, tgt)
  hasher = Libis::Tools::Checksum.new(:SHA256)
  hasher.digest(src) == hasher.digest(tgt)
end
config_file(config) click to toggle source
# File lib/libis/ingester/console/reorg_lib.rb, line 140
def config_file(config)
  File.join(ENV['HOME'], ".reorg#{config}.data")
end
configurations() click to toggle source
# File lib/libis/ingester/console/reorg_lib.rb, line 144
def configurations
  Dir.glob(File.join(ENV['HOME'], '.reorg*.data')).map {|x| x.scan(/reorg(.*).data$/).first.first rescue ''}
end
database_menu() click to toggle source
# File lib/libis/ingester/console/database_lib.rb, line 5
def database_menu
  loop do
    item = selection_menu(
        'Database menu',
        [:seed, :delete_orphans, :delete_finished_runs]
    )
    break unless item
    send("db_#{item}") if item.is_a?(Symbol)
    @options.clear
  end
end
db_delete_finished_runs() click to toggle source
# File lib/libis/ingester/console/database_lib.rb, line 27
def db_delete_finished_runs
  delete_all_finshed_runs
end
db_delete_orphans() click to toggle source
# File lib/libis/ingester/console/database_lib.rb, line 23
def db_delete_orphans
  delete_orphan_items
end
db_menu(title, items, options = {}, &block) click to toggle source
# File lib/libis/ingester/console/include.rb, line 21
def db_menu(title, items, options = {}, &block)
  @db_page ||= Hash.new(0)
  key = title.downcase.to_sym
  return @options[key] if @options[key]
  if (name = @options["#{key}_name".to_sym])
    item = items.find_by(name: name)
    if item
      @options[key] = item
      return item
    end
  end
  options[:proc] = lambda {|item| @options[title.downcase.to_sym] = item}
  block = Proc.new {|item| item.name} unless block_given?
  paging = paging_size
  if items.count > paging
    paged_items = items.limit(paging)
    no_pages = (items.count - 1) / paging + 1
    max_page = no_pages - 1
    min_page = 0
    page = @db_page[title]
    options[:hidden] ||= {}
    hidden = options[:hidden]
    loop do
      page = [page, min_page].max
      page = [page, max_page].min
      options[:hidden] = hidden.dup
      options[:hidden]['previous'] = Proc.new {:previous} if page > min_page
      options[:hidden]['next'] = Proc.new {:next} if page < max_page
      options[:hidden]['goto'] = Proc.new {:goto}
      result = selection_menu("#{title} (#{page * paging + 1}-#{(page + 1) * paging}) page #{page + 1}/#{no_pages}",
                              paged_items.offset(page * paging), options, &block)
      case result
      when :previous
        page -= 1 if page > min_page
      when :next
        page += 1 if page < max_page
      when :goto
        page = @hl.ask("Enter page number (#{min_page + 1}..#{no_pages})", Integer) do |q|
          q.in = Range.new(min_page + 1, no_pages)
        end - 1
      else
        @db_page[title] = page
        return result
      end
    end
  else
    selection_menu(title, items, options, &block)
  end
end
db_seed() click to toggle source
# File lib/libis/ingester/console/database_lib.rb, line 17
def db_seed
  puts 'Seeding database ...'
  @initializer.seed_database
  puts 'Done.'
end
delete_all_failed_runs(job = nil) click to toggle source
# File lib/libis/ingester/console/delete_lib.rb, line 51
def delete_all_failed_runs(job = nil)
  quiet = false
  # noinspection RubyResolve
  (job&.runs || ::Libis::Ingester::Run).no_timeout.each do |run|
    next unless run.check_status(:FAILED)
    if quiet
      puts "Deleting #{run.name} ..."
    else
      q = @hl.ask("Deleting #{run.name} ... OK ? [Yes/No/All/Quit] ") { |a| a.character = false, a.validate = /[ynaq]/i }.downcase
      next if q == 'n'
      return if q == 'q'
      quiet = true if q == 'a'
    end
    run.destroy!
  end
end
delete_all_finshed_runs(job = nil) click to toggle source
# File lib/libis/ingester/console/delete_lib.rb, line 34
def delete_all_finshed_runs(job = nil)
  quiet = false
  # noinspection RubyResolve
  (job&.runs || ::Libis::Ingester::Run).no_timeout.each do |run|
    next unless run.check_status(:DONE)
    if quiet
      puts "Deleting #{run.name} ..."
    else
      q = @hl.ask("Deleting #{run.name} ... OK ? [Yes/No/All/Quit] ") { |a| a.character = false, a.validate = /[ynaq]/i }.downcase
      next if q == 'n'
      return if q == 'q'
      quiet = true if q == 'a'
    end
    run.destroy!
  end
end
delete_all_runs(job = nil) click to toggle source
# File lib/libis/ingester/console/delete_lib.rb, line 68
def delete_all_runs(job = nil)
  quiet = false
  # noinspection RubyResolve
  (job&.runs || ::Libis::Ingester::Run).no_timeout.each do |run|
    if quiet
      puts "Deleting #{run.name} ..."
    else
      q = @hl.ask("Deleting #{run.name} ... OK ? [Yes/No/All/Quit] ") { |a| a.character = false, a.validate = /[ynaq]/i }.downcase
      next if q == 'n'
      return if q == 'q'
      quiet = true if q == 'a'
    end
    run.destroy!
  end
end
delete_item(item, quiet = false) click to toggle source
# File lib/libis/ingester/console/delete_lib.rb, line 27
def delete_item(item, quiet = false)
  item.destroy! if quiet || @hl.agree(
      "This will delete the #{item.class.to_s.split('::').last.downcase} '#{item.name}' from " +
          "#{item.parent.class.to_s.split('::').last.downcase} '#{item.parent.name}'. OK?", false
  )
end
delete_menu() click to toggle source
# File lib/libis/ingester/console/delete_lib.rb, line 4
def delete_menu
  loop do
      break unless select_organization
      loop do
        break unless select_job
        loop do
          @options[:job].reload_relations
          break unless select_run
          delete_run(@options[:run])
          @options[:job].save!
          @options[:run] = nil
        end
        @options[:job] = nil
      end
      @options[:organization] = nil
  end
end
delete_orphan_items() click to toggle source
# File lib/libis/ingester/console/delete_lib.rb, line 84
def delete_orphan_items
  quiet = false
  (::Libis::Ingester::Item).no_timeout.each do |item|
    next if item.is_a? Libis::Ingester::Run
    next if item.parent
    if quiet
      puts "Deleting #{item.name} ..."
    else
      q = @hl.ask("Deleting #{item.name} ... OK ? [Yes/No/All/Quit] ") { |a| a.character = false, a.validate = /[ynaq]/i }.downcase
      next if q == 'n'
      return if q == 'q'
      quiet = true if q == 'a'
    end
    item.destroy!
  end
end
delete_run(item, quiet = false) click to toggle source
# File lib/libis/ingester/console/delete_lib.rb, line 22
def delete_run(item, quiet = false)
  puts "Deleting #{item.name} ..." if quiet
  item.destroy! if quiet || @hl.agree("This will destroy all evidence of run #{item.name}. OK?", false)
end
for_csv(string) click to toggle source
# File lib/libis/ingester/console/reorg_lib.rb, line 82
def for_csv(string)
  ; string =~ /,\n/ ? "\"#{string.gsub('"', '""')}\"" : string;
end
for_tsv(string) click to toggle source
# File lib/libis/ingester/console/reorg_lib.rb, line 78
def for_tsv(string)
  ; string =~ /\t\n/ ? "\"#{string.gsub('"', '""')}\"" : string;
end
for_xml(string, type = :attr) click to toggle source
# File lib/libis/ingester/console/reorg_lib.rb, line 86
def for_xml(string, type = :attr)
  ; string.encode(xml: type);
end
for_yml(string) click to toggle source
# File lib/libis/ingester/console/reorg_lib.rb, line 90
def for_yml(string)
  ; string.inspect.to_yaml;
end
get_base_dir(base_dir) click to toggle source
# File lib/libis/ingester/console/reorg_lib.rb, line 18
def get_base_dir(base_dir)
  unless @unattended && base_dir
    puts
    puts 'First of all supply the path to the directory that needs to be reorganized.'
    base_dir = select_path(true, false, (base_dir || '.'))
  end
  File.absolute_path(base_dir)
end
get_config(config) click to toggle source
# File lib/libis/ingester/console/reorg_lib.rb, line 6
def get_config(config)
  unless @unattended || config
    puts
    puts 'You can supply a configuration name. Configurations are stored in a ${HOME}/.reorg<Config>.data file.'
    puts 'The configuration parameters you set are automatically retrieved and saved under that name.'
    puts 'These configurations exist:'
    puts_config
    config = @hl.ask('Configuration to use.') {|q| q.default = config}
  end
  config
end
get_dummy_operation(dummy_operation) click to toggle source
# File lib/libis/ingester/console/reorg_lib.rb, line 133
def get_dummy_operation(dummy_operation)
  if !@unattended || dummy_operation.nil?
    dummy_operation = !@hl.agree('Perform physical operation on the files?', true)
  end
  dummy_operation
end
get_file_operation(file_operation) click to toggle source
# File lib/libis/ingester/console/reorg_lib.rb, line 57
def get_file_operation(file_operation)
  if !@unattended || file_operation.nil?
    puts
    puts 'Which operation do you want to perform on the files?'
    file_operation = @hl.ask('File operation? ', [:move, :copy, :link]) {|q| q.default = file_operation || :move}
  end
  file_operation
end
get_initializer() click to toggle source
# File lib/libis/ingester/console/include.rb, line 93
def get_initializer
  @initializer = ::Libis::Ingester::Initializer.init(@options[:config] || 'site.config.yml')
end
get_operator_email() click to toggle source
# File lib/libis/ingester/console/menu.rb, line 17
def get_operator_email
  users = YAML.load_file('data/users.yml')
  @operator_email = @hl.choose do |menu|
    menu.prompt = 'Select user: '
    users.each { |u| menu.choice(u['name']) {u['email']} }
    menu.choice('other') { puts "Add user to 'data/users.yml' file"; exit }
  end
end
get_parse_regex(parse_regex) click to toggle source
# File lib/libis/ingester/console/reorg_lib.rb, line 27
def get_parse_regex(parse_regex)
  unless @unattended && parse_regex
    puts
    puts 'Now enter a regular expression that needs to be applied to each file in the directory.'
    puts 'Create groups for reference later in the directory structure to be created.'
    parse_regex = @hl.ask('Enter REGEX: ') {|q| q.default = parse_regex}
  end
  Regexp.new(parse_regex)
end
get_path_expression(path_expression) click to toggle source
# File lib/libis/ingester/console/reorg_lib.rb, line 37
def get_path_expression(path_expression)
  unless @unattended && path_expression
    puts
    puts 'Supply the relative or absolute path for each matching file (including file name).'
    puts 'Use $x for referencing the value of the the x-th group in the regex. "file_name" refers to the original file name.'
    path_expression = @hl.ask('Enter path expression: ') {|q| q.default = path_expression}
  end
  path_expression
end
get_processes() click to toggle source
# File lib/libis/ingester/console/include.rb, line 144
def get_processes
  processes = []
  ::Sidekiq::ProcessSet.new.each {|process| processes.push(process)}
  processes
end
get_report_file(report_file) click to toggle source
# File lib/libis/ingester/console/reorg_lib.rb, line 47
def get_report_file(report_file)
  if !@unattended || report_file.nil?
    puts
    puts 'Enter a file name for the report. Extension (csv/tsv/xml/yml) specifies the type. Type "-none-" for no report.'
    report_file = @hl.ask('Report file name: ') {|q| q.default = report_file}
  end
  report_file = nil if !report_file || report_file.empty? || report_file == '-none-'
  report_file
end
get_sidekiq() click to toggle source
# File lib/libis/ingester/console/include.rb, line 87
def get_sidekiq
  @initializer = ::Libis::Ingester::Initializer.instance
  @initializer.configure(@options[:config] || 'site.config.yml')
  @initializer.sidekiq
end
item_array_menu(item) click to toggle source
# File lib/libis/ingester/console/status_lib.rb, line 77
def item_array_menu(item)
  return true if item.blank?
  puts 'Selected:'
  item.each { |run| puts "- #{run.name} - #{run.status_label}" }
  menu = {}
  menu['-'] = Proc.new do
    if @hl.agree('Destroy all selected runs?', false)
      item.each { |i| delete_run(i, true) }
    end
    true
  end
  menu['retry'] = Proc.new do
    queue = select_defined_queue
    item.each do |run|
      Libis::Ingester::RunWorker.push_retry_job(run.id.to_s, queue.name) if run.is_a?(Libis::Ingester::Run)
    end if queue
    true
  end
  selection_menu('action', [], hidden: menu, header: '', prompt: '', layout: :one_line)
end
item_info(item) click to toggle source
# File lib/libis/ingester/console/include.rb, line 338
def item_info(item)
  ap item
end
item_menu(item) click to toggle source
# File lib/libis/ingester/console/status_lib.rb, line 98
def item_menu(item)
  loop do
    item_status(item)
    menu = {'.' => Proc.new { item }}
    menu['?'] = Proc.new { item_info(item); item }
    menu['+'] = Proc.new { select_item(item) } if item.items.count > 0
    menu['-'] = Proc.new { delete_run(item); nil } if item.is_a?(Libis::Ingester::Run)
    menu['-'] = Proc.new { r = item.parent; delete_item(item); r } unless item.is_a?(Libis::Ingester::Run)
    menu['log'] = Proc.new do
      item = item.is_a?(Libis::Ingester::Run) ? item : item.get_run
      # noinspection RubyResolve
      pid = Process.spawn 'less', item.log_filename
      wait_for(pid)
      item
    end
    menu['errors'] = Proc.new do
      item = item.is_a?(Libis::Ingester::Run) ? item : item.get_run
      # noinspection RubyResolve
      cmd = [
          'GREP_COLORS="ms=01;31:mc=01;31:sl=02;31:cx=:fn=35:ln=32:bn=32:se=36"',
          'grep', '--color=always', '-Pn', '-B 2', '-A 4',
          '"( ERROR | WARN | FATAL)"',
          "\"#{item.log_filename}\"",
          '|',
          'less', '-R'
      ].join(' ')
      puts cmd
      pid = Process.spawn cmd
      wait_for(pid)
      item
    end
    menu['retry'] = Proc.new do
      item = item.is_a?(Libis::Ingester::Run) ? item : item.get_run
      queue = select_defined_queue
      Libis::Ingester::RunWorker.push_retry_job(item.id.to_s, queue.name) if queue
      item
    end
    # menu['again'] = Proc.new do
    #   item = item.is_a?(Libis::Ingester::Run) ? item : item.get_run
    #   queue = select_defined_queue
    #   Libis::Ingester::RunWorker.push_restart_job(item.id.to_s, queue.name) if queue
    #   item
    # end
    item = selection_menu('action', [], hidden: menu, header: '', prompt: '', layout: :one_line) || item.parent
    break unless item
  end
  true
end
item_status(item) click to toggle source
# File lib/libis/ingester/console/status_lib.rb, line 7
def item_status(item)
  puts "Status overview for [#{item.class.name.split('::').last}] '#{item.name}':"
  format_str = '%-30s %-12s %-10s %-20s %-20s %10s %10s'
  puts format_str % %w'Task Progress Status Started Updated Elapsed Running'
  puts '-' * 118
  item.reload.status_log.inject({}) do |hash, status|
    task = status['task'].gsub(/[^\/]*\//, '')
    data = {
        level: status['task'].count("/"),
        status: status['status'].to_s.capitalize,
        start: status['created'].localtime,
        end: status['updated'].localtime
    }
    data[:level] += 1 unless task == 'Run'
    if status['progress']
      data[:progress] = status['progress'].to_s
      data[:progress] += ' of ' + status['max'].to_s if status['max']
    end
    data[:running] = time_diff(data[:start], data[:end]).in_seconds
    if hash[task]
      data[:running] += hash[task][:running]
      data.delete(:start)
    end
    (hash[task] ||= {}).merge! data
    hash
  end.each do |task, data|
    data_array = [
        '- ' * data[:level] + task,
        data[:progress].to_s,
        data[:status],
        data[:start].strftime('%d/%m/%Y %T'),
        data[:end].strftime('%d/%m/%Y %T'),
        time_diff_in_hours(data[:start], data[:end]),
        time_diff_in_hours(Time.new(0), Time.new(0) + data[:running])
    ]
    puts format_str % data_array
  end
end
list_threads(process) click to toggle source
# File lib/libis/ingester/console/process_lib.rb, line 20
def list_threads(process)
  format = '%-15s %-15s %-25s %s'
  puts format % %w(thread queue started payload)
  Sidekiq::Workers.new.each do |process_id, thread_id, work|
    next unless process_id == process['identity']
    puts format % [
        thread_id,
        work['queue'],
        Time.at(work['run_at']).localtime,
        payload_detail(work['payload']['args'])
    ]
  end
end
load_config(config, options = {}) click to toggle source
# File lib/libis/ingester/console/reorg_lib.rb, line 160
def load_config(config, options = {})
  x = read_config(config)
  options[:base_dir] = x[:dir]
  options[:parse_regex] = x[:regex]
  options[:path_expression] = x[:expr]
  options[:report_file] = x[:report]
  options[:file_operation] = if x.has_key?(:move)
                               x[:move] ? :move : :copy
                             else
                               x[:operation]
                             end
  options[:interactive] = x[:interactive]
  options[:overwrite] = x[:overwrite]
  options
end
open_report(report_file) click to toggle source
# File lib/libis/ingester/console/reorg_lib.rb, line 66
def open_report(report_file)
  if report_file
    # noinspection RubyStringKeysInHashInspection
    @report_type = {'.csv' => :csv, '.tsv' => :tsv, '.xml' => :xml, '.yml' => :yml}[File.extname(report_file)]
    unless @report_type
      puts "Unknown file type: #{File.extname(report_file)}"
      exit
    end
    @report = File.open(report_file, 'w+')
  end
end
paging_size() click to toggle source
# File lib/libis/ingester/console/include.rb, line 17
def paging_size
  (TTY::Screen.rows - 3) / 5 * 5
end
payload_detail(payload_args) click to toggle source
# File lib/libis/ingester/console/process_lib.rb, line 34
def payload_detail(payload_args)
  payload_args.map do |arg|
    next arg.to_s unless arg.is_a?(String)
    run = Libis::Ingester::Run.find_by(id: arg)
    next run.name if run
    arg
  end.join(', ')
end
process_menu() click to toggle source
# File lib/libis/ingester/console/process_lib.rb, line 52
def process_menu
  menu = {}
  action = lambda { |process| list_threads(process); action_menu(process); true }
  loop do
    break unless select_process(nil, hidden: menu, proc: action)
  end
end
puts_config() click to toggle source
# File lib/libis/ingester/console/reorg_lib.rb, line 187
def puts_config
  configurations.each do |c|
    x = read_config(c)
    puts "- #{c}:"
    x.each {|k, v| puts "   - #{k}: #{v}"}
  end
end
queue_menu() click to toggle source
# File lib/libis/ingester/console/queue_lib.rb, line 4
def queue_menu
  loop do
    break unless (queue = select_defined_queue(with_create: true, with_delete: true))
    loop do
      break unless (worker = select_worker(queue, true))
      if worker.is_a?(Array)
        puts 'Selected:'
        worker.each { |w| puts worker_detail(w) }
        worker.each do |w|
          w.delete
        end if @hl.agree('Delete all selected workers from the queue?', false)
      else
        worker.delete if @hl.agree("About to delete '#{worker_name(worker)}' from the queue. OK?", false)
      end
    end
  end
end
quiet_sidekiq(process) click to toggle source
# File lib/libis/ingester/console/process_lib.rb, line 6
def quiet_sidekiq(process)
  return unless (pid = sidekiq_pid(process))
  puts "Halting Sidekiq #{process['tag']} process #{pid}."
  process.quiet!
  loop do
    get_processes.each do |p|
      if p['pid'] == pid
        return if p.stopping?
      end
    end
    sleep(0.2)
  end
end
read_config(config) click to toggle source
# File lib/libis/ingester/console/reorg_lib.rb, line 176
def read_config(config)
  result = {}
  File.open(config_file(config), 'r') do |f|
    f.readlines.each do |l|
      v = l.strip.split(/: /)
      result[v.first.to_sym] = v.last if v.size > 1
    end
  end rescue nil
  result
end
reset_database() click to toggle source
# File lib/libis/ingester/console/delete_lib.rb, line 101
def reset_database
  if @hl.agree('This will reset the complete database to its initial state. OK?', false)
    puts 'Clearing database ...'
    @initializer.database.clear
  end

  puts 'Seeding database ...'
  @initializer.seed_database

  puts 'Done.'
end
retry_menu() click to toggle source
# File lib/libis/ingester/console/retry_lib.rb, line 4
def retry_menu
  loop do
    @options[:job] = nil
    break unless select_job
    loop do
      @options[:run] = nil
      break unless select_run

      queue = select_defined_queue
      next unless queue

      Libis::Ingester::RunWorker.push_retry_job(@options[:run].id.to_s, queue.name)

      puts "Retrying Run #{@options[:run].name} ..."
    end
  end
end
save_config(options, config) click to toggle source
# File lib/libis/ingester/console/reorg_lib.rb, line 148
def save_config(options, config)
  File.open(config_file(config), 'w') do |f|
    f.puts "dir: #{options[:base_dir]}"
    f.puts "regex: #{options[:parse_regex]}"
    f.puts "expr: #{options[:path_expression]}"
    f.puts "report: #{options[:report_file]}"
    f.puts "operation: #{options[:file_operation]}"
    f.puts "interactive: #{options[:interactive]}"
    f.puts "overwrite: #{options[:overwrite]}"
  end
end
select_active_queue(options = {}) click to toggle source
# File lib/libis/ingester/console/include.rb, line 168
def select_active_queue(options = {})
  queues = Set.new
  get_processes.each do |process|
    next if process.stopping?
    process['queues'].each {|queue| queues.add(queue)}
  end
  queuelist = queues.each_with_object([]) {|q, l| l << Sidekiq::Queue.new(q)}
  select_queue(queuelist, options)
end
select_bulk_option(options) click to toggle source
# File lib/libis/ingester/console/include.rb, line 278
def select_bulk_option(options)
  return nil if options.empty?
  option = selection_menu('Bulk parameter', options) {|opt| "#{opt.first} : #{opt.last}"}
  return nil unless option
  maxlevel = @hl.ask('Number of subdir levels to process', Integer) {|q| q.default = 1}

  dirs = `find #{option.last} -mindepth #{maxlevel} -maxdepth #{maxlevel} -type d -print`.split("\n")
  {key: option.first, values: dirs}
end
select_defined_queue(options = {}) click to toggle source
# File lib/libis/ingester/console/include.rb, line 178
def select_defined_queue(options = {})
  select_queue(Sidekiq::Queue.all, options)
end
select_item(item) click to toggle source
# File lib/libis/ingester/console/include.rb, line 288
def select_item(item)
  items = item.get_items
  paging = paging_size
  @item_page ||= Hash.new(0)
  return item if items.size == 0
  if items.count > paging
    paged_items = items.limit(paging)
    no_pages = (items.count - 1) / paging + 1
    max_page = no_pages - 1
    min_page = 0
    page = @item_page[item.name]
    options = {header: "Subitems of #{item.name}"}
    loop do
      page = [page, min_page].max
      page = [page, max_page].min
      options[:hidden] = {}
      options[:hidden]['previous'] = Proc.new {:previous} if page > min_page
      options[:hidden]['next'] = Proc.new {:next} if page < max_page
      options[:hidden]['goto'] = Proc.new {:goto}
      result = selection_menu(
          "item (#{page * paging + 1}-#{(page + 1) * paging}) / #{items.count} page #{page + 1}/#{no_pages}",
          paged_items.offset(page * paging),
          options
      ) {|i|
        "#{i.class.name.split('::').last}: '#{i.name}' (#{i.items.count} items) [#{i.status_label}]"
      } || item
      case result
      when :previous
        page -= 1 if page > min_page
      when :next
        page += 1 if page < max_page
      when :goto
        page = @hl.ask("Enter page number (#{min_page + 1}..#{no_pages})", Integer) do |q|
          q.in = Range.new(min_page + 1, no_pages)
        end - 1
      else
        @item_page[item.name] = page
        return result
      end
    end
  else
    selection_menu('item', item.get_items, header: "Subitems of #{item.name}") {|i|
      "#{i.class.name.split('::').last}: '#{i.name}' (#{i.items.count} items) [#{i.status_label}]"
    } || item
  end
end
select_job() click to toggle source
# File lib/libis/ingester/console/include.rb, line 123
def select_job

  return false unless select_organization

  # noinspection RubyResolve
  db_menu('Job', @options[:organization].jobs, parent: @options[:organization].name) do |job|
    "#{job.description} (#{job.runs.count} runs)"
  end
end
select_options(job) click to toggle source
# File lib/libis/ingester/console/include.rb, line 244
def select_options(job)
  options = {}
  job.workflow.config['input'].each do |key, value|
    options[key] = value['default']
  end if job.workflow.config['input']
  job.input.each do |key, value|
    options[key] = value
  end if job.input

  set_option = Proc.new {|opt|
    key, value = opt
    if key =~ /(location|dir)/
      dir = select_path(true, false, value)
      options[key] = File.absolute_path(dir) unless dir.nil? || dir.empty? || !File.directory?(dir) || !File.exist?(dir)
    elsif key =~ /file$/
      file = select_path(true, true, value)
      options[key] = File.absolute_path(file) unless file.nil? || file.empty? || !File.file?(file) || !File.exist?(file)
    else
      options[key] = value ? @hl.ask("#{key} : ") {|q| q.default = value} : @hl.ask("#{key} : ")
    end
    true
  }

  loop do
    option = selection_menu('Parameters', options, parent: job.name, proc: set_option) {|opt|
      "#{opt.first} : #{opt.last}"
    }
    break unless option
  end unless options.empty?

  options

end
select_organization() click to toggle source

noinspection RubyResolve

# File lib/libis/ingester/console/include.rb, line 113
def select_organization

  # return false unless select_user

  # db_menu('Organization', @options[:user].organizations, parent: @options[:user].name) do |org|
  db_menu('Organization', Libis::Ingester::Organization.all) do |org|
    "#{org.name} (#{org.jobs.count} jobs)"
  end
end
select_path(dir = true, file = true, base_dir) click to toggle source
# File lib/libis/ingester/console/menu.rb, line 76
def select_path(dir = true, file = true, base_dir)
  base_dir ||= '.'
  base_dir = File.absolute_path(File.join(base_dir, '..')) until File.exists?(base_dir) && File.directory?(base_dir)
  old_completer = Readline.completion_proc
  old_append_character = Readline.completion_append_character
  puts 'Enter path. <TAB> to complete. Double <TAB> to see list.'
  prompt = "#{base_dir} > "
  Readline.completion_append_character = ''
  Readline.completion_proc = Proc.new do |str|
    str = File.join(base_dir, str) unless str =~ /^\//
    Dir[str + '*']
        .reject {|d| d =~ /\.\.?$/}
        .reject {|d| !file && File.file?(d)}
        .reject {|d| !dir && File.directory?(d)}
        .map do |d|
      d = File.directory?(d) ? d + '/' : d
      d.gsub(/^#{Regexp.escape(base_dir)}\/?/, '')
    end
  end
  str = Readline.readline(prompt, true)
  str = File.join(base_dir, str) unless str =~ /^\//
  str
ensure
  Readline.completion_proc = old_completer
  Readline.completion_append_character = old_append_character
end
select_process(processes = nil, options = {}) click to toggle source
# File lib/libis/ingester/console/include.rb, line 150
def select_process(processes = nil, options = {})
  processes ||= get_processes
  format = '%-40s %-30s %s'
  xformat = '   ' + format
  xformat = ' ' + xformat if processes.count > 9
  header = xformat % %w(Process Threads Queues)
  selection_menu('Process', processes, options.merge(header: header)) do |process|
    name = '%s [%s]' % [process['tag'], process['hostname']]
    workers = '(%d of %d busy)' % [process['busy'], process['concurrency']]
    workers += ' **HALTED**' if process['quiet'] == 'true'
    format % [
        '%.39s %s' % [name, '.' * [0, 39 - name.size].max],
        '%.29s %s' % [workers, '.' * [0, 29 - workers.size].max],
        process['queues'].join(', ')
    ]
  end
end
select_queue(queue_list, options = {}) click to toggle source
# File lib/libis/ingester/console/include.rb, line 182
def select_queue(queue_list, options = {})
  process_map = {}
  get_processes.each do |process|
    process['queues'].each do |queue|
      process_map[queue] ||= []
      process_map[queue] << process['tag']
    end
  end

  menu = {}
  menu['+'] = Proc.new {
    name = @hl.ask('queue name:') {|q| q.validate = /\A[a-z][a-z0-9_]*\Z/}
    Sidekiq::Client.new.redis_pool.with do |conn|
      conn.multi do
        conn.sadd('queues'.freeze, name)
      end
    end
    Sidekiq::Queue.new(name)
    true
  } if options[:with_create]
  menu['-'] = Proc.new do
    queue = select_defined_queue
    queue.clear if queue.is_a?(Sidekiq::Queue)
    true
  end if options[:with_delete]

  format = '%-30s %-20s %s'
  xformat = '   ' + format
  xformat = ' ' + xformat if queue_list.count > 9
  header = xformat % %w(Name Waiting Processes)
  selection_menu('Queue', queue_list, hidden: menu, header: header) do |queue|
    format % [
        '%.29s %s' % [queue.name, '.' * [0, 29 - queue.name.size].max],
        '%d %s' % [queue.size, '.' * [0, 19 - queue.size.to_s.size].max],
        (process_map[queue.name] || []).join(' ')
    ]
  end
end
select_run(options = {}) click to toggle source
# File lib/libis/ingester/console/include.rb, line 133
def select_run(options = {})
  return unless select_job

  options.merge!(parent: @options[:job].description) {|_k, _v1, _v2| _v1}

  # noinspection RubyResolve
  db_menu('Run', @options[:job].runs, options) {
      |run| "#{run.name} - #{run.status_label}"
  }
end
select_user() click to toggle source

noinspection RubyResolve

# File lib/libis/ingester/console/include.rb, line 98
def select_user

  return false unless db_menu('User', Libis::Ingester::User.all) do |user|
    "#{user.name} (#{user.organizations.count} organizations)"
  end

  loop do
    @options[:password] = @hl.ask('Password: ') {|q| q.echo = '.'} unless @options[:password]
    return true if @options[:user].authenticate(@options[:password])
    @options[:password] = nil
  end

end
select_worker(queue = nil, multiselect = false) click to toggle source
# File lib/libis/ingester/console/include.rb, line 221
def select_worker(queue = nil, multiselect = false)
  queue ||= select_defined_queue
  return unless queue.is_a?(Sidekiq::Queue)
  workers = []
  queue.each {|worker| workers << worker}
  selection_menu('Run', workers, multiselect: multiselect) do |worker|
    worker_detail(worker)
  end
end
selection_menu(title, items, options = {}) { |item| ... } click to toggle source
# File lib/libis/ingester/console/menu.rb, line 26
def selection_menu(title, items, options = {})
  (options[:hidden] ||= {}).merge!('' => Proc.new {nil})
  keys = options[:hidden].keys.map {|key| key == '' ? '<return>' : key}
  keys << '*' if options[:multiselect]
  prompt = "#{options[:prompt] || "Select #{title}"} (#{keys.join('/')})"
  @hl.choose do |menu|
    menu.index = options[:index] if options[:index]
    menu.prompt = prompt
    menu.header = options[:header] || "\n#{title}#{options[:parent] ? ' for ' + options[:parent] : ''}"
    menu.select_by = :index_or_name
    menu.layout = options[:layout] if options[:layout]
    (options[:prepend] || {}).each {|label, proc| menu.choice(label) {proc.call(label)}}
    items.each do |item|
      menu.choice(block_given? ? yield(item) : item) {options[:proc] ? options[:proc].call(item) : item}
    end
    (options[:append] || {}).each {|label, proc| menu.choice(label) {proc.call(label)}}
    (options[:hidden] || {}).each {|label, proc| menu.hidden(label) {proc.call(label)}}
    if options[:multiselect]
      menu.hidden('*') {
        answer = @hl.ask('Enter a list of numbers and/or ranges: ')
        return true if answer.blank?
        result = Set.new
        answer.split(/\s*[,;\s]\s*/).each do |entry|
          case entry
          when /^\d+\.\.\d+$/
            begin
              range = entry.split('..').map {|d| Integer(d)}
              items[(range[0] - 1)..(range[1] - 1)].each do |item|
                result << item
              end
            rescue => e
              puts "Error - problem interpreting range '#{entry}': #{e.message}"
            end
          when /^\d+$/
            begin
              result << items[entry.to_i - 1]
            rescue => e
              puts "Error - problem interpreting number '#{entry}': #{e.message}"
            end
          else
            puts "Error - malformed entry: '#{entry}'"
          end
        end
        result.reject {|v| v.blank?}
        result.map {|item| options[:proc] ? options[:proc].call(item) : item}
      }
    end
  end
end
status_menu() click to toggle source
# File lib/libis/ingester/console/status_lib.rb, line 46
def status_menu
  loop do
    break unless select_organization
    loop do
      break unless select_job
      loop do
        @options[:job].reload_relations
        opts = {
            'finished delete' => Proc.new { delete_all_finshed_runs(@options[:job]); true },
            'failed delete' => Proc.new { delete_all_failed_runs(@options[:job]); true },
            'all delete' => Proc.new { delete_all_runs(@options[:job]); true },
            '.' => Proc.new { true }
        }
        break unless (item = select_run(multiselect: true, hidden: opts))
        item = case item
                 when Array
                   item_array_menu(item)
                 when Libis::Ingester::Run
                   item_menu(item)
                 else
                   item
               end
        break unless item
        @options[:run] = nil
      end
      @options[:job] = nil
    end
    @options[:organization] = nil
  end
end
submit_menu() click to toggle source
# File lib/libis/ingester/console/submit_lib.rb, line 4
def submit_menu
  loop do
    @options[:job] = nil
    return unless select_job
    queue = select_defined_queue
    next unless queue

    job = @options[:job]
    options = [job.id.to_s]

    options[1] = select_options(job)

    run_name = @hl.ask('Specific run name: ')

    bulk = select_bulk_option(options[1])

    options[1]['run_name'] = clean_string run_name
    if @operator_email && !@operator_email.empty?
      # noinspection RubyStringKeysInHashInspection
      options[1]['run_config'] = {
          'error_to' => @operator_email,
          'success_to' => @operator_email
      }
      puts "Status reports will be sent to: #{@operator_email}"
    end

    if bulk
      key = bulk[:key]
      next unless @hl.agree("Ready to submit #{bulk[:values].count} runs for job #{@options[:job].name}. OK?", false)

      base_dir = options[1][key]
      base_name = options[1]['run_name']
      bulk[:values].each do |value|
        options[1]['run_name'] = clean_string "#{base_name}/#{clean_string(value.gsub(/^#{base_dir}/, ''))}"
        options[1][key] = value
        Sidekiq::Client.push(
            'class' => 'Libis::Ingester::JobWorker',
            'queue' => queue.name,
            'retry' => false,
            'args' => options
        )
        puts "Job #{@options[:job].name} submitted for #{key} = #{value}"
      end
    else
      next unless @hl.agree("Ready to submit run for job #{@options[:job].name} with #{options[1]}. OK?", false)
      Sidekiq::Client.push(
          'class' => 'Libis::Ingester::JobWorker',
          'queue' => queue.name,
          'retry' => false,
          'args' => options
      )
      puts "Run for job #{@options[:job].name} submitted with #{options[1]}."
    end
  end
end
time_diff(start_time, end_time) click to toggle source
# File lib/libis/ingester/console/include.rb, line 344
def time_diff(start_time, end_time)
  TimeDifference.between(start_time, end_time)
end
time_diff_human(start_time, end_time) click to toggle source
# File lib/libis/ingester/console/include.rb, line 348
def time_diff_human(start_time, end_time)
  diff_parts = []
  time_diff(start_time, end_time).in_general.each do |part, quantity|
    next if quantity <= 0
    part = part.to_s.humanize
    part = part.singularize if quantity <= 1
    diff_parts << "#{quantity} #{part}"
  end

  last_part = diff_parts.pop
  return last_part if diff_parts.empty?

  [diff_parts.join(', '), last_part].join(' and ')

end
time_diff_in_hours(start_time, end_time) click to toggle source
# File lib/libis/ingester/console/include.rb, line 364
def time_diff_in_hours(start_time, end_time)
  seconds = time_diff(start_time, end_time).in_seconds.round
  minutes = seconds / 60
  seconds = seconds % 60
  hours = minutes / 60
  minutes = minutes % 60
  "#{'%4d' % hours}:#{'%02d' % minutes}:#{'%02d' % seconds}"
end
wait_for(pid) click to toggle source
# File lib/libis/ingester/console/status_lib.rb, line 147
def wait_for(pid)
  Process.wait pid
rescue Interrupt
  wait_for pid
end
worker_detail(worker) click to toggle source
# File lib/libis/ingester/console/include.rb, line 235
def worker_detail(worker)
  result = worker_name(worker)
  parameters = worker.args.last.map {|p| "\t\t#{p.first} = #{p.last}"}
  if parameters.size > 0
    result += "\n" + parameters.join("\n")
  end
  result
end
worker_name(worker) click to toggle source
# File lib/libis/ingester/console/include.rb, line 231
def worker_name(worker)
  "#{worker.enqueued_at.localtime} : #{worker.klass.constantize.subject(worker.args.first).name rescue nil}"
end
write_report(old_name, new_folder, new_name, remark = nil) click to toggle source
# File lib/libis/ingester/console/reorg_lib.rb, line 94
def write_report(old_name, new_folder, new_name, remark = nil)
  return unless @report
  case @report_type
    when :tsv
      @report.puts "old_name\tnew_folder\tnew_name\tremark" if @report.size == 0
      @report.puts "#{for_tsv(old_name)}\t#{for_tsv(new_folder)}" +
                       "\t#{for_tsv(new_name)}\t#{for_tsv(remark)}"
    when :csv
      @report.puts 'old_name,new_folder,new_name' if @report.size == 0
      @report.puts "#{for_csv(old_name)},#{for_csv(new_folder)}" +
                       ",#{for_csv(new_name)},#{for_csv(remark)}"
    when :xml
      @report.puts '<?xml version="1.0" encoding="UTF-8"?>' if @report.size == 0
      @report.puts '<report>' if @report.size == 1
      @report.puts '  <file>'
      @report.puts "    <old_name>#{for_xml(old_name, :text)}</old_name>"
      @report.puts "    <new_folder>#{for_xml(new_folder, :text)}</new_folder>"
      @report.puts "    <new_name>#{for_xml(new_name, :text)}</new_name>"
      @report.puts "    <remark>#{for_xml(remark, :text)}</remark>" if remark
      @report.puts '  </file>'
    when :yml
      @report.puts '# Reorganisation report' if @report.size == 0
      @report.puts "- old_name: #{for_yml(old_name)}" +
                       "\n  new_folder: #{for_yml(new_folder)}" +
                       "\n  new_name: #{for_yml(new_name)}" +
                       (remark ? "\n  remark: #{for_yml(remark)}" : '')
    else
      #nothing
  end
end