class At_email::Tasks::Imap_To_Fs

Constants

METADATA_ATTRIBUTES
REQUESTED_ATTRIBUTES

Public Class Methods

new() click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 15
def initialize
  @dir_base_path = $config[:output_base_dir] + '/' + $config[:account_id]
end

Public Instance Methods

bootstrap_thread(folder_name, uid) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 359
def bootstrap_thread(folder_name, uid)
  message_config = get_message_config(folder_name, uid)
  set_thread_property(folder_name, uid, 'File Name', File.basename(message_config['file_path']))
  set_thread_property(folder_name, uid, 'File Path', message_config['file_path'])
  set_thread_property(folder_name, uid, 'Server Size', message_config['size'])
  set_thread_property(folder_name, uid, 'Retry Count', 0)
end
check_folder_included(folder_name) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 614
def check_folder_included(folder_name)
  ignore_list = []
  ignore_list << "[Gmail]"
  ignore_list << "[Gmail]/All Mail"
  ignore_list << "[Gmail]/Important"
  ignore_list << "[Gmail]/Starred"
  if ignore_list.include?(folder_name)
    return false
  else
    return true
  end
end
create_output_directory(folder_name) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 482
def create_output_directory(folder_name)
  dir_path = @dir_base_path + '/' + folder_name
  if !Dir.exists?(dir_path)
    $logger.debug folder_name + ' - Creating Directory: ' + dir_path
    FileUtils.mkdir_p dir_path
  end
end
decrease_thread_count(folder_name) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 269
def decrease_thread_count(folder_name)
  @account_metadata[:folders][folder_name]['process']['threads']['count'] -= 1
end
delete_directories(folder_name) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 171
def delete_directories(folder_name)
  server_dirs = @account_metadata[:folders][folder_name]['directories']['server']
  @account_metadata[:folders][folder_name]['directories']['local'].each do |local_dir|
    if !server_dirs[local_dir]
      $logger.info( folder_name + ' - Deleting Directory: ' + local_dir )
    end
  end
end
delete_files(folder_name) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 205
def delete_files(folder_name)
  @account_metadata[:folders][folder_name]['files']['deletes'].each do |file_path|
    if File.exists?(file_path)
      $logger.info( folder_name + ' -  Deleting File: ' + file_path )
      File.delete(file_path)
    end
  end
end
execute() click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 627
def execute
  @connection = At_email::Account::ImapConnection.new()
  @connection.login
  get_imap
  generate_metadata_for_account
  @account_metadata[:task][:start_time] = Time.now
  get_data_for_all_folders
  @connection.disconnect
  @account_metadata[:task][:end_time] = Time.now
  task_report
end
format_file_size(file_size) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 746
def format_file_size(file_size)
  output = format_number(file_size) + ' bytes'
  return output
end
format_number(number) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 751
def format_number(number)
  output = number.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
  return output
end
generate_metadata_for_account() click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 82
def generate_metadata_for_account()
  @account_metadata = {}
  @account_metadata[:account] = {}
  @account_metadata[:account]['directories'] = {}
  @account_metadata[:account]['directories']['server'] = get_server_dir_list()
  @account_metadata[:account]['directories']['local'] = get_local_dir_list()
  @account_metadata[:account]['messages'] = {}
  @account_metadata[:account]['messages']['server'] = {}
  @account_metadata[:account]['messages']['server']['folders'] = get_server_message_folder_list()
  @account_metadata[:task] = {}
  @account_metadata[:task] = {}
  @account_metadata[:folders] = {}
end
generate_metadata_for_folder(folder_name) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 96
def generate_metadata_for_folder(folder_name)
  @account_metadata[:folders][folder_name] = {}
  @account_metadata[:folders][folder_name]['messages'] = {}
  @account_metadata[:folders][folder_name]['messages']['server'] = {}
  @account_metadata[:folders][folder_name]['messages']['server']['uids'] = get_uids()
  @account_metadata[:folders][folder_name]['messages']['server']['metadata'] = get_imap_metadata(@account_metadata[:folders][folder_name]['messages']['server']['uids'])
  @account_metadata[:folders][folder_name]['messages']['server']['config'] = update_message_config(folder_name, @account_metadata[:folders][folder_name]['messages']['server']['metadata'])
  @account_metadata[:folders][folder_name]['files'] = {}
  @account_metadata[:folders][folder_name]['files']['server'] = get_server_file_list(folder_name, @account_metadata[:folders][folder_name]['messages']['server']['config'])
  @account_metadata[:folders][folder_name]['files']['local'] = get_local_file_list(folder_name)
  @account_metadata[:folders][folder_name]['files']['deletes'] = get_delete_list(folder_name)
  @account_metadata[:folders][folder_name]['process'] = {}
end
get_data_for_all_folders() click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 595
def get_data_for_all_folders
  folder_list = @account_metadata[:account]['messages']['server']['folders']
  $logger.debug ''
  $logger.debug 'Folder Count: ' + folder_list.count.to_s
  folder_list.each do |folder_name|
    if check_folder_included(folder_name)
      $logger.debug ' ' + folder_name
    else
      $logger.debug ' ' + folder_name + ' - SKIPPED'
    end
  end
  folder_list.each do |folder_name|
    if check_folder_included(folder_name)
      write_data_for_folder(folder_name)
    end
  end
  $logger.debug ''
end
get_delete_list(folder_name) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 192
def get_delete_list(folder_name)
  output = []
  server_files = @account_metadata[:folders][folder_name]['files']['server']
  @account_metadata[:folders][folder_name]['files']['local'].each do |file_path|
    if !server_files[file_path]
      if File.exists?(file_path)
        output << file_path
      end
    end
  end
  return output
end
get_download_list(folder_name) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 180
def get_download_list(folder_name)
  output = []
  local_file_list = @account_metadata[:folders][folder_name]['files']['local']
  @account_metadata[:folders][folder_name]['messages']['server']['config'].each do |uid, message_config|
    file_path = message_config['file_path']
    if !local_file_list.include?(file_path)
      output << uid
    end
  end
  return output
end
get_duration(start_time, end_time) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 723
def get_duration(start_time, end_time)
  output = end_time.to_f - start_time.to_f
  return output
end
get_duration_formatted(start_time, end_time) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 728
def get_duration_formatted(start_time, end_time)
  duration_seconds = get_duration(start_time, end_time)
  if duration_seconds
    if duration_seconds <= 60
      output = sprintf("%.6f", duration_seconds) + ' Sec(s)'
    else
      nanoseconds = (sprintf("%.6f", duration_seconds).to_f * 1000000) % 1000000
      seconds = duration_seconds % 60
      minutes = (duration_seconds / 60) % 60
      hours = duration_seconds / (60 * 60)
      output = format("%02d:%02d:%02d.%6d", hours, minutes, seconds, nanoseconds)
    end
  else
    return false
  end
  return output
end
get_file_path_for_message(folder_name, message_config) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 72
def get_file_path_for_message(folder_name, message_config)
  output = get_file_path_prefix(folder_name, message_config)+'.'+$config[:storage_file_extension]
  return output
end
get_file_path_prefix(folder_name, message_config) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 54
def get_file_path_prefix(folder_name, message_config)
  if $config[:storage_file_name_uid_padding]
    if $config[:storage_file_name_uid_padding_length] <= 1
      storage_file_name_uid_padding = 2
    else
      storage_file_name_uid_padding = $config[:storage_file_name_uid_padding_length]
    end
    uid_based_prefix = message_config['uid'] + (10 ** (storage_file_name_uid_padding - 1))
  else
    uid_based_prefix = message_config['uid']
  end
  hash_length = $config[:storage_file_name_hash_length]
  dir_path = @dir_base_path + '/' + folder_name
  file_name_hash = Digest::MD5.hexdigest(Digest::MD5.hexdigest(YAML.dump(message_config['envelope']))+'-'+Digest::MD5.hexdigest(YAML.dump(message_config['flags'])))[0...hash_length]
  output = dir_path+'/'+uid_based_prefix.to_s+'-'+file_name_hash
  return output
end
get_formatted_thread_property(folder_name, uid, property_name) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 248
def get_formatted_thread_property(folder_name, uid, property_name)
  ### getting property
  name = property_name
  value = get_thread_property(folder_name, uid, name) || 'Unable to get property: ' + name
  #####
  output = name + ': ' + value.to_s
  return output
end
get_imap() click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 19
def get_imap
  @imap = @connection.imap
end
get_imap_metadata(uid_list) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 34
def get_imap_metadata(uid_list)
  output = {}
  if uid_list.count > 0
    output = @imap.uid_fetch(uid_list, METADATA_ATTRIBUTES)
  end
  return output
end
get_local_dir_list() click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 148
def get_local_dir_list()
  output = []
  if Dir.exists?(@dir_base_path)
    Dir.chdir(@dir_base_path)
    file_list = Dir.glob("**/*/")
    file_list.each do |object_name|
      if File.directory?(object_name)
        output << @dir_base_path + '/' + object_name
      end
    end
  end
  return output
end
get_local_file_list(folder_name) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 124
def get_local_file_list(folder_name)
  output = []
  dir_path = @dir_base_path + '/' + folder_name
  if Dir.exists?(dir_path)
    Dir.chdir(dir_path)
    file_list = Dir.glob("*."+$config[:storage_file_extension]).sort_by{|f| f.split(" ")[0].to_i }
    file_list.each do |file_name|
      if File.file?(file_name)
        output << dir_path+'/'+file_name
      end
    end
  end
  return output
end
get_message_config(folder_name, uid) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 350
def get_message_config(folder_name, uid)
  if @account_metadata[:folders][folder_name]['messages']['server']['config'][uid]
    output = @account_metadata[:folders][folder_name]['messages']['server']['config'][uid]
    return output
  else
    raise '***** ERROR ***** - Message Config: Cannot get message config for folder: ' + folder_name + ' - UID:' + uid
  end
end
get_message_data(uid) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 42
def get_message_data(uid)
  data_raw = @imap.uid_fetch(uid.to_i, REQUESTED_ATTRIBUTES)
  if data_raw
    if data_raw[0]
      if data_raw[0].attr
        output = data_raw[0].attr
        return output
      end
    end
  end
end
get_ratio_formatted(a, b, delimiter = '/') click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 214
def get_ratio_formatted(a, b, delimiter = '/')
  output = a.to_s + delimiter + b.to_s
  return output
end
get_server_dir_list() click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 162
def get_server_dir_list()
  output = []
  message_folder_list = get_server_message_folder_list()
  message_folder_list.each do |message_folder|
    output << @dir_base_path + '/' + message_folder
  end
  return output
end
get_server_file_list(folder_name, server_message_config) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 139
def get_server_file_list(folder_name, server_message_config)
  output = {}
  server_message_config.each do |uid, message_config|
    file_path = get_file_path_for_message(folder_name, message_config)
    output[file_path] = uid
  end
  return output
end
get_server_message_folder_list() click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 23
def get_server_message_folder_list()
  output = []
  root_path = @imap.list("", "")[0].name
  folder_data = @imap.list(root_path, "*")
  output = []
  folder_data.map do |this_folder_data|
    output << this_folder_data.name
  end
  return output
end
get_task_stats() click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 690
def get_task_stats()
  output = {}
  output[:account] = {}
  output[:folders] = {}
  @account_metadata[:folders].each do |folder_name, folder_data|
    output[:folders][folder_name] = {}
    output[:folders][folder_name]['messages'] = {}
    output[:folders][folder_name]['messages']['size'] = 0
    folder_data['messages']['server']['config'].each do |uid, message_config|
      output[:folders][folder_name]['messages']['size'] += message_config['size']
    end
    output[:folders][folder_name]['threads'] = {}
    output[:folders][folder_name]['threads']['Total'] = 0
    output[:folders][folder_name]['threads']['Success'] = 0
    output[:folders][folder_name]['threads']['Failed'] = 0
    if folder_data['process']
      if folder_data['process']['threads']
        if folder_data['process']['threads']['uids']
          folder_data['process']['threads']['uids'].each do |uid, thread_data|
            output[:folders][folder_name]['threads']['Total'] += 1
            if thread_data['State'] === 'Successful'
              output[:folders][folder_name]['threads']['Success'] += 1
            else
              output[:folders][folder_name]['threads']['Failed'] += 1
            end
          end
        end
      end
    end
  end
  return output
end
get_thread_id_string(uid, msg_ratio) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 273
def get_thread_id_string(uid, msg_ratio)
  uid_string = get_uid_string_formatted(uid)
  output = msg_ratio + ' - ' + uid_string
end
get_thread_property(folder_name, uid, property) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 234
def get_thread_property(folder_name, uid, property)
  if @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property]
    output = @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property]
    return output
  else
    return false
  end
end
get_uid_string_formatted(uid) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 219
def get_uid_string_formatted(uid)
  output = 'UID:' + uid.to_s
  return output
end
get_uids() click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 77
def get_uids()
  uid_list = @imap.uid_search(["ALL"]).sort
  return uid_list
end
increase_thread_count(folder_name) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 265
def increase_thread_count(folder_name)
  @account_metadata[:folders][folder_name]['process']['threads']['count'] += 1
end
increase_thread_property(folder_name, uid, property) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 257
def increase_thread_property(folder_name, uid, property)
  @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property] += 1
end
kill_thread(folder_name, uid) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 500
def kill_thread(folder_name, uid)
  log_folder_event(folder_name, 'WARN', 'ERROR', 'Thread Failure - ' + get_uid_string_formatted(uid))
  log_folder_event(folder_name, 'WARN', 'ERROR', ' Killing Thread')
  Thread.kill(get_thread_property(folder_name, uid, 'Thread'))
  ### getting formatted thread property
  property_name = 'Queue Time'
  log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  log_folder_event(folder_name, 'WARN', 'ERROR', ' ' + log_string)
  ### getting formatted thread property
  property_name = 'End Time'
  log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  log_folder_event(folder_name, 'WARN', 'ERROR', ' ' + log_string)
  $logger.error folder_name + ' -  Duration: ' + get_duration_formatted(get_thread_property(folder_name, uid, 'Queue Time'), get_thread_property(folder_name, uid, 'End Time'))
  ### getting formatted thread property
  property_name = 'File Name'
  log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  log_folder_event(folder_name, 'WARN', 'ERROR', ' ' + log_string)
end
log_folder_event(folder_name, event_level, event_tag, event_data) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 345
def log_folder_event(folder_name, event_level, event_tag, event_data)
  log_string = folder_name + ' - ' + event_data
  $logger.event(event_level, event_tag, log_string)
end
log_folder_info(folder_name) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 471
def log_folder_info(folder_name)
  $logger.debug folder_name + ' - Server Files: ' + @account_metadata[:folders][folder_name]['files']['server'].count.to_s
  @account_metadata[:folders][folder_name]['files']['server'].each do |file_path, uid|
    $logger.debug folder_name + ' -  ' + File.basename(file_path)
  end
  $logger.debug folder_name + ' - Local Files: ' + @account_metadata[:folders][folder_name]['files']['local'].count.to_s
  @account_metadata[:folders][folder_name]['files']['local'].each do |file_path|
    $logger.debug folder_name + ' -  ' + File.basename(file_path)
  end
end
log_thread_event(folder_name, uid, msg_ratio, event_level, event_tag='', event_data) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 338
def log_thread_event(folder_name, uid, msg_ratio, event_level, event_tag='', event_data)
  uid_string = get_uid_string_formatted(uid) || 'Unable to get UID string'
  thread_id_string = get_thread_id_string(uid, msg_ratio) || 'Unable to get thread ID string'
  log_string = thread_id_string + ' - ' + event_data
  log_folder_event(folder_name, event_level, event_tag, log_string)
end
log_thread_on_failure(folder_name, uid, msg_ratio, event_data) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 306
def log_thread_on_failure(folder_name, uid, msg_ratio, event_data)
  ### getting property
  property_name = 'Queue Time'
  queue_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  ### getting property
  property_name = 'End Time'
  end_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  ### getting property
  property_name = 'Server Size'
  server_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  ### getting property
  property_name = 'Local Size'
  local_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  ### getting data
  duration_formatted = get_duration_formatted(queue_time, end_time) || 'Unable to get data'
  #####
  ### getting data
  size_formatted = server_size + ' Bytes'
  #####
  log_string = 'Duration: ' + duration_formatted + ' -  Size: ' + size_formatted
  log_thread_event(folder_name, uid, msg_ratio, 'ERROR', 'E', log_string)
  log_thread_event(folder_name, uid, msg_ratio, 'ERROR', 'ERROR', event_data)
end
log_thread_on_retry(folder_name, uid, msg_ratio, event_data) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 334
def log_thread_on_retry(folder_name, uid, msg_ratio, event_data)
  log_thread_event(folder_name, uid, msg_ratio, 'WARN', 'W', event_data)
end
log_thread_on_success(folder_name, uid, msg_ratio) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 278
def log_thread_on_success(folder_name, uid, msg_ratio)
  ### getting property
  property_name = 'Queue Time'
  queue_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  ### getting property
  property_name = 'End Time'
  end_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  ### getting property
  property_name = 'Server Size'
  server_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  ### getting property
  property_name = 'Local Size'
  local_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  #####
  ### getting data
  duration_formatted = get_duration_formatted(queue_time, end_time) || 'Unable to get data'
  #####
  ### getting data
  size_ratio_formatted = get_ratio_formatted(local_size, server_size, ' / ') || 'Unable to get data'
  size_formatted = size_ratio_formatted + ' Bytes'
  #####
  log_string = 'Duration: ' + duration_formatted + ' -  Size: ' + size_formatted
  log_thread_event(folder_name, uid, msg_ratio, 'INFO', '', log_string)
end
log_thread_property(folder_name, uid, msg_ratio, property_name) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 243
def log_thread_property(folder_name, uid, msg_ratio, property_name)
  log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
  log_thread_event(folder_name, uid, msg_ratio, 'INFO', '', log_string)
end
process_deletes(folder_name) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 490
def process_deletes(folder_name)
  if $config[:delete_on_sync]
    if @account_metadata[:folders][folder_name]['files']['deletes'].count > 0
      $logger.info folder_name + ' - Delete Count: ' + @account_metadata[:folders][folder_name]['files']['deletes'].count.to_s
      $logger.info folder_name + ' - Executing Deletes'
      delete_files(folder_name)
    end
  end
end
process_downloads(folder_name, download_list) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 522
def process_downloads(folder_name, download_list)
  process_healthy = true
  if download_list.count > 0
    $logger.info( folder_name + ' - ' + download_list.count.to_s + ' Message(s) to download - Thread Limit: ' + $config[:threads].to_s )
    message_count = 0
    ### Creating / resetting active thread data structure
    @account_metadata[:folders][folder_name]['process']['threads'] = {}
    @account_metadata[:folders][folder_name]['process']['threads']['count'] = 0
    @account_metadata[:folders][folder_name]['process']['threads']['uids'] = {}
    message_total = download_list.count
    download_list.each do |uid|
      if process_healthy
        message_count += 1
        uid_string = get_uid_string_formatted(uid)
        msg_ratio = get_ratio_formatted(message_count, message_total)
        last_queue_time = Time.now
        if @account_metadata[:folders][folder_name]['process']['threads']['count'] >= $config[:threads]
          $logger.debug folder_name + ' - Thread Count: ' + @account_metadata[:folders][folder_name]['process']['threads']['count'].to_s + ' - Waiting...'
        end
        while @account_metadata[:folders][folder_name]['process']['threads']['count'] >= $config[:threads] && process_healthy
          if (Time.now.to_i - last_queue_time.to_i) >= ($config[:thread_timeout] * $config[:timeout_warning_ratio])
            if (Time.now.to_i - last_queue_time.to_i) >= $config[:thread_timeout]
              $logger.error folder_name + ' - *E* Thread queuing has stopped - Timeout: ' + $config[:thread_timeout].to_s + ' Sec(s)'
              process_healthy = false
            else
              $logger.warn folder_name + ' - *W* Thread queuing has stopped - Timeout: ' + (Time.now.to_i - last_queue_time.to_i).to_s + '/' + $config[:thread_timeout].to_s + ' Sec(s)'
              sleep rand(10..20)
            end
          else
            sleep 0.2
          end
        end
        if process_healthy
          @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid] = {}
          queue_thread(folder_name, uid, msg_ratio)
          sleep 0.1
          while get_thread_property(folder_name, uid, 'State') === 'Queued'
            if (Time.now.to_i - get_thread_property(folder_name, uid, 'Queue Time').to_i) >= ($config[:thread_timeout] * $config[:timeout_warning_ratio])
              if (Time.now.to_i - get_thread_property(folder_name, uid, 'Queue Time').to_i) >= $config[:thread_timeout]
                $logger.error folder_name + ' - ' + msg_ratio + ' - ' + uid_string + ' - *E* Thread state - Moving on... - Timeout: ' + $config[:thread_timeout].to_s + ' Sec(s)'
                set_thread_state(folder_name, uid, msg_ratio, 'Failed')
                kill_thread(folder_name, uid)
              else
                $logger.warn folder_name + ' - ' + msg_ratio + ' - ' + uid_string + ' - *W* Thread state - Timeout: ' + (Time.now.to_i - get_thread_property(folder_name, uid, 'Queue Time').to_i).to_s + '/' + $config[:thread_timeout].to_s + ' Sec(s)'
                sleep rand(10..20)
              end
            else
              sleep 0.2
            end
          end
        end
      end
    end
    if process_healthy
      sleep 2
      process_shutdown(folder_name)
    end
    process_thread_data_before_finish(folder_name)
  end
end
process_shutdown(folder_name) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 457
def process_shutdown(folder_name)
  time_started_waiting = Time.now.to_i
  while @account_metadata[:folders][folder_name]['process']['threads']['count'] > 0
    time_since_started_waiting = Time.now.to_i - time_started_waiting
    if time_since_started_waiting >= $config[:thread_timeout]
      $logger.warn folder_name + ' - *E* - Thread pool shutdown - Timeout: ' + $config[:thread_timeout].to_s + ' Sec(s)'
      @account_metadata[:folders][folder_name]['process']['threads']['count'] = 0
    else
      $logger.info folder_name + ' - Waiting for threads - Remaining: ' + @account_metadata[:folders][folder_name]['process']['threads']['count'].to_s + '/' + $config[:threads].to_s + ' - Timeout: ' + time_since_started_waiting.to_s + '/' + $config[:thread_timeout].to_s + ' Sec(s)'
      sleep 10
    end
  end
end
process_thread_data_before_finish(folder_name) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 441
def process_thread_data_before_finish(folder_name)
  has_failed = false
  @account_metadata[:folders][folder_name]['process']['threads']['uids'].each do |uid, thread_data|
    if thread_data['State'] != 'Successful'
      thread_data['State'] = 'Failed'
    end
    if thread_data['State'] === 'Failed'
      kill_thread(folder_name, uid)
      has_failed = true
    end
  end
  if has_failed
    sleep 5
  end
end
queue_thread(folder_name, uid, msg_ratio) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 381
def queue_thread(folder_name, uid, msg_ratio)
  set_thread_state(folder_name, uid, msg_ratio, 'Queued')
  bootstrap_thread(folder_name, uid)
  set_thread_property(folder_name, uid, 'Queue Time', Time.now)
  increase_thread_count(folder_name)
  Thread.abort_on_exception = true
  @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid]['Thread'] = Thread.new do
    sleep 0.1
    write_data_for_uid(folder_name, uid, msg_ratio)
  end
end
retry_check(folder_name, uid, msg_ratio) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 367
def retry_check(folder_name, uid, msg_ratio)
  if get_thread_property(folder_name, uid, 'Retry Count') <= 3
    log_thread_on_retry(folder_name, uid, msg_ratio, 'Queuing for a retry')
    increase_thread_property(folder_name, uid, 'Retry Count')
    set_thread_state(folder_name, uid, msg_ratio, 'Retrying')
    queue_thread(folder_name, uid, msg_ratio)
    return true
  else
    set_thread_property(folder_name, uid, 'End Time', Time.now)
    set_thread_state(folder_name, uid, msg_ratio, 'Failed')
    return false
  end
end
set_thread_property(folder_name, uid, property, value) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 230
def set_thread_property(folder_name, uid, property, value)
  @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property] = value
end
set_thread_state(folder_name, uid, msg_ratio, state) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 224
def set_thread_state(folder_name, uid, msg_ratio, state)
  uid_string = get_uid_string_formatted(uid)
  set_thread_property(folder_name, uid, 'State', state.to_s)
  log_thread_property(folder_name, uid, msg_ratio, 'State')
end
task_report() click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 639
def task_report
  stats = get_task_stats()
  $logger.info ''
  $logger.info '##### Task Report #####'
  $logger.info ''
  $logger.info ' Task: ' + $config[:task]
  $logger.info ''
  $logger.info ' Server: ' + $config[:server]
  $logger.info ' Username: ' + $config[:username]
  $logger.info ' Output Directory: ' + @dir_base_path
  $logger.info ''
  $logger.info ' Start Time: ' + @account_metadata[:task][:start_time].to_s
  $logger.info ' End Time: ' + @account_metadata[:task][:end_time].to_s
  $logger.info ' Duration: ' + get_duration_formatted(@account_metadata[:task][:start_time], @account_metadata[:task][:end_time])
  $logger.info ''
  $logger.info ' Folders Processed: ' + @account_metadata[:folders].count.to_s
  $logger.info ''
  @account_metadata[:folders].each do |folder_name, folder_data|
    $logger.info '  ' + folder_name
    $logger.info ''
    $logger.info '   Message Stats'
    $logger.info '    Total: ' + folder_data['messages']['server']['config'].count.to_s
    $logger.info '    Total Size: ' + stats[:folders][folder_name]['messages']['size'].to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse + ' bytes'
    $logger.info ''
    $logger.info '   Thread Stats'
    $logger.info '    Total: ' + stats[:folders][folder_name]['threads']['Total'].to_s
    $logger.info '    Success: ' + stats[:folders][folder_name]['threads']['Success'].to_s
    $logger.info '    Failed: ' + stats[:folders][folder_name]['threads']['Failed'].to_s
    if folder_data['process']
      if folder_data['process']['threads']
        if folder_data['process']['threads']['uids']
          $logger.info '   Threads'
          folder_data['process']['threads']['uids'].each do |uid, thread_data|
            if thread_data['State'] === 'Successful'
              $logger.debug '    UID: ' + uid.to_s
              $logger.debug '     Path: ' + folder_data['messages']['server']['config'][uid]['file_path']
              $logger.debug '     Thread State: ' + thread_data['State']
            else
              $logger.info '    UID: ' + uid.to_s
              $logger.info '     Path: ' + message_config['file_path']
              $logger.info '     Thread State: ' + thread_data['State']
            end
          end
        end
      end
    end
    $logger.info ''
  end
  $logger.info ''
end
update_message_config(folder_name, metadata) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 110
def update_message_config(folder_name, metadata)
  output = {}
  metadata.each do |message_metadata|
    uid = message_metadata.attr['UID']
    output[uid] = {}
    output[uid]['uid'] = uid
    output[uid]['envelope'] = message_metadata.attr['ENVELOPE'].to_h
    output[uid]['flags'] = message_metadata.attr['FLAGS']
    output[uid]['size'] = message_metadata.attr['RFC822.SIZE']
    output[uid]['file_path'] = get_file_path_for_message(folder_name, output[uid])
  end
  return output
end
write_data_for_folder(folder_name) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 583
def write_data_for_folder(folder_name)
  $logger.debug ''
  $logger.info 'Checking ' + folder_name
  @imap.examine(folder_name)
  generate_metadata_for_folder(folder_name)
  log_folder_info(folder_name)
  download_list = get_download_list(folder_name)
  create_output_directory(folder_name)
  process_deletes(folder_name)
  process_downloads(folder_name, download_list)
end
write_data_for_uid(folder_name, uid, msg_ratio) click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 393
def write_data_for_uid(folder_name, uid, msg_ratio)
  uid_string = get_uid_string_formatted(uid)
  if get_thread_property(folder_name, uid, 'State') === 'Queued'
    set_thread_state(folder_name, uid, msg_ratio, 'Active')
    log_thread_property(folder_name, uid, msg_ratio, 'File Name')
    file_path = get_thread_property(folder_name, uid, 'File Path')
    file_path_tmp = file_path + '.tmp'
    if message_data = get_message_data(uid)
      File.write(file_path_tmp, Zlib::Deflate.deflate(YAML.dump(message_data)))
      FileUtils.mv(file_path_tmp, file_path)
      set_thread_property(folder_name, uid, 'Local Size', File.size(get_thread_property(folder_name, uid, 'File Path')))
      if get_thread_property(folder_name, uid, 'Local Size') > (get_thread_property(folder_name, uid, 'Server Size') * 3)
        if !retry_check(folder_name, uid, msg_ratio)
          log_thread_on_failure(folder_name, uid, msg_ratio, 'File size mismatch after download')
        end
        if File.exists?(file_path)
          ### getting property
          property_name = 'File Name'
          file_name = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
          #####
          log_thread_event(folder_name, uid, msg_ratio, 'WARN', 'W', 'File size mismatch detected, cleaning up')
          log_thread_event(folder_name, uid, msg_ratio, 'WARN', 'W', 'Deleting File: ' + file_name)
          File.delete(file_path)
        end
      else
        set_thread_property(folder_name, uid, 'End Time', Time.now)
        set_thread_state(folder_name, uid, msg_ratio, 'Successful')
        log_thread_on_success(folder_name, uid, msg_ratio)
      end
    else
      if !retry_check(folder_name, uid, msg_ratio)
        log_thread_on_failure(folder_name, uid, msg_ratio, 'Unable to get message data')
      end
    end
    decrease_thread_count(folder_name)
  else
    ### getting property
    property_name = 'State'
    formatted_property = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name
    #####
    log_string = 'Thread state incorrect - ' + formatted_property
    log_folder_event(folder_name, 'ERROR', 'E', log_string)
    raise log_string
  end
end