class At_email::Tasks::Imap_To_Fs
Constants
- METADATA_ATTRIBUTES
- REQUESTED_ATTRIBUTES
Public Class Methods
new()
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 15 def initialize @dir_base_path = $config[:output_base_dir] + '/' + $config[:account_id] end
Public Instance Methods
bootstrap_thread(folder_name, uid)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 359 def bootstrap_thread(folder_name, uid) message_config = get_message_config(folder_name, uid) set_thread_property(folder_name, uid, 'File Name', File.basename(message_config['file_path'])) set_thread_property(folder_name, uid, 'File Path', message_config['file_path']) set_thread_property(folder_name, uid, 'Server Size', message_config['size']) set_thread_property(folder_name, uid, 'Retry Count', 0) end
check_folder_included(folder_name)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 614 def check_folder_included(folder_name) ignore_list = [] ignore_list << "[Gmail]" ignore_list << "[Gmail]/All Mail" ignore_list << "[Gmail]/Important" ignore_list << "[Gmail]/Starred" if ignore_list.include?(folder_name) return false else return true end end
create_output_directory(folder_name)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 482 def create_output_directory(folder_name) dir_path = @dir_base_path + '/' + folder_name if !Dir.exists?(dir_path) $logger.debug folder_name + ' - Creating Directory: ' + dir_path FileUtils.mkdir_p dir_path end end
decrease_thread_count(folder_name)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 269 def decrease_thread_count(folder_name) @account_metadata[:folders][folder_name]['process']['threads']['count'] -= 1 end
delete_directories(folder_name)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 171 def delete_directories(folder_name) server_dirs = @account_metadata[:folders][folder_name]['directories']['server'] @account_metadata[:folders][folder_name]['directories']['local'].each do |local_dir| if !server_dirs[local_dir] $logger.info( folder_name + ' - Deleting Directory: ' + local_dir ) end end end
delete_files(folder_name)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 205 def delete_files(folder_name) @account_metadata[:folders][folder_name]['files']['deletes'].each do |file_path| if File.exists?(file_path) $logger.info( folder_name + ' - Deleting File: ' + file_path ) File.delete(file_path) end end end
execute()
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 627 def execute @connection = At_email::Account::ImapConnection.new() @connection.login get_imap generate_metadata_for_account @account_metadata[:task][:start_time] = Time.now get_data_for_all_folders @connection.disconnect @account_metadata[:task][:end_time] = Time.now task_report end
format_file_size(file_size)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 746 def format_file_size(file_size) output = format_number(file_size) + ' bytes' return output end
format_number(number)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 751 def format_number(number) output = number.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse return output end
generate_metadata_for_account()
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 82 def generate_metadata_for_account() @account_metadata = {} @account_metadata[:account] = {} @account_metadata[:account]['directories'] = {} @account_metadata[:account]['directories']['server'] = get_server_dir_list() @account_metadata[:account]['directories']['local'] = get_local_dir_list() @account_metadata[:account]['messages'] = {} @account_metadata[:account]['messages']['server'] = {} @account_metadata[:account]['messages']['server']['folders'] = get_server_message_folder_list() @account_metadata[:task] = {} @account_metadata[:task] = {} @account_metadata[:folders] = {} end
generate_metadata_for_folder(folder_name)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 96 def generate_metadata_for_folder(folder_name) @account_metadata[:folders][folder_name] = {} @account_metadata[:folders][folder_name]['messages'] = {} @account_metadata[:folders][folder_name]['messages']['server'] = {} @account_metadata[:folders][folder_name]['messages']['server']['uids'] = get_uids() @account_metadata[:folders][folder_name]['messages']['server']['metadata'] = get_imap_metadata(@account_metadata[:folders][folder_name]['messages']['server']['uids']) @account_metadata[:folders][folder_name]['messages']['server']['config'] = update_message_config(folder_name, @account_metadata[:folders][folder_name]['messages']['server']['metadata']) @account_metadata[:folders][folder_name]['files'] = {} @account_metadata[:folders][folder_name]['files']['server'] = get_server_file_list(folder_name, @account_metadata[:folders][folder_name]['messages']['server']['config']) @account_metadata[:folders][folder_name]['files']['local'] = get_local_file_list(folder_name) @account_metadata[:folders][folder_name]['files']['deletes'] = get_delete_list(folder_name) @account_metadata[:folders][folder_name]['process'] = {} end
get_data_for_all_folders()
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 595 def get_data_for_all_folders folder_list = @account_metadata[:account]['messages']['server']['folders'] $logger.debug '' $logger.debug 'Folder Count: ' + folder_list.count.to_s folder_list.each do |folder_name| if check_folder_included(folder_name) $logger.debug ' ' + folder_name else $logger.debug ' ' + folder_name + ' - SKIPPED' end end folder_list.each do |folder_name| if check_folder_included(folder_name) write_data_for_folder(folder_name) end end $logger.debug '' end
get_delete_list(folder_name)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 192 def get_delete_list(folder_name) output = [] server_files = @account_metadata[:folders][folder_name]['files']['server'] @account_metadata[:folders][folder_name]['files']['local'].each do |file_path| if !server_files[file_path] if File.exists?(file_path) output << file_path end end end return output end
get_download_list(folder_name)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 180 def get_download_list(folder_name) output = [] local_file_list = @account_metadata[:folders][folder_name]['files']['local'] @account_metadata[:folders][folder_name]['messages']['server']['config'].each do |uid, message_config| file_path = message_config['file_path'] if !local_file_list.include?(file_path) output << uid end end return output end
get_duration(start_time, end_time)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 723 def get_duration(start_time, end_time) output = end_time.to_f - start_time.to_f return output end
get_duration_formatted(start_time, end_time)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 728 def get_duration_formatted(start_time, end_time) duration_seconds = get_duration(start_time, end_time) if duration_seconds if duration_seconds <= 60 output = sprintf("%.6f", duration_seconds) + ' Sec(s)' else nanoseconds = (sprintf("%.6f", duration_seconds).to_f * 1000000) % 1000000 seconds = duration_seconds % 60 minutes = (duration_seconds / 60) % 60 hours = duration_seconds / (60 * 60) output = format("%02d:%02d:%02d.%6d", hours, minutes, seconds, nanoseconds) end else return false end return output end
get_file_path_for_message(folder_name, message_config)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 72 def get_file_path_for_message(folder_name, message_config) output = get_file_path_prefix(folder_name, message_config)+'.'+$config[:storage_file_extension] return output end
get_file_path_prefix(folder_name, message_config)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 54 def get_file_path_prefix(folder_name, message_config) if $config[:storage_file_name_uid_padding] if $config[:storage_file_name_uid_padding_length] <= 1 storage_file_name_uid_padding = 2 else storage_file_name_uid_padding = $config[:storage_file_name_uid_padding_length] end uid_based_prefix = message_config['uid'] + (10 ** (storage_file_name_uid_padding - 1)) else uid_based_prefix = message_config['uid'] end hash_length = $config[:storage_file_name_hash_length] dir_path = @dir_base_path + '/' + folder_name file_name_hash = Digest::MD5.hexdigest(Digest::MD5.hexdigest(YAML.dump(message_config['envelope']))+'-'+Digest::MD5.hexdigest(YAML.dump(message_config['flags'])))[0...hash_length] output = dir_path+'/'+uid_based_prefix.to_s+'-'+file_name_hash return output end
get_formatted_thread_property(folder_name, uid, property_name)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 248 def get_formatted_thread_property(folder_name, uid, property_name) ### getting property name = property_name value = get_thread_property(folder_name, uid, name) || 'Unable to get property: ' + name ##### output = name + ': ' + value.to_s return output end
get_imap()
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 19 def get_imap @imap = @connection.imap end
get_imap_metadata(uid_list)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 34 def get_imap_metadata(uid_list) output = {} if uid_list.count > 0 output = @imap.uid_fetch(uid_list, METADATA_ATTRIBUTES) end return output end
get_local_dir_list()
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 148 def get_local_dir_list() output = [] if Dir.exists?(@dir_base_path) Dir.chdir(@dir_base_path) file_list = Dir.glob("**/*/") file_list.each do |object_name| if File.directory?(object_name) output << @dir_base_path + '/' + object_name end end end return output end
get_local_file_list(folder_name)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 124 def get_local_file_list(folder_name) output = [] dir_path = @dir_base_path + '/' + folder_name if Dir.exists?(dir_path) Dir.chdir(dir_path) file_list = Dir.glob("*."+$config[:storage_file_extension]).sort_by{|f| f.split(" ")[0].to_i } file_list.each do |file_name| if File.file?(file_name) output << dir_path+'/'+file_name end end end return output end
get_message_config(folder_name, uid)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 350 def get_message_config(folder_name, uid) if @account_metadata[:folders][folder_name]['messages']['server']['config'][uid] output = @account_metadata[:folders][folder_name]['messages']['server']['config'][uid] return output else raise '***** ERROR ***** - Message Config: Cannot get message config for folder: ' + folder_name + ' - UID:' + uid end end
get_message_data(uid)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 42 def get_message_data(uid) data_raw = @imap.uid_fetch(uid.to_i, REQUESTED_ATTRIBUTES) if data_raw if data_raw[0] if data_raw[0].attr output = data_raw[0].attr return output end end end end
get_ratio_formatted(a, b, delimiter = '/')
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 214 def get_ratio_formatted(a, b, delimiter = '/') output = a.to_s + delimiter + b.to_s return output end
get_server_dir_list()
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 162 def get_server_dir_list() output = [] message_folder_list = get_server_message_folder_list() message_folder_list.each do |message_folder| output << @dir_base_path + '/' + message_folder end return output end
get_server_file_list(folder_name, server_message_config)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 139 def get_server_file_list(folder_name, server_message_config) output = {} server_message_config.each do |uid, message_config| file_path = get_file_path_for_message(folder_name, message_config) output[file_path] = uid end return output end
get_server_message_folder_list()
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 23 def get_server_message_folder_list() output = [] root_path = @imap.list("", "")[0].name folder_data = @imap.list(root_path, "*") output = [] folder_data.map do |this_folder_data| output << this_folder_data.name end return output end
get_task_stats()
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 690 def get_task_stats() output = {} output[:account] = {} output[:folders] = {} @account_metadata[:folders].each do |folder_name, folder_data| output[:folders][folder_name] = {} output[:folders][folder_name]['messages'] = {} output[:folders][folder_name]['messages']['size'] = 0 folder_data['messages']['server']['config'].each do |uid, message_config| output[:folders][folder_name]['messages']['size'] += message_config['size'] end output[:folders][folder_name]['threads'] = {} output[:folders][folder_name]['threads']['Total'] = 0 output[:folders][folder_name]['threads']['Success'] = 0 output[:folders][folder_name]['threads']['Failed'] = 0 if folder_data['process'] if folder_data['process']['threads'] if folder_data['process']['threads']['uids'] folder_data['process']['threads']['uids'].each do |uid, thread_data| output[:folders][folder_name]['threads']['Total'] += 1 if thread_data['State'] === 'Successful' output[:folders][folder_name]['threads']['Success'] += 1 else output[:folders][folder_name]['threads']['Failed'] += 1 end end end end end end return output end
get_thread_id_string(uid, msg_ratio)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 273 def get_thread_id_string(uid, msg_ratio) uid_string = get_uid_string_formatted(uid) output = msg_ratio + ' - ' + uid_string end
get_thread_property(folder_name, uid, property)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 234 def get_thread_property(folder_name, uid, property) if @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property] output = @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property] return output else return false end end
get_uid_string_formatted(uid)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 219 def get_uid_string_formatted(uid) output = 'UID:' + uid.to_s return output end
get_uids()
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 77 def get_uids() uid_list = @imap.uid_search(["ALL"]).sort return uid_list end
increase_thread_count(folder_name)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 265 def increase_thread_count(folder_name) @account_metadata[:folders][folder_name]['process']['threads']['count'] += 1 end
increase_thread_property(folder_name, uid, property)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 257 def increase_thread_property(folder_name, uid, property) @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property] += 1 end
kill_thread(folder_name, uid)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 500 def kill_thread(folder_name, uid) log_folder_event(folder_name, 'WARN', 'ERROR', 'Thread Failure - ' + get_uid_string_formatted(uid)) log_folder_event(folder_name, 'WARN', 'ERROR', ' Killing Thread') Thread.kill(get_thread_property(folder_name, uid, 'Thread')) ### getting formatted thread property property_name = 'Queue Time' log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name ##### log_folder_event(folder_name, 'WARN', 'ERROR', ' ' + log_string) ### getting formatted thread property property_name = 'End Time' log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name ##### log_folder_event(folder_name, 'WARN', 'ERROR', ' ' + log_string) $logger.error folder_name + ' - Duration: ' + get_duration_formatted(get_thread_property(folder_name, uid, 'Queue Time'), get_thread_property(folder_name, uid, 'End Time')) ### getting formatted thread property property_name = 'File Name' log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name ##### log_folder_event(folder_name, 'WARN', 'ERROR', ' ' + log_string) end
log_folder_event(folder_name, event_level, event_tag, event_data)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 345 def log_folder_event(folder_name, event_level, event_tag, event_data) log_string = folder_name + ' - ' + event_data $logger.event(event_level, event_tag, log_string) end
log_folder_info(folder_name)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 471 def log_folder_info(folder_name) $logger.debug folder_name + ' - Server Files: ' + @account_metadata[:folders][folder_name]['files']['server'].count.to_s @account_metadata[:folders][folder_name]['files']['server'].each do |file_path, uid| $logger.debug folder_name + ' - ' + File.basename(file_path) end $logger.debug folder_name + ' - Local Files: ' + @account_metadata[:folders][folder_name]['files']['local'].count.to_s @account_metadata[:folders][folder_name]['files']['local'].each do |file_path| $logger.debug folder_name + ' - ' + File.basename(file_path) end end
log_thread_event(folder_name, uid, msg_ratio, event_level, event_tag='', event_data)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 338 def log_thread_event(folder_name, uid, msg_ratio, event_level, event_tag='', event_data) uid_string = get_uid_string_formatted(uid) || 'Unable to get UID string' thread_id_string = get_thread_id_string(uid, msg_ratio) || 'Unable to get thread ID string' log_string = thread_id_string + ' - ' + event_data log_folder_event(folder_name, event_level, event_tag, log_string) end
log_thread_on_failure(folder_name, uid, msg_ratio, event_data)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 306 def log_thread_on_failure(folder_name, uid, msg_ratio, event_data) ### getting property property_name = 'Queue Time' queue_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name ##### ### getting property property_name = 'End Time' end_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name ##### ### getting property property_name = 'Server Size' server_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name ##### ### getting property property_name = 'Local Size' local_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name ##### ### getting data duration_formatted = get_duration_formatted(queue_time, end_time) || 'Unable to get data' ##### ### getting data size_formatted = server_size + ' Bytes' ##### log_string = 'Duration: ' + duration_formatted + ' - Size: ' + size_formatted log_thread_event(folder_name, uid, msg_ratio, 'ERROR', 'E', log_string) log_thread_event(folder_name, uid, msg_ratio, 'ERROR', 'ERROR', event_data) end
log_thread_on_retry(folder_name, uid, msg_ratio, event_data)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 334 def log_thread_on_retry(folder_name, uid, msg_ratio, event_data) log_thread_event(folder_name, uid, msg_ratio, 'WARN', 'W', event_data) end
log_thread_on_success(folder_name, uid, msg_ratio)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 278 def log_thread_on_success(folder_name, uid, msg_ratio) ### getting property property_name = 'Queue Time' queue_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name ##### ### getting property property_name = 'End Time' end_time = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name ##### ### getting property property_name = 'Server Size' server_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name ##### ### getting property property_name = 'Local Size' local_size = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name ##### ### getting data duration_formatted = get_duration_formatted(queue_time, end_time) || 'Unable to get data' ##### ### getting data size_ratio_formatted = get_ratio_formatted(local_size, server_size, ' / ') || 'Unable to get data' size_formatted = size_ratio_formatted + ' Bytes' ##### log_string = 'Duration: ' + duration_formatted + ' - Size: ' + size_formatted log_thread_event(folder_name, uid, msg_ratio, 'INFO', '', log_string) end
log_thread_property(folder_name, uid, msg_ratio, property_name)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 243 def log_thread_property(folder_name, uid, msg_ratio, property_name) log_string = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name log_thread_event(folder_name, uid, msg_ratio, 'INFO', '', log_string) end
process_deletes(folder_name)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 490 def process_deletes(folder_name) if $config[:delete_on_sync] if @account_metadata[:folders][folder_name]['files']['deletes'].count > 0 $logger.info folder_name + ' - Delete Count: ' + @account_metadata[:folders][folder_name]['files']['deletes'].count.to_s $logger.info folder_name + ' - Executing Deletes' delete_files(folder_name) end end end
process_downloads(folder_name, download_list)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 522 def process_downloads(folder_name, download_list) process_healthy = true if download_list.count > 0 $logger.info( folder_name + ' - ' + download_list.count.to_s + ' Message(s) to download - Thread Limit: ' + $config[:threads].to_s ) message_count = 0 ### Creating / resetting active thread data structure @account_metadata[:folders][folder_name]['process']['threads'] = {} @account_metadata[:folders][folder_name]['process']['threads']['count'] = 0 @account_metadata[:folders][folder_name]['process']['threads']['uids'] = {} message_total = download_list.count download_list.each do |uid| if process_healthy message_count += 1 uid_string = get_uid_string_formatted(uid) msg_ratio = get_ratio_formatted(message_count, message_total) last_queue_time = Time.now if @account_metadata[:folders][folder_name]['process']['threads']['count'] >= $config[:threads] $logger.debug folder_name + ' - Thread Count: ' + @account_metadata[:folders][folder_name]['process']['threads']['count'].to_s + ' - Waiting...' end while @account_metadata[:folders][folder_name]['process']['threads']['count'] >= $config[:threads] && process_healthy if (Time.now.to_i - last_queue_time.to_i) >= ($config[:thread_timeout] * $config[:timeout_warning_ratio]) if (Time.now.to_i - last_queue_time.to_i) >= $config[:thread_timeout] $logger.error folder_name + ' - *E* Thread queuing has stopped - Timeout: ' + $config[:thread_timeout].to_s + ' Sec(s)' process_healthy = false else $logger.warn folder_name + ' - *W* Thread queuing has stopped - Timeout: ' + (Time.now.to_i - last_queue_time.to_i).to_s + '/' + $config[:thread_timeout].to_s + ' Sec(s)' sleep rand(10..20) end else sleep 0.2 end end if process_healthy @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid] = {} queue_thread(folder_name, uid, msg_ratio) sleep 0.1 while get_thread_property(folder_name, uid, 'State') === 'Queued' if (Time.now.to_i - get_thread_property(folder_name, uid, 'Queue Time').to_i) >= ($config[:thread_timeout] * $config[:timeout_warning_ratio]) if (Time.now.to_i - get_thread_property(folder_name, uid, 'Queue Time').to_i) >= $config[:thread_timeout] $logger.error folder_name + ' - ' + msg_ratio + ' - ' + uid_string + ' - *E* Thread state - Moving on... - Timeout: ' + $config[:thread_timeout].to_s + ' Sec(s)' set_thread_state(folder_name, uid, msg_ratio, 'Failed') kill_thread(folder_name, uid) else $logger.warn folder_name + ' - ' + msg_ratio + ' - ' + uid_string + ' - *W* Thread state - Timeout: ' + (Time.now.to_i - get_thread_property(folder_name, uid, 'Queue Time').to_i).to_s + '/' + $config[:thread_timeout].to_s + ' Sec(s)' sleep rand(10..20) end else sleep 0.2 end end end end end if process_healthy sleep 2 process_shutdown(folder_name) end process_thread_data_before_finish(folder_name) end end
process_shutdown(folder_name)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 457 def process_shutdown(folder_name) time_started_waiting = Time.now.to_i while @account_metadata[:folders][folder_name]['process']['threads']['count'] > 0 time_since_started_waiting = Time.now.to_i - time_started_waiting if time_since_started_waiting >= $config[:thread_timeout] $logger.warn folder_name + ' - *E* - Thread pool shutdown - Timeout: ' + $config[:thread_timeout].to_s + ' Sec(s)' @account_metadata[:folders][folder_name]['process']['threads']['count'] = 0 else $logger.info folder_name + ' - Waiting for threads - Remaining: ' + @account_metadata[:folders][folder_name]['process']['threads']['count'].to_s + '/' + $config[:threads].to_s + ' - Timeout: ' + time_since_started_waiting.to_s + '/' + $config[:thread_timeout].to_s + ' Sec(s)' sleep 10 end end end
process_thread_data_before_finish(folder_name)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 441 def process_thread_data_before_finish(folder_name) has_failed = false @account_metadata[:folders][folder_name]['process']['threads']['uids'].each do |uid, thread_data| if thread_data['State'] != 'Successful' thread_data['State'] = 'Failed' end if thread_data['State'] === 'Failed' kill_thread(folder_name, uid) has_failed = true end end if has_failed sleep 5 end end
queue_thread(folder_name, uid, msg_ratio)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 381 def queue_thread(folder_name, uid, msg_ratio) set_thread_state(folder_name, uid, msg_ratio, 'Queued') bootstrap_thread(folder_name, uid) set_thread_property(folder_name, uid, 'Queue Time', Time.now) increase_thread_count(folder_name) Thread.abort_on_exception = true @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid]['Thread'] = Thread.new do sleep 0.1 write_data_for_uid(folder_name, uid, msg_ratio) end end
retry_check(folder_name, uid, msg_ratio)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 367 def retry_check(folder_name, uid, msg_ratio) if get_thread_property(folder_name, uid, 'Retry Count') <= 3 log_thread_on_retry(folder_name, uid, msg_ratio, 'Queuing for a retry') increase_thread_property(folder_name, uid, 'Retry Count') set_thread_state(folder_name, uid, msg_ratio, 'Retrying') queue_thread(folder_name, uid, msg_ratio) return true else set_thread_property(folder_name, uid, 'End Time', Time.now) set_thread_state(folder_name, uid, msg_ratio, 'Failed') return false end end
set_thread_property(folder_name, uid, property, value)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 230 def set_thread_property(folder_name, uid, property, value) @account_metadata[:folders][folder_name]['process']['threads']['uids'][uid][property] = value end
set_thread_state(folder_name, uid, msg_ratio, state)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 224 def set_thread_state(folder_name, uid, msg_ratio, state) uid_string = get_uid_string_formatted(uid) set_thread_property(folder_name, uid, 'State', state.to_s) log_thread_property(folder_name, uid, msg_ratio, 'State') end
task_report()
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 639 def task_report stats = get_task_stats() $logger.info '' $logger.info '##### Task Report #####' $logger.info '' $logger.info ' Task: ' + $config[:task] $logger.info '' $logger.info ' Server: ' + $config[:server] $logger.info ' Username: ' + $config[:username] $logger.info ' Output Directory: ' + @dir_base_path $logger.info '' $logger.info ' Start Time: ' + @account_metadata[:task][:start_time].to_s $logger.info ' End Time: ' + @account_metadata[:task][:end_time].to_s $logger.info ' Duration: ' + get_duration_formatted(@account_metadata[:task][:start_time], @account_metadata[:task][:end_time]) $logger.info '' $logger.info ' Folders Processed: ' + @account_metadata[:folders].count.to_s $logger.info '' @account_metadata[:folders].each do |folder_name, folder_data| $logger.info ' ' + folder_name $logger.info '' $logger.info ' Message Stats' $logger.info ' Total: ' + folder_data['messages']['server']['config'].count.to_s $logger.info ' Total Size: ' + stats[:folders][folder_name]['messages']['size'].to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse + ' bytes' $logger.info '' $logger.info ' Thread Stats' $logger.info ' Total: ' + stats[:folders][folder_name]['threads']['Total'].to_s $logger.info ' Success: ' + stats[:folders][folder_name]['threads']['Success'].to_s $logger.info ' Failed: ' + stats[:folders][folder_name]['threads']['Failed'].to_s if folder_data['process'] if folder_data['process']['threads'] if folder_data['process']['threads']['uids'] $logger.info ' Threads' folder_data['process']['threads']['uids'].each do |uid, thread_data| if thread_data['State'] === 'Successful' $logger.debug ' UID: ' + uid.to_s $logger.debug ' Path: ' + folder_data['messages']['server']['config'][uid]['file_path'] $logger.debug ' Thread State: ' + thread_data['State'] else $logger.info ' UID: ' + uid.to_s $logger.info ' Path: ' + message_config['file_path'] $logger.info ' Thread State: ' + thread_data['State'] end end end end end $logger.info '' end $logger.info '' end
update_message_config(folder_name, metadata)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 110 def update_message_config(folder_name, metadata) output = {} metadata.each do |message_metadata| uid = message_metadata.attr['UID'] output[uid] = {} output[uid]['uid'] = uid output[uid]['envelope'] = message_metadata.attr['ENVELOPE'].to_h output[uid]['flags'] = message_metadata.attr['FLAGS'] output[uid]['size'] = message_metadata.attr['RFC822.SIZE'] output[uid]['file_path'] = get_file_path_for_message(folder_name, output[uid]) end return output end
write_data_for_folder(folder_name)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 583 def write_data_for_folder(folder_name) $logger.debug '' $logger.info 'Checking ' + folder_name @imap.examine(folder_name) generate_metadata_for_folder(folder_name) log_folder_info(folder_name) download_list = get_download_list(folder_name) create_output_directory(folder_name) process_deletes(folder_name) process_downloads(folder_name, download_list) end
write_data_for_uid(folder_name, uid, msg_ratio)
click to toggle source
# File lib/at_email/tasks/imap_to_fs.rb, line 393 def write_data_for_uid(folder_name, uid, msg_ratio) uid_string = get_uid_string_formatted(uid) if get_thread_property(folder_name, uid, 'State') === 'Queued' set_thread_state(folder_name, uid, msg_ratio, 'Active') log_thread_property(folder_name, uid, msg_ratio, 'File Name') file_path = get_thread_property(folder_name, uid, 'File Path') file_path_tmp = file_path + '.tmp' if message_data = get_message_data(uid) File.write(file_path_tmp, Zlib::Deflate.deflate(YAML.dump(message_data))) FileUtils.mv(file_path_tmp, file_path) set_thread_property(folder_name, uid, 'Local Size', File.size(get_thread_property(folder_name, uid, 'File Path'))) if get_thread_property(folder_name, uid, 'Local Size') > (get_thread_property(folder_name, uid, 'Server Size') * 3) if !retry_check(folder_name, uid, msg_ratio) log_thread_on_failure(folder_name, uid, msg_ratio, 'File size mismatch after download') end if File.exists?(file_path) ### getting property property_name = 'File Name' file_name = get_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name ##### log_thread_event(folder_name, uid, msg_ratio, 'WARN', 'W', 'File size mismatch detected, cleaning up') log_thread_event(folder_name, uid, msg_ratio, 'WARN', 'W', 'Deleting File: ' + file_name) File.delete(file_path) end else set_thread_property(folder_name, uid, 'End Time', Time.now) set_thread_state(folder_name, uid, msg_ratio, 'Successful') log_thread_on_success(folder_name, uid, msg_ratio) end else if !retry_check(folder_name, uid, msg_ratio) log_thread_on_failure(folder_name, uid, msg_ratio, 'Unable to get message data') end end decrease_thread_count(folder_name) else ### getting property property_name = 'State' formatted_property = get_formatted_thread_property(folder_name, uid, property_name) || 'Unable to get property: ' + property_name ##### log_string = 'Thread state incorrect - ' + formatted_property log_folder_event(folder_name, 'ERROR', 'E', log_string) raise log_string end end