class ExportMongoS3::Application
Public Class Methods
new(argv)
click to toggle source
# File lib/export_mongo_s3/application.rb, line 4 def initialize(argv) @parser = OptionParser.new @params = parse_options(argv) @config = parse_config(@params[:config]) db_options = @config[:mongo] db_options.merge!(collections: @config[:export][:collections]) @db = Db.new(db_options) @storage = Storage.new(@config[:s3]) end
Public Instance Methods
run()
click to toggle source
# File lib/export_mongo_s3/application.rb, line 21 def run case when @params[:export_list] # EXPORT_LIST show_uploaded_files(@params[:export_list]) when @params[:export_all] # EXPORT_ALL dbs_str = @config[:export][:dbs] if dbs_str.nil? || dbs_str.empty? raise 'config.yml::export.dbs is empty' end db_names = dbs_str.split(',').each { |db_name| db_name.strip! } export(db_names) when @params[:export] # EXPORT export([@params[:export]]) when @params[:cron_update] || @params[:cron_clear] # CRON cron_options = { update: @params[:cron_update], clear: @params[:cron_clear], config: @params[:config], time: @config[:export][:cron_time], } Scheduler.new(cron_options).execute else puts "\n#{@parser}\n" exit(0) end end
Private Instance Methods
create_config(path)
click to toggle source
# File lib/export_mongo_s3/application.rb, line 131 def create_config(path) path = '.' if path.nil? || path == '' file = File.join(path, 'config.yml') if File.exists?(file) raise "create_config: '#{file}' already exists" elsif File.exists?(file.downcase) raise "create_config: '#{file.downcase}' exists, which could conflict with '#{file}'" elsif !File.exists?(File.dirname(file)) raise "create_config: directory '#{File.dirname(file)}' does not exist" else file_template = File.join(ExportMongoS3.root_path, 'lib/helpers/config.yml.template') FileUtils.cp file_template, file end puts "[done] file #{file} was created" end
export(db_names)
click to toggle source
# File lib/export_mongo_s3/application.rb, line 84 def export(db_names) db_names.each do |db_name| puts "export db #{db_name.upcase}:" tmp_dir = get_temp_dir begin export_path = File.join(tmp_dir, db_name) puts "\t make csv..." @db.export_db(db_name, export_path) if Dir["#{export_path}/*"].empty? puts "\t [skip] db is empty" else puts "\t compress..." compression_level = get_compression_level zip_result = %x(zip -#{compression_level} -r '#{export_path}.zip' '#{export_path}/' -j) raise "Error zip. Msg: #{zip_result}" unless $?.exitstatus.zero? FileUtils.rm_rf(export_path) zip_file_path = "#{export_path}.zip" if File.exists?(zip_file_path) puts "\t upload to s3..." @storage.upload(db_name, zip_file_path) File.delete(zip_file_path) end end ensure FileUtils.remove_entry_secure(tmp_dir) end end puts '[done] export' end
get_compression_level()
click to toggle source
# File lib/export_mongo_s3/application.rb, line 226 def get_compression_level compression_level = @config[:export][:compression_level] if compression_level.nil? || compression_level == '' result = 6 #default value else result = Integer(compression_level) unless result >= 0 and result <= 9 raise 'compression_level is not in [0..9]' end end result end
get_temp_dir()
click to toggle source
# File lib/export_mongo_s3/application.rb, line 216 def get_temp_dir temp_dir = @config[:export][:temp_directory] if temp_dir.nil? || temp_dir == '' temp_dir = Dir.tmpdir end Dir.mktmpdir(nil, temp_dir) end
parse_config(config_path)
click to toggle source
# File lib/export_mongo_s3/application.rb, line 197 def parse_config(config_path) begin config = YAML.load(File.read(config_path)) rescue Errno::ENOENT raise "Could not find config file '#{config_path}'" rescue ArgumentError => error raise "Could not parse config file '#{config_path}' - #{error}" end config.deep_symbolize_keys! raise 'config.yml. Section <export> not found' if config[:export].nil? raise 'config.yml. Section <mongo> not found' if config[:mongo].nil? raise 'config.yml. Section <s3> not found' if config[:s3].nil? config end
parse_options(argv)
click to toggle source
# File lib/export_mongo_s3/application.rb, line 152 def parse_options(argv) params = {} @parser.on('--export_all', 'Export databases specified in config.yml and upload to S3 bucket') do params[:export_all] = true end @parser.on('--export DB_NAME', String, 'Export database and upload to S3 bucket') do |db_name| params[:export] = db_name end @parser.on('-l', '--list_exported [DB_NAME]', String, 'Show list of exported dbs') do |db_name| params[:export_list] = db_name || '' end @parser.on('--write_cron', 'Add/update export_all job') do params[:cron_update] = true end @parser.on('--clear_cron', 'Clear export_all job') do params[:cron_clear] = true end @parser.on('-c', '--config PATH', String, 'Path to config *.yml. Default: ./config.yml') do |path| params[:config] = path || '' end @parser.on('--create_config [PATH]', String, 'Create template config.yml in current/PATH directory') do |path| create_config(path) exit(0) end @parser.on('-h', '--help', 'Show help') do puts "\n#{@parser}\n" exit(0) end @parser.parse!(argv) if [params[:export_all], params[:export], params[:export_list], params[:cron_update], params[:cron_clear]].compact.length > 1 raise 'Can only export_all, export, list_exported, write_cron or clear_cron. Choose one.' end if params[:config].nil? || params[:config] == '' params[:config] = './config.yml' end params[:config] = File.absolute_path(params[:config]) params end
show_uploaded_files(db_name)
click to toggle source
# File lib/export_mongo_s3/application.rb, line 62 def show_uploaded_files(db_name) files = @storage.get_uploaded_files(db_name) if files.empty? puts 'Files not found' else puts sprintf('%-30s %-15s %-20s', 'name', 'size, MB', 'last_modified') files.each do |file| file_name = file.key file_size = file.content_length / 1024 / 1024 file_last_modified = file.last_modified puts sprintf('%-30s %-15s %-20s', file_name, file_size, file_last_modified) end end end