class ExportMongoS3::Application

Public Class Methods

new(argv) click to toggle source
# File lib/export_mongo_s3/application.rb, line 4
def initialize(argv)
  @parser = OptionParser.new

  @params = parse_options(argv)
  @config = parse_config(@params[:config])

  db_options = @config[:mongo]
  db_options.merge!(collections: @config[:export][:collections])

  @db = Db.new(db_options)

  @storage = Storage.new(@config[:s3])
end

Public Instance Methods

run() click to toggle source
# File lib/export_mongo_s3/application.rb, line 21
def run

  case
    when @params[:export_list] # EXPORT_LIST
      show_uploaded_files(@params[:export_list])

    when @params[:export_all] # EXPORT_ALL
      dbs_str = @config[:export][:dbs]

      if dbs_str.nil? || dbs_str.empty?
        raise 'config.yml::export.dbs is empty'
      end

      db_names = dbs_str.split(',').each { |db_name| db_name.strip! }

      export(db_names)

    when @params[:export] # EXPORT
      export([@params[:export]])

    when @params[:cron_update] || @params[:cron_clear] # CRON
      cron_options =
          {
              update: @params[:cron_update],
              clear:  @params[:cron_clear],
              config: @params[:config],
              time:   @config[:export][:cron_time],
          }

      Scheduler.new(cron_options).execute

    else
      puts "\n#{@parser}\n"
      exit(0)
  end

end

Private Instance Methods

create_config(path) click to toggle source
# File lib/export_mongo_s3/application.rb, line 131
def create_config(path)

  path = '.' if path.nil? || path == ''

  file = File.join(path, 'config.yml')

  if File.exists?(file)
    raise "create_config: '#{file}' already exists"
  elsif File.exists?(file.downcase)
    raise "create_config: '#{file.downcase}' exists, which could conflict with '#{file}'"
  elsif !File.exists?(File.dirname(file))
    raise "create_config: directory '#{File.dirname(file)}' does not exist"
  else
    file_template = File.join(ExportMongoS3.root_path, 'lib/helpers/config.yml.template')

    FileUtils.cp file_template, file
  end

  puts "[done] file #{file} was created"
end
export(db_names) click to toggle source
# File lib/export_mongo_s3/application.rb, line 84
def export(db_names)

  db_names.each do |db_name|

    puts "export db #{db_name.upcase}:"

    tmp_dir = get_temp_dir

    begin

      export_path = File.join(tmp_dir, db_name)

      puts "\t make csv..."
      @db.export_db(db_name, export_path)

      if Dir["#{export_path}/*"].empty?
        puts "\t [skip] db is empty"

      else

        puts "\t compress..."
        compression_level = get_compression_level
        zip_result        = %x(zip -#{compression_level} -r '#{export_path}.zip' '#{export_path}/' -j)
        raise "Error zip. Msg: #{zip_result}" unless $?.exitstatus.zero?

        FileUtils.rm_rf(export_path)

        zip_file_path = "#{export_path}.zip"

        if File.exists?(zip_file_path)
          puts "\t upload to s3..."
          @storage.upload(db_name, zip_file_path)

          File.delete(zip_file_path)
        end

      end

    ensure
      FileUtils.remove_entry_secure(tmp_dir)
    end

  end

  puts '[done] export'
end
get_compression_level() click to toggle source
# File lib/export_mongo_s3/application.rb, line 226
def get_compression_level
  compression_level = @config[:export][:compression_level]

  if compression_level.nil? || compression_level == ''
    result = 6 #default value
  else
    result = Integer(compression_level)
    unless result >= 0 and result <= 9
      raise 'compression_level is not in [0..9]'
    end
  end

  result
end
get_temp_dir() click to toggle source
# File lib/export_mongo_s3/application.rb, line 216
def get_temp_dir
  temp_dir = @config[:export][:temp_directory]

  if temp_dir.nil? || temp_dir == ''
    temp_dir = Dir.tmpdir
  end

  Dir.mktmpdir(nil, temp_dir)
end
parse_config(config_path) click to toggle source
# File lib/export_mongo_s3/application.rb, line 197
def parse_config(config_path)

  begin
    config = YAML.load(File.read(config_path))
  rescue Errno::ENOENT
    raise "Could not find config file '#{config_path}'"
  rescue ArgumentError => error
    raise "Could not parse config file '#{config_path}' - #{error}"
  end

  config.deep_symbolize_keys!

  raise 'config.yml. Section <export> not found' if config[:export].nil?
  raise 'config.yml. Section <mongo> not found' if config[:mongo].nil?
  raise 'config.yml. Section <s3> not found' if config[:s3].nil?

  config
end
parse_options(argv) click to toggle source
# File lib/export_mongo_s3/application.rb, line 152
def parse_options(argv)
  params = {}

  @parser.on('--export_all', 'Export databases specified in config.yml and upload to S3 bucket') do
    params[:export_all] = true
  end
  @parser.on('--export DB_NAME', String, 'Export database and upload to S3 bucket') do |db_name|
    params[:export] = db_name
  end
  @parser.on('-l', '--list_exported [DB_NAME]', String, 'Show list of exported dbs') do |db_name|
    params[:export_list] = db_name || ''
  end
  @parser.on('--write_cron', 'Add/update export_all job') do
    params[:cron_update] = true
  end
  @parser.on('--clear_cron', 'Clear export_all job') do
    params[:cron_clear] = true
  end
  @parser.on('-c', '--config PATH', String, 'Path to config *.yml. Default: ./config.yml') do |path|
    params[:config] = path || ''
  end
  @parser.on('--create_config [PATH]', String, 'Create template config.yml in current/PATH directory') do |path|
    create_config(path)
    exit(0)
  end
  @parser.on('-h', '--help', 'Show help') do
    puts "\n#{@parser}\n"
    exit(0)
  end

  @parser.parse!(argv)

  if [params[:export_all], params[:export], params[:export_list], params[:cron_update], params[:cron_clear]].compact.length > 1
    raise 'Can only export_all, export, list_exported, write_cron or clear_cron. Choose one.'
  end

  if params[:config].nil? || params[:config] == ''
    params[:config] = './config.yml'
  end

  params[:config] = File.absolute_path(params[:config])

  params
end
show_uploaded_files(db_name) click to toggle source
# File lib/export_mongo_s3/application.rb, line 62
def show_uploaded_files(db_name)

  files = @storage.get_uploaded_files(db_name)

  if files.empty?
    puts 'Files not found'
  else

    puts sprintf('%-30s %-15s %-20s', 'name', 'size, MB', 'last_modified')

    files.each do |file|

      file_name          = file.key
      file_size          = file.content_length / 1024 / 1024
      file_last_modified = file.last_modified

      puts sprintf('%-30s %-15s %-20s', file_name, file_size, file_last_modified)
    end

  end
end