module TinyBackup

Constants

VERSION

Public Class Methods

configure() { |config ||= config| ... } click to toggle source

Set global settings for TinyBackup like TinyBackup.configure {|config| config.max_versions = 100 }

# File lib/configure.rb, line 3
def self.configure &block
  yield @config ||= Config.new
end

Public Instance Methods

backup_now() click to toggle source

Create a backup of the current database and choose automatically to create a .zip or .diff file.

# File lib/tiny_backup.rb, line 19
def backup_now
  # if the resource is locked, we skip to ensure block
  lock
  locked_by_this_method = true

  tmp_files = []
  nvf = new_version_filename

  stream = StringIO.new
  ActiveRecord::SchemaDumper.dump(ActiveRecord::Base.connection, stream)
  schema_rb = stream.string

  if nvf.split(".").last == "zip"
    # there is no .zip file so we must create one and add schema.rb and .csv file for each table
    # TODO: use a much better compression like Zlib::BEST_COMPRESSION to reduce the zip size, but this will consume processing power
    ZIPLIB.open("#{config.backup_folder}/#{nvf}", ZIPLIB::CREATE) do |f|
      t_benchmark = Benchmark.ms do
        f.get_output_stream("schema.rb") do |ff|
          ff.write schema_rb
          tmp_files << ff if ZIPOLD
        end
      end
      puts "-- backup_schema\n   -> #{'%.4f' % (t_benchmark/1000)}s\n" if !config.silent

      ActiveRecord::Base.connection.tables.each do |table|
        next if table == "schema_migrations"
        query_count = ActiveRecord::Base.connection.execute("SELECT COUNT(*) FROM #{table}").first.first

        t_benchmark = Benchmark.ms do
          if query_count > 0
            rows = []
            query_index = 0

            loop do
              break if query_index >= query_count
              query = ActiveRecord::Base.connection.execute("SELECT * FROM #{table} LIMIT #{config.per_page} OFFSET #{query_index}")
              rows << add_query(query.fields) if query_index == 0
              query.each { |row| rows << add_query(row) }
              query_index += config.per_page
            end

            f.get_output_stream("#{table}.csv") do |ff|
              ff.write rows.join
              tmp_files << ff if ZIPOLD
            end
          end
        end
        puts "-- backup_table(\"#{table}\")\n   -> #{'%.4f' % (t_benchmark/1000)}s\n" if !config.silent
      end
    end

  else
    # a new .diff file is created with the diff between origin_zip and tmp_origin_zip(made by merging all the versions into origin)
    tmp_origin_zip = "#{config.backup_folder}/#{compact_original(:all)}"
    tables = ActiveRecord::Base.connection.tables
    is_empty = true

    File.open("#{config.backup_folder}/#{nvf}", "wb") do |f|
      ZIPLIB.open(tmp_origin_zip) do |zf|
        zf.entries.each do |zf_entry|

          if zf_entry.name == "schema.rb"
            t_benchmark = Benchmark.ms do
              tables.delete "schema_migrations"
              this_diff = diff_files zf.read(zf_entry.name), schema_rb

              if this_diff.present?
                is_empty = false
                f.write "***************\n"
                f.write "*** schema.rb \n"
                f.write "\n"
                this_diff.each { |i| f.write i }
                f.write "\n\n"
              end
            end
            puts "-- backup_schema\n   -> #{'%.4f' % (t_benchmark/1000)}s\n" if !config.silent
          else

            table = zf_entry.name.split(".").first
            tables.delete table
            begin
              query_count = ActiveRecord::Base.connection.execute("SELECT COUNT(*) FROM #{table}").first.first
            rescue ActiveRecord::StatementInvalid
              next
            end

            rows = []
            query_index = 0

            t_benchmark = Benchmark.ms do
              loop do
                break if query_index >= query_count
                query = ActiveRecord::Base.connection.execute("SELECT * FROM #{table} LIMIT #{config.per_page} OFFSET #{query_index}")
                rows << add_query(query.fields) if query_index == 0
                query.each { |row| rows << add_query(row) }
                query_index += config.per_page
              end

              this_diff = diff_files zf.read(zf_entry.name), rows.join

              if this_diff.present?
                is_empty = false
                f.write "***************\n"
                f.write "*** #{zf_entry.name} \n"
                f.write "\n"
                this_diff.each { |i| f.write i }
                f.write "\n\n"
              end
            end
            puts "-- backup_table(\"#{table}\")\n   -> #{'%.4f' % (t_benchmark/1000)}s\n" if !config.silent
          end

        end
      end

      # tables that are created recently and doesn't have a .csv file in the tmp_origin_zip
      tables.each do |table|
        begin
          query_count = ActiveRecord::Base.connection.execute("SELECT COUNT(*) FROM #{table}").first.first
        rescue ActiveRecord::StatementInvalid
          next
        end

        rows = []
        query_index = 0

        t_benchmark = Benchmark.ms do
          loop do
            break if query_index >= query_count
            query = ActiveRecord::Base.connection.execute("SELECT * FROM #{table} LIMIT #{config.per_page} OFFSET #{query_index}")

            rows << add_query(query.fields) if query_index == 0
            query.each { |row| rows << add_query(row) }
            query_index += config.per_page
          end

          this_diff = diff_files "", rows.join

          if this_diff.present?
            is_empty = false
            f.write "***************\n"
            f.write "*** #{table}.csv \n"
            f.write "\n"
            this_diff.each { |i| f.write i }
            f.write "\n\n"
          end
        end
        puts "-- backup_table(\"#{table}\")\n   -> #{'%.4f' % (t_benchmark/1000)}s\n" if !config.silent
      end
    end

    File.delete tmp_origin_zip
    File.delete("#{config.backup_folder}/#{nvf}") if is_empty
  end

  # keep max versions
  version_files = Dir.glob("#{config.backup_folder}/#{config.version_prefix}*").sort
  if config.max_versions < version_files.length
    # throw files to garbage
    tmp_files << Dir.glob("#{config.backup_folder}/#{config.zip_prefix}*").first
    tmp_files << version_files.first

    File.rename "#{config.backup_folder}/#{compact_original(1)}", dup_file("#{config.backup_folder}/#{config.zip_prefix}_#{Time.now.strftime(config.date_format)}.zip")
  end

  # delete temporary files before method exit
  rescue => e
    @method_error = e
  ensure
    unlock if locked_by_this_method
    tmp_files.each { |i| File.delete(i) rescue nil } if tmp_files.present?
    raise @method_error if @method_error.present?
    return true
end
compact_all() click to toggle source

Merge into the .zip file and delete all the .diff files to clear the space.

# File lib/tiny_backup.rb, line 195
def compact_all
  lock # if the resource is locked, we skip to ensure block
  locked_by_this_method = true

  tmp_files = []
  tmp_files += Dir.glob("#{config.backup_folder}/#{config.zip_prefix}*")
  tmp_files += Dir.glob("#{config.backup_folder}/#{config.version_prefix}*")

  # make the temporary zip be the original and apply the updated_at time-stamp
  File.rename "#{config.backup_folder}/#{compact_original(:all)}", dup_file("#{config.backup_folder}/#{config.zip_prefix}_#{Time.now.strftime(config.date_format)}.zip")

  # delete temporary files before method exit
  rescue => e
    @method_error = e
  ensure
    unlock if locked_by_this_method
    tmp_files.each { |i| File.delete(i) rescue nil } if tmp_files.present?
    raise @method_error if @method_error.present?
    return true
end
config() click to toggle source

Global settings for TinyBackup

# File lib/configure.rb, line 8
def config
  @config ||= Config.new
end
restore_db(version_number, just_temporary=true, with_backup=true) click to toggle source

Change the database to match the selected integer version_number. @param version_number [Integer or Symbol] can be :all to restore all backup data or 0 to restore only the data collected in the .zip file @param just_temporary [Boolean] if is false, the unused version files will be deleted and the latest backup version will be synchronized with the database data. @param with_backup [Boolean] if is false, the attempt to backup not saved data is canceled

# File lib/tiny_backup.rb, line 220
def restore_db version_number, just_temporary=true, with_backup=true
  # do this before deleting the database and lose data
  backup_now if just_temporary && with_backup

  lock # if the resource is locked, we skip to ensure block
  locked_by_this_method = true

  tmp_files = []

  if just_temporary && version_number != :all
    puts "you want to restore just temporary: DO NOT start a backup BEFORE calling TinyBackup.restore_db(:all)\n" if !config.silent
  end

  version_files = Dir.glob("#{config.backup_folder}/#{config.version_prefix}*")
  if version_number == :all
    version_count = version_files.length
  else
    good_versions = version_files.find_all { |i| i.gsub("#{config.backup_folder}/#{config.version_prefix}", "").split("_").first.to_i <= version_number.to_i }
    version_count = good_versions.length
    tmp_files += version_files - good_versions if !just_temporary
  end

  tmp_origin_zip = compact_original version_count
  tmp_files << "#{config.backup_folder}/#{tmp_origin_zip}"
  tmp_files << "#{config.backup_folder}/schema_tmp.rb"

  db_name = Rails.configuration.database_configuration[Rails.env]["database"]
  db_collation = ActiveRecord::Base.connection.collation
  ActiveRecord::Base.connection.drop_database   db_name
  ActiveRecord::Base.connection.create_database db_name, collation: db_collation
  ActiveRecord::Base.connection.reconnect!

  # prepare the structure
  ZIPLIB.open("#{config.backup_folder}/#{tmp_origin_zip}") do |zf|
    zf.entries.each do |zf_entry|
      if zf_entry.name == "schema.rb"
        File.open("#{config.backup_folder}/schema_tmp.rb", "wb") { |f| f.write zf.read(zf_entry.name) }
        break
      end
    end
  end
  schema_verbose = ActiveRecord::Schema.verbose
  ActiveRecord::Schema.verbose = !config.silent
  ActiveRecord::Schema.load("#{config.backup_folder}/schema_tmp.rb")
  ActiveRecord::Schema.verbose = schema_verbose

  # add the data
  ZIPLIB.open("#{config.backup_folder}/#{tmp_origin_zip}") do |zf|
    zf.entries.each do |zf_entry|
      next if zf_entry.name == "schema.rb"
      table_rows = zf.read(zf_entry.name).split("\n")
      table_header = table_rows.shift
      table_name = zf_entry.name.split(".").first

      t_benchmark = Benchmark.ms do
        table_rows.in_groups_of(config.per_page, false) do |tr_group|
          ActiveRecord::Base.connection.execute insert_row(table_name, table_header, tr_group)
        end
      end
      puts "-- insert_data(\"#{table_name}\")\n   -> #{'%.4f' % (t_benchmark/1000)}s\n" if !config.silent
    end
  end

  # delete temporary files before method exit
  rescue => e
    @method_error = e
  ensure
    unlock if locked_by_this_method
    tmp_files.each { |i| File.delete(i) rescue nil } if tmp_files.present?
    raise @method_error if @method_error.present?
    return true
end

Private Instance Methods

add_query(values) click to toggle source

@param values [Array] csv row @return [String] of parsed csv row to be inserted into database

# File lib/tiny_backup.rb, line 514
def add_query values
  values.map do |val|
    if val.nil?
      "NULL"
    elsif val.is_a?(Date) || val.is_a?(DateTime) || val.is_a?(Time)
      "\"#{val.strftime('%Y-%m-%d %H:%M:%S')}\"" # this should be a DB recognized format
    elsif val.is_a?(Integer) || val.is_a?(Float) || val.is_a?(BigDecimal)
      val
    else
      val.inspect
    end
  end.join(",") + "\n"
end
compact_original(versions) click to toggle source

Merge X versions starting with the lowest version_number into a temporary .zip file that looks like the original zip. @param versions [Integer or Symbol] can be :all to compact all the versions and delete all the .diff files @return [String] the path of the temporary zip file

# File lib/tiny_backup.rb, line 309
def compact_original versions
  versions = Dir.glob("#{config.backup_folder}/*").length if versions == :all
  tmp_files = []
  tmp_filename = "tmp_#{config.zip_prefix}_#{Time.now.to_i}.zip"
  origin_zip = Dir.glob("#{config.backup_folder}/#{config.zip_prefix}*").first

  # add all files from original zip to a big hash
  zip_files = {}
  ZIPLIB.open(origin_zip) do |zf|
    zf.entries.each { |zf_entry| zip_files[zf_entry.name] = zf.read(zf_entry.name) }
  end

  # modify the hash on every version
  Dir.glob("#{config.backup_folder}/#{config.version_prefix}*").sort.first(versions).each do |version_file|
    diff_hash = prepare_diff version_file

    zip_files.each do |k, v|
      next if diff_hash[k].nil?

      if zip_files[k].nil?
        diff_hash.delete k
        next
      end

      diff_hash[k].find_all { |i| i[0] == "<" && i.include?("create_table") }.each do |deleted_table|
        if zip_files["#{deleted_table.split("\"")[1]}.csv"].present?
          zip_files["#{deleted_table.split("\"")[1]}.csv"] = nil # using delete will result in a stack level too deep
        end
      end if k == "schema.rb"

      zip_files[k] = update_file v, diff_hash[k]

      diff_hash.delete k
    end

    diff_hash.each { |k, v| zip_files[k] = update_file("", v) }
  end

  # save the big hash
  ZIPLIB.open("#{config.backup_folder}/#{tmp_filename}", ZIPLIB::CREATE) do |f|
    zip_files.each do |k, v|
      f.get_output_stream(k) do |ff|
        ff.write v
        tmp_files << ff if ZIPOLD
      end if v != "\n"
    end
  end

  # delete temporary files before method exit
  rescue => e
    @method_error = e
  ensure
    tmp_files.each { |i| File.delete(i) rescue nil } if tmp_files.present?
    raise @method_error if @method_error.present?
    return tmp_filename
end
diff_files(file1, file2) click to toggle source

Two temporary files will be created, but deleted shortly after the operation is finished @param file1 [String] contents of the file @param file2 [String] contents of the file @eturn [Array] that contains lines of the diff between file1 and file2

# File lib/tiny_backup.rb, line 482
def diff_files file1, file2
  diff_filename = "#{config.backup_folder}/diff"

  File.open("#{diff_filename}1", "wb") { |f| f.write file1 }
  File.open("#{diff_filename}2", "wb") { |f| f.write file2 }

  diff_lines = []
  IO.popen("diff #{diff_filename}1 #{diff_filename}2").each { |diff_line| diff_lines << diff_line }
  # TODO: use something that doesn't depend on the operating system(that can work on Windows too)

  File.delete("#{diff_filename}1")
  File.delete("#{diff_filename}2")

  diff_lines
end
diff_operation(operation, sign) click to toggle source

@param operation [String] operation as seen in the .diff file @param sign [String] one of these values “a”, “c” or “d” meaning add, change or delete operation @return [Array] of two Integer values of the range interval in the diff operation

# File lib/tiny_backup.rb, line 462
def diff_operation operation, sign
  val1, val2 = operation.split sign
  return diff_range(val1), diff_range(val2)
end
diff_range(value) click to toggle source

@param value [String] a single line number like “3” or a range of lines separated by , like “3,6” @return [Array] that contains the starting position of the change and how many operations it will take

# File lib/tiny_backup.rb, line 469
def diff_range value
  if value.include? ","
    l, r = value.split(",")
    [l.to_i - 1, r.to_i - l.to_i + 1]
  else
    [value.to_i - 1, 1]
  end
end
dup_file(filename) click to toggle source

It happens if there are two files with the same date(after using the date_format). Does not create a file! @param filename [String] @return [String] a new filename that will fix the duplicated filename issue by appending an index

# File lib/tiny_backup.rb, line 390
def dup_file filename
  return filename unless File.exist? filename
  index = 1
  name, ext = filename.split(".")

  loop do
    dup_filename = "#{name}_#{index}.#{ext}"
    return dup_filename unless File.exist? dup_filename
    index += 1
  end
end
insert_row(table_name, table_header, table_rows) click to toggle source

Create an SQL INSERT query. @param table_name [String] @param table_header [String] contains column names delimited by “ and separated by , (like '”col1“, ”col2“, ”col3“') @param table_rows [Array] each row is a String with the same format as table_header @return [String] the query used to insert a row of data

# File lib/tiny_backup.rb, line 301
def insert_row(table_name, table_header, table_rows)
  "INSERT INTO #{table_name} (#{table_header.gsub("\"", "`")}) VALUES " +
    table_rows.map { |i| "(#{i})" }.join(",")
end
lock() click to toggle source

Lock the library operations and stop the ActiveRecord logger

# File lib/tiny_backup.rb, line 529
def lock
  @logger = ActiveRecord::Base.logger
  ActiveRecord::Base.logger = nil

  if File.exist?("#{config.backup_folder}/.lock")
    raise "Another operation is running. More info you can find in the '#{config.backup_folder}/.lock' file"
  else
    File.open("#{config.backup_folder}/.lock", "wb") do |f|
      f.write caller.join("\n") + "\n"
    end
  end
end
new_version_filename() click to toggle source

@return [String] of the current file created by backup_now function

# File lib/tiny_backup.rb, line 499
def new_version_filename
  backup_folder_files =  Dir.glob("#{config.backup_folder}/*").map { |i| i.gsub("#{config.backup_folder}/", "") }

  zip_file = backup_folder_files.find { |i| i.starts_with?(config.zip_prefix) && i.ends_with?("zip") }
  return "#{config.zip_prefix}_#{Time.now.strftime(config.date_format)}.zip" if zip_file.nil?

  version_files = backup_folder_files.find_all { |i| i.starts_with?(config.version_prefix) && i.ends_with?("diff") }.sort
  return "#{config.version_prefix}1_#{Time.now.strftime(config.date_format)}.diff" if version_files.blank?

  last_version = version_files.last.gsub(config.version_prefix, "").split("_").first.to_i
  return "#{config.version_prefix}#{last_version + 1}_#{Time.now.strftime(config.date_format)}.diff"
end
prepare_diff(diff_path) click to toggle source

@param diff_path [String] @return [Hash] of the parsed .diff file where each key is a file to be changed and the value is an Array or String of the .diff file lines

# File lib/tiny_backup.rb, line 368
def prepare_diff diff_path
  diff_hash = {}

  current_key = nil
  File.read(diff_path).split("\n").each do |diff_line|
    if diff_line.starts_with? "***************"
      current_key = nil
    elsif diff_line.starts_with? "*** "
      current_key = diff_line.gsub("*", "").strip
    elsif current_key.present?
      diff_hash[current_key] ||= []
      diff_hash[current_key] << diff_line
    end
  end

  diff_hash
end
unlock() click to toggle source

Unlock the library operations and restart ActiveRecord logger

# File lib/tiny_backup.rb, line 543
def unlock
  ActiveRecord::Base.logger = @logger

  File.delete("#{config.backup_folder}/.lock") if File.exist?("#{config.backup_folder}/.lock")
end
update_file(file, diff_lines) click to toggle source

@param file [String] content of the original file @param diff_lines [Array] each item is a line String of the diff file @return [String] of the updated file after applying the diff

# File lib/tiny_backup.rb, line 405
def update_file file, diff_lines
  file = file.split "\n"
  current_operation = nil
  offset = 0

  diff_lines.each_with_index do |diff_line, index|
    if diff_line[0] != "<" && diff_line[0] != ">" && (diff_line.include?("a") || diff_line.include?("c") || diff_line.include?("d"))
      # check linux diff ooutput to understand this
      current_operation = diff_line

      if current_operation.include? "a"
        # Addition operation
        l, r = diff_operation current_operation, "a"

        insert_lines = []
        loop do
          break if diff_lines[index].nil? || (diff_lines[index][0] != ">" && insert_lines.present?)
          insert_lines << diff_lines[index][2..-1]  if diff_lines[index][0] == ">"
          index += 1
        end

        file.insert offset + l.first + 1, *insert_lines
        offset += [*insert_lines].length

      elsif current_operation.include? "c"
        # Changing operation
        f, t = diff_operation current_operation, "c"

        insert_lines = []
        loop do
          break if diff_lines[index].nil? || (diff_lines[index][0] != ">" && insert_lines.present?)
          insert_lines << diff_lines[index][2..-1]  if diff_lines[index][0] == ">"
          index += 1
        end

        f.last.times { file.delete_at offset + f.first }
        file.insert offset + f.first, *insert_lines
        offset += [*insert_lines].length
        offset -= f.last

      elsif current_operation.include? "d"
        # Deletion operation
        r, l = diff_operation current_operation, "d"

        r.last.times { file.delete_at offset + r.first }
        offset -= r.last
      end
    end
  end

  file = file.map { |i| i.to_s.force_encoding("ASCII-8BIT") } # I HATE ENCODING
  file.join("\n")  + "\n"
end