module TinyBackup
Constants
- VERSION
Public Class Methods
Set global settings for TinyBackup
like TinyBackup.configure
{|config| config.max_versions = 100 }
# File lib/configure.rb, line 3 def self.configure &block yield @config ||= Config.new end
Public Instance Methods
Create a backup of the current database and choose automatically to create a .zip or .diff file.
# File lib/tiny_backup.rb, line 19 def backup_now # if the resource is locked, we skip to ensure block lock locked_by_this_method = true tmp_files = [] nvf = new_version_filename stream = StringIO.new ActiveRecord::SchemaDumper.dump(ActiveRecord::Base.connection, stream) schema_rb = stream.string if nvf.split(".").last == "zip" # there is no .zip file so we must create one and add schema.rb and .csv file for each table # TODO: use a much better compression like Zlib::BEST_COMPRESSION to reduce the zip size, but this will consume processing power ZIPLIB.open("#{config.backup_folder}/#{nvf}", ZIPLIB::CREATE) do |f| t_benchmark = Benchmark.ms do f.get_output_stream("schema.rb") do |ff| ff.write schema_rb tmp_files << ff if ZIPOLD end end puts "-- backup_schema\n -> #{'%.4f' % (t_benchmark/1000)}s\n" if !config.silent ActiveRecord::Base.connection.tables.each do |table| next if table == "schema_migrations" query_count = ActiveRecord::Base.connection.execute("SELECT COUNT(*) FROM #{table}").first.first t_benchmark = Benchmark.ms do if query_count > 0 rows = [] query_index = 0 loop do break if query_index >= query_count query = ActiveRecord::Base.connection.execute("SELECT * FROM #{table} LIMIT #{config.per_page} OFFSET #{query_index}") rows << add_query(query.fields) if query_index == 0 query.each { |row| rows << add_query(row) } query_index += config.per_page end f.get_output_stream("#{table}.csv") do |ff| ff.write rows.join tmp_files << ff if ZIPOLD end end end puts "-- backup_table(\"#{table}\")\n -> #{'%.4f' % (t_benchmark/1000)}s\n" if !config.silent end end else # a new .diff file is created with the diff between origin_zip and tmp_origin_zip(made by merging all the versions into origin) tmp_origin_zip = "#{config.backup_folder}/#{compact_original(:all)}" tables = ActiveRecord::Base.connection.tables is_empty = true File.open("#{config.backup_folder}/#{nvf}", "wb") do |f| ZIPLIB.open(tmp_origin_zip) do |zf| zf.entries.each do |zf_entry| if zf_entry.name == "schema.rb" t_benchmark = Benchmark.ms do tables.delete "schema_migrations" this_diff = diff_files zf.read(zf_entry.name), schema_rb if this_diff.present? is_empty = false f.write "***************\n" f.write "*** schema.rb \n" f.write "\n" this_diff.each { |i| f.write i } f.write "\n\n" end end puts "-- backup_schema\n -> #{'%.4f' % (t_benchmark/1000)}s\n" if !config.silent else table = zf_entry.name.split(".").first tables.delete table begin query_count = ActiveRecord::Base.connection.execute("SELECT COUNT(*) FROM #{table}").first.first rescue ActiveRecord::StatementInvalid next end rows = [] query_index = 0 t_benchmark = Benchmark.ms do loop do break if query_index >= query_count query = ActiveRecord::Base.connection.execute("SELECT * FROM #{table} LIMIT #{config.per_page} OFFSET #{query_index}") rows << add_query(query.fields) if query_index == 0 query.each { |row| rows << add_query(row) } query_index += config.per_page end this_diff = diff_files zf.read(zf_entry.name), rows.join if this_diff.present? is_empty = false f.write "***************\n" f.write "*** #{zf_entry.name} \n" f.write "\n" this_diff.each { |i| f.write i } f.write "\n\n" end end puts "-- backup_table(\"#{table}\")\n -> #{'%.4f' % (t_benchmark/1000)}s\n" if !config.silent end end end # tables that are created recently and doesn't have a .csv file in the tmp_origin_zip tables.each do |table| begin query_count = ActiveRecord::Base.connection.execute("SELECT COUNT(*) FROM #{table}").first.first rescue ActiveRecord::StatementInvalid next end rows = [] query_index = 0 t_benchmark = Benchmark.ms do loop do break if query_index >= query_count query = ActiveRecord::Base.connection.execute("SELECT * FROM #{table} LIMIT #{config.per_page} OFFSET #{query_index}") rows << add_query(query.fields) if query_index == 0 query.each { |row| rows << add_query(row) } query_index += config.per_page end this_diff = diff_files "", rows.join if this_diff.present? is_empty = false f.write "***************\n" f.write "*** #{table}.csv \n" f.write "\n" this_diff.each { |i| f.write i } f.write "\n\n" end end puts "-- backup_table(\"#{table}\")\n -> #{'%.4f' % (t_benchmark/1000)}s\n" if !config.silent end end File.delete tmp_origin_zip File.delete("#{config.backup_folder}/#{nvf}") if is_empty end # keep max versions version_files = Dir.glob("#{config.backup_folder}/#{config.version_prefix}*").sort if config.max_versions < version_files.length # throw files to garbage tmp_files << Dir.glob("#{config.backup_folder}/#{config.zip_prefix}*").first tmp_files << version_files.first File.rename "#{config.backup_folder}/#{compact_original(1)}", dup_file("#{config.backup_folder}/#{config.zip_prefix}_#{Time.now.strftime(config.date_format)}.zip") end # delete temporary files before method exit rescue => e @method_error = e ensure unlock if locked_by_this_method tmp_files.each { |i| File.delete(i) rescue nil } if tmp_files.present? raise @method_error if @method_error.present? return true end
Merge into the .zip file and delete all the .diff files to clear the space.
# File lib/tiny_backup.rb, line 195 def compact_all lock # if the resource is locked, we skip to ensure block locked_by_this_method = true tmp_files = [] tmp_files += Dir.glob("#{config.backup_folder}/#{config.zip_prefix}*") tmp_files += Dir.glob("#{config.backup_folder}/#{config.version_prefix}*") # make the temporary zip be the original and apply the updated_at time-stamp File.rename "#{config.backup_folder}/#{compact_original(:all)}", dup_file("#{config.backup_folder}/#{config.zip_prefix}_#{Time.now.strftime(config.date_format)}.zip") # delete temporary files before method exit rescue => e @method_error = e ensure unlock if locked_by_this_method tmp_files.each { |i| File.delete(i) rescue nil } if tmp_files.present? raise @method_error if @method_error.present? return true end
Global settings for TinyBackup
# File lib/configure.rb, line 8 def config @config ||= Config.new end
Change the database to match the selected integer version_number
. @param version_number [Integer or Symbol] can be :all to restore all backup data or 0 to restore only the data collected in the .zip file @param just_temporary [Boolean] if is false, the unused version files will be deleted and the latest backup version will be synchronized with the database data. @param with_backup [Boolean] if is false, the attempt to backup not saved data is canceled
# File lib/tiny_backup.rb, line 220 def restore_db version_number, just_temporary=true, with_backup=true # do this before deleting the database and lose data backup_now if just_temporary && with_backup lock # if the resource is locked, we skip to ensure block locked_by_this_method = true tmp_files = [] if just_temporary && version_number != :all puts "you want to restore just temporary: DO NOT start a backup BEFORE calling TinyBackup.restore_db(:all)\n" if !config.silent end version_files = Dir.glob("#{config.backup_folder}/#{config.version_prefix}*") if version_number == :all version_count = version_files.length else good_versions = version_files.find_all { |i| i.gsub("#{config.backup_folder}/#{config.version_prefix}", "").split("_").first.to_i <= version_number.to_i } version_count = good_versions.length tmp_files += version_files - good_versions if !just_temporary end tmp_origin_zip = compact_original version_count tmp_files << "#{config.backup_folder}/#{tmp_origin_zip}" tmp_files << "#{config.backup_folder}/schema_tmp.rb" db_name = Rails.configuration.database_configuration[Rails.env]["database"] db_collation = ActiveRecord::Base.connection.collation ActiveRecord::Base.connection.drop_database db_name ActiveRecord::Base.connection.create_database db_name, collation: db_collation ActiveRecord::Base.connection.reconnect! # prepare the structure ZIPLIB.open("#{config.backup_folder}/#{tmp_origin_zip}") do |zf| zf.entries.each do |zf_entry| if zf_entry.name == "schema.rb" File.open("#{config.backup_folder}/schema_tmp.rb", "wb") { |f| f.write zf.read(zf_entry.name) } break end end end schema_verbose = ActiveRecord::Schema.verbose ActiveRecord::Schema.verbose = !config.silent ActiveRecord::Schema.load("#{config.backup_folder}/schema_tmp.rb") ActiveRecord::Schema.verbose = schema_verbose # add the data ZIPLIB.open("#{config.backup_folder}/#{tmp_origin_zip}") do |zf| zf.entries.each do |zf_entry| next if zf_entry.name == "schema.rb" table_rows = zf.read(zf_entry.name).split("\n") table_header = table_rows.shift table_name = zf_entry.name.split(".").first t_benchmark = Benchmark.ms do table_rows.in_groups_of(config.per_page, false) do |tr_group| ActiveRecord::Base.connection.execute insert_row(table_name, table_header, tr_group) end end puts "-- insert_data(\"#{table_name}\")\n -> #{'%.4f' % (t_benchmark/1000)}s\n" if !config.silent end end # delete temporary files before method exit rescue => e @method_error = e ensure unlock if locked_by_this_method tmp_files.each { |i| File.delete(i) rescue nil } if tmp_files.present? raise @method_error if @method_error.present? return true end
Private Instance Methods
@param values [Array] csv row @return [String] of parsed csv row to be inserted into database
# File lib/tiny_backup.rb, line 514 def add_query values values.map do |val| if val.nil? "NULL" elsif val.is_a?(Date) || val.is_a?(DateTime) || val.is_a?(Time) "\"#{val.strftime('%Y-%m-%d %H:%M:%S')}\"" # this should be a DB recognized format elsif val.is_a?(Integer) || val.is_a?(Float) || val.is_a?(BigDecimal) val else val.inspect end end.join(",") + "\n" end
Merge X versions starting with the lowest version_number
into a temporary .zip file that looks like the original zip. @param versions [Integer or Symbol] can be :all to compact all the versions and delete all the .diff files @return [String] the path of the temporary zip file
# File lib/tiny_backup.rb, line 309 def compact_original versions versions = Dir.glob("#{config.backup_folder}/*").length if versions == :all tmp_files = [] tmp_filename = "tmp_#{config.zip_prefix}_#{Time.now.to_i}.zip" origin_zip = Dir.glob("#{config.backup_folder}/#{config.zip_prefix}*").first # add all files from original zip to a big hash zip_files = {} ZIPLIB.open(origin_zip) do |zf| zf.entries.each { |zf_entry| zip_files[zf_entry.name] = zf.read(zf_entry.name) } end # modify the hash on every version Dir.glob("#{config.backup_folder}/#{config.version_prefix}*").sort.first(versions).each do |version_file| diff_hash = prepare_diff version_file zip_files.each do |k, v| next if diff_hash[k].nil? if zip_files[k].nil? diff_hash.delete k next end diff_hash[k].find_all { |i| i[0] == "<" && i.include?("create_table") }.each do |deleted_table| if zip_files["#{deleted_table.split("\"")[1]}.csv"].present? zip_files["#{deleted_table.split("\"")[1]}.csv"] = nil # using delete will result in a stack level too deep end end if k == "schema.rb" zip_files[k] = update_file v, diff_hash[k] diff_hash.delete k end diff_hash.each { |k, v| zip_files[k] = update_file("", v) } end # save the big hash ZIPLIB.open("#{config.backup_folder}/#{tmp_filename}", ZIPLIB::CREATE) do |f| zip_files.each do |k, v| f.get_output_stream(k) do |ff| ff.write v tmp_files << ff if ZIPOLD end if v != "\n" end end # delete temporary files before method exit rescue => e @method_error = e ensure tmp_files.each { |i| File.delete(i) rescue nil } if tmp_files.present? raise @method_error if @method_error.present? return tmp_filename end
Two temporary files will be created, but deleted shortly after the operation is finished @param file1 [String] contents of the file @param file2 [String] contents of the file @eturn [Array] that contains lines of the diff between file1
and file2
# File lib/tiny_backup.rb, line 482 def diff_files file1, file2 diff_filename = "#{config.backup_folder}/diff" File.open("#{diff_filename}1", "wb") { |f| f.write file1 } File.open("#{diff_filename}2", "wb") { |f| f.write file2 } diff_lines = [] IO.popen("diff #{diff_filename}1 #{diff_filename}2").each { |diff_line| diff_lines << diff_line } # TODO: use something that doesn't depend on the operating system(that can work on Windows too) File.delete("#{diff_filename}1") File.delete("#{diff_filename}2") diff_lines end
@param operation [String] operation as seen in the .diff file @param sign [String] one of these values “a”, “c” or “d” meaning add, change or delete operation @return [Array] of two Integer values of the range interval in the diff operation
# File lib/tiny_backup.rb, line 462 def diff_operation operation, sign val1, val2 = operation.split sign return diff_range(val1), diff_range(val2) end
@param value [String] a single line number like “3” or a range of lines separated by , like “3,6” @return [Array] that contains the starting position of the change and how many operations it will take
# File lib/tiny_backup.rb, line 469 def diff_range value if value.include? "," l, r = value.split(",") [l.to_i - 1, r.to_i - l.to_i + 1] else [value.to_i - 1, 1] end end
It happens if there are two files with the same date(after using the date_format). Does not create a file! @param filename [String] @return [String] a new filename that will fix the duplicated filename issue by appending an index
# File lib/tiny_backup.rb, line 390 def dup_file filename return filename unless File.exist? filename index = 1 name, ext = filename.split(".") loop do dup_filename = "#{name}_#{index}.#{ext}" return dup_filename unless File.exist? dup_filename index += 1 end end
Create an SQL INSERT query. @param table_name [String] @param table_header [String] contains column names delimited by “ and separated by , (like '”col1“, ”col2“, ”col3“') @param table_rows [Array] each row is a String with the same format as table_header @return [String] the query used to insert a row of data
# File lib/tiny_backup.rb, line 301 def insert_row(table_name, table_header, table_rows) "INSERT INTO #{table_name} (#{table_header.gsub("\"", "`")}) VALUES " + table_rows.map { |i| "(#{i})" }.join(",") end
Lock the library operations and stop the ActiveRecord logger
# File lib/tiny_backup.rb, line 529 def lock @logger = ActiveRecord::Base.logger ActiveRecord::Base.logger = nil if File.exist?("#{config.backup_folder}/.lock") raise "Another operation is running. More info you can find in the '#{config.backup_folder}/.lock' file" else File.open("#{config.backup_folder}/.lock", "wb") do |f| f.write caller.join("\n") + "\n" end end end
@return [String] of the current file created by backup_now
function
# File lib/tiny_backup.rb, line 499 def new_version_filename backup_folder_files = Dir.glob("#{config.backup_folder}/*").map { |i| i.gsub("#{config.backup_folder}/", "") } zip_file = backup_folder_files.find { |i| i.starts_with?(config.zip_prefix) && i.ends_with?("zip") } return "#{config.zip_prefix}_#{Time.now.strftime(config.date_format)}.zip" if zip_file.nil? version_files = backup_folder_files.find_all { |i| i.starts_with?(config.version_prefix) && i.ends_with?("diff") }.sort return "#{config.version_prefix}1_#{Time.now.strftime(config.date_format)}.diff" if version_files.blank? last_version = version_files.last.gsub(config.version_prefix, "").split("_").first.to_i return "#{config.version_prefix}#{last_version + 1}_#{Time.now.strftime(config.date_format)}.diff" end
@param diff_path [String] @return [Hash] of the parsed .diff file where each key is a file to be changed and the value is an Array or String of the .diff file lines
# File lib/tiny_backup.rb, line 368 def prepare_diff diff_path diff_hash = {} current_key = nil File.read(diff_path).split("\n").each do |diff_line| if diff_line.starts_with? "***************" current_key = nil elsif diff_line.starts_with? "*** " current_key = diff_line.gsub("*", "").strip elsif current_key.present? diff_hash[current_key] ||= [] diff_hash[current_key] << diff_line end end diff_hash end
Unlock the library operations and restart ActiveRecord logger
# File lib/tiny_backup.rb, line 543 def unlock ActiveRecord::Base.logger = @logger File.delete("#{config.backup_folder}/.lock") if File.exist?("#{config.backup_folder}/.lock") end
@param file [String] content of the original file @param diff_lines [Array] each item is a line String of the diff file @return [String] of the updated file after applying the diff
# File lib/tiny_backup.rb, line 405 def update_file file, diff_lines file = file.split "\n" current_operation = nil offset = 0 diff_lines.each_with_index do |diff_line, index| if diff_line[0] != "<" && diff_line[0] != ">" && (diff_line.include?("a") || diff_line.include?("c") || diff_line.include?("d")) # check linux diff ooutput to understand this current_operation = diff_line if current_operation.include? "a" # Addition operation l, r = diff_operation current_operation, "a" insert_lines = [] loop do break if diff_lines[index].nil? || (diff_lines[index][0] != ">" && insert_lines.present?) insert_lines << diff_lines[index][2..-1] if diff_lines[index][0] == ">" index += 1 end file.insert offset + l.first + 1, *insert_lines offset += [*insert_lines].length elsif current_operation.include? "c" # Changing operation f, t = diff_operation current_operation, "c" insert_lines = [] loop do break if diff_lines[index].nil? || (diff_lines[index][0] != ">" && insert_lines.present?) insert_lines << diff_lines[index][2..-1] if diff_lines[index][0] == ">" index += 1 end f.last.times { file.delete_at offset + f.first } file.insert offset + f.first, *insert_lines offset += [*insert_lines].length offset -= f.last elsif current_operation.include? "d" # Deletion operation r, l = diff_operation current_operation, "d" r.last.times { file.delete_at offset + r.first } offset -= r.last end end end file = file.map { |i| i.to_s.force_encoding("ASCII-8BIT") } # I HATE ENCODING file.join("\n") + "\n" end