class DbSubsetter::Importer

Manages importing a subset of data

Public Class Methods

new(filename) click to toggle source
# File lib/db_subsetter/importer.rb, line 6
def initialize(filename)
  raise ArgumentError, 'invalid input file' unless File.exist?(filename)

  @data = SQLite3::Database.new(filename)
  @dialect = case ActiveRecord::Base.connection_config[:adapter]
             when 'mysql2'
               DbSubsetter::Dialect::MySQL
             when 'postgresql'
               DbSubsetter::Dialect::Postgres
             when 'sqlite3'
               DbSubsetter::Dialect::Sqlite
             when 'sqlserver'
               DbSubsetter::Dialect::MSSQL
             else
               DbSubsetter::Dialect::Generic
             end
end

Public Instance Methods

import(verbose = true) click to toggle source
# File lib/db_subsetter/importer.rb, line 32
def import(verbose = true)
  @verbose = verbose
  @dialect.import do
    tables.each do |table|
      import_table(table)
    end
  end
end
tables() click to toggle source
# File lib/db_subsetter/importer.rb, line 24
def tables
  all_tables = []
  @data.execute('SELECT name FROM tables') do |row|
    all_tables << row[0]
  end
  all_tables
end

Private Instance Methods

columns(table) click to toggle source
# File lib/db_subsetter/importer.rb, line 70
def columns(table)
  raw = @data.execute('SELECT columns FROM tables WHERE name = ?', [table]).first[0]
  JSON.parse(raw)
end
import_table(table) click to toggle source
# File lib/db_subsetter/importer.rb, line 43
def import_table(table)
  $stdout.sync
  started_at = Time.now
  print "Importing #{table}" if @verbose
  @dialect.truncate_table(table)

  ActiveRecord::Base.connection.begin_db_transaction

  all_rows = @data.execute("SELECT data FROM #{table.underscore}")
  all_rows.each_slice(@dialect::INSERT_BATCH_SIZE) do |rows|
    quoted_rows = rows.map { |row| '(' + quoted_values(row).join(',') + ')' }.join(',')
    insert_sql = "INSERT INTO #{quoted_table_name(table)} (#{quoted_column_names(table).join(',')}) VALUES #{quoted_rows}"
    ActiveRecord::Base.connection.execute(insert_sql)
    print '.' if @verbose
  end

  ActiveRecord::Base.connection.commit_db_transaction
  puts " (#{(Time.now - started_at).round(3)}s)" if @verbose
end
quoted_column_names(table) click to toggle source
# File lib/db_subsetter/importer.rb, line 79
def quoted_column_names(table)
  columns(table).map { |column| ActiveRecord::Base.connection.quote_column_name(column) }
end
quoted_table_name(table) click to toggle source
# File lib/db_subsetter/importer.rb, line 75
def quoted_table_name(table)
  ActiveRecord::Base.connection.quote_table_name(table)
end
quoted_values(row) click to toggle source
# File lib/db_subsetter/importer.rb, line 63
def quoted_values(row)
  out = JSON.parse(row[0])
  out = out.map { |x| ActiveRecord::Base.connection.type_cast(x, nil) }
  out = out.map { |x| ActiveRecord::Base.connection.quote(x) }
  out
end