class BusinessCatalyst::CSV::FileSplitter
Business Catalyst can only import 10k product rows at a time. Use this class to help split up files.
Usage¶ ↑
headers = MyRowClass.headers splitter = FileSplitter.new("output_base_name", max_rows_per_file: 10_000, header_row: headers) splitter.start do |splitter| product_rows.each do |row| splitter.start_row splitter.current_file << row.to_a end end
If you want to watch for when a new file is opened and maybe do some logging, you can use the on_file_change
method:
splitter.on_file_change do |splitter| puts "opened #{splitter.current_file.path}" end
Attributes
all_files[R]
base_name[RW]
current_file[R]
current_row[R]
file_names[R]
header_row[RW]
logger[RW]
max_rows_per_file[RW]
total_rows[R]
verbose[RW]
Public Class Methods
new(base_name, options = {})
click to toggle source
# File lib/business_catalyst/csv/file_splitter.rb, line 36 def initialize(base_name, options = {}) @base_name = base_name @max_rows_per_file = options.fetch(:max_rows_per_file) { 10_000 } @header_row = options.fetch(:header_row) { nil } @verbose = options.fetch(:verbose) { false } @logger = options.fetch(:logger) { nil } @current_row = 0 @total_rows = 0 @all_files = [] @current_file = nil end
Public Instance Methods
close()
click to toggle source
# File lib/business_catalyst/csv/file_splitter.rb, line 74 def close current_file.close if current_file end
on_file_change(&block)
click to toggle source
# File lib/business_catalyst/csv/file_splitter.rb, line 70 def on_file_change(&block) @on_file_change = Proc.new(&block) end
start() { |self| ... }
click to toggle source
# File lib/business_catalyst/csv/file_splitter.rb, line 50 def start begin increment_file(1, max_rows_per_file) # will be off by one unless we set manually yield self ensure close end rename_last_file end
start_row()
click to toggle source
Call before writing new row
# File lib/business_catalyst/csv/file_splitter.rb, line 61 def start_row @current_row += 1 @total_rows += 1 if @current_row > max_rows_per_file increment_file @current_row = 1 end end
Private Instance Methods
file_name_with_range(base_name, min, max)
click to toggle source
# File lib/business_catalyst/csv/file_splitter.rb, line 102 def file_name_with_range(base_name, min, max) "#{base_name}_#{min}-#{max}.csv" end
increment_file(min_row = total_rows, max_row = total_rows + max_rows_per_file - 1)
click to toggle source
# File lib/business_catalyst/csv/file_splitter.rb, line 80 def increment_file(min_row = total_rows, max_row = total_rows + max_rows_per_file - 1) @current_file.close if @current_file new_file_name = file_name_with_range(base_name, min_row, max_row) @all_files << new_file_name log "writing to #{new_file_name}" if verbose @current_file = ::CSV.open(new_file_name, 'wb') if header_row @current_file << header_row end trigger_on_file_change end
log(msg)
click to toggle source
# File lib/business_catalyst/csv/file_splitter.rb, line 115 def log(msg) if logger logger.info msg else puts msg end end
rename_last_file()
click to toggle source
# File lib/business_catalyst/csv/file_splitter.rb, line 106 def rename_last_file last_file = @all_files.pop if last_file new_name = last_file.sub(/-\d+\.csv\z/i, "-#{total_rows}.csv") FileUtils.mv last_file, new_name @all_files << new_name end end
trigger_on_file_change()
click to toggle source
# File lib/business_catalyst/csv/file_splitter.rb, line 92 def trigger_on_file_change if @on_file_change.kind_of?(Proc) if @on_file_change.arity >= 1 @on_file_change.call(self) else @on_file_change.call end end end