module Beam::Upload
Public Instance Methods
add_to_batch(batch, record)
click to toggle source
# File lib/beam/upload.rb, line 135 def add_to_batch(batch, record) batch << record if record if batch.size >= @batch_size import_from_batch(batch, true) batch = [] end batch end
delete_csv_if_present()
click to toggle source
deletes csv file upfront before we unzip the file
# File lib/beam/upload.rb, line 62 def delete_csv_if_present File.delete "#{@file_path}/#{@csv_file_name}" if File.exists? "#{@file_path}/#{@csv_file_name}" end
error_in_upload(rows)
click to toggle source
Creates error file with error rows, e.g. for users.csv file, it creates errors_users.csv
# File lib/beam/upload.rb, line 68 def error_in_upload(rows) csv_file_path = "#{@file_path}/errors_#{@csv_file_name}" begin CSV.open(csv_file_path, "wb") do |csv| rows.each do |row| csv << row end end rescue Exception => e Rails.logger.error e.formatted_exception("Building error file for #{@file_name} failed!") end end
import_from_batch(batch)
click to toggle source
# File lib/beam/upload.rb, line 144 def import_from_batch(batch) import(batch, :validate => false) if @batch_process end
invalid_file_message()
click to toggle source
# File lib/beam/upload.rb, line 159 def invalid_file_message "Please upload the right template and verify data before upload" end
log_and_return_error(row_hash, e, index)
click to toggle source
# File lib/beam/upload.rb, line 154 def log_and_return_error(row_hash, e, index) Rails.logger.error e.formatted_exception("Error on #{index}: \n #{row_hash.values}") row_hash.values + [invalid_file_message] end
log_and_return_validation_error(record, row_hash, index)
click to toggle source
# File lib/beam/upload.rb, line 148 def log_and_return_validation_error(record, row_hash, index) error_msg = record.errors.messages.values.join(', ') Rails.logger.error("Error on #{index}: \n #{row_hash.values + [error_msg]}") row_hash.values + [error_msg] end
parse()
click to toggle source
parses the csv file, creates record for the model and returns response hash,e.g. {:errors=>1, :status=>200, :total_rows=>4, :error_rows=>[[“Test1”, nil, “is invalid”]]} also, it creates error file to consume
# File lib/beam/upload.rb, line 95 def parse response = { errors: 0, status: 200, total_rows: 0, error_rows: []} index = 0 batch = [] begin CSV.foreach("#{@file_path}/#{@csv_file_name}", :encoding => 'iso-8859-1:UTF-8', headers: true) do |row| index += 1 row_hash = row.to_hash response[:total_rows] += 1 begin record, response[:errors], error_row = validate_record(response[:errors], row_hash, index) response[:error_rows] << error_row if error_row if @batch_process batch = add_to_batch(batch, record) else record.save! end rescue Exception => e response[:errors] +=1 response[:error_rows] << log_and_return_error(row_hash, e, index) end end import_from_batch(batch) rescue Exception => e response[:errors] = 1 response[:error_rows] = log_and_return_error({}, e, '') response[:status] = 500 end if response[:error_rows].blank? post_parse_success_calls elsif @error_file_needed error_in_upload(response[:error_rows]) end response end
post_parse_success_calls()
click to toggle source
# File lib/beam/upload.rb, line 163 def post_parse_success_calls # abstract method end
unzip_file()
click to toggle source
unzips a zipped-csv-file in a given directory
# File lib/beam/upload.rb, line 57 def unzip_file `unzip #{@file_path}/#{@file_name} -d #{@file_path}` end
upload_config()
click to toggle source
# File lib/beam/upload.rb, line 20 def upload_config @error_file_needed = Beam.config[:error_file_needed] @batch_process = Beam.config[:batch_process] @batch_size = Beam.config[:batch_size] @zipped = Beam.config[:zipped] end
upload_file(file_name, file_path, callback_method='parse')
click to toggle source
params:
file_name: zip file name, e.g. users.csv.zip file_path: path/to/zip-file, e.g. Rails.root/tmp callback_method: method which does the job to parse CSV and load records, default is parse method
# File lib/beam/upload.rb, line 31 def upload_file(file_name, file_path, callback_method='parse') status = {} upload_config @file_name = file_name @file_path = file_path @original_zip_file = "#{@file_path}/#{@file_name}" @csv_file_name = @file_name.gsub(/\.zip/,'') begin if @zipped delete_csv_if_present unzip_file end status = self.send(callback_method) rescue Exception => e error_in_upload([[invalid_file_message]]) if @error_file_needed Rails.logger.error e.formatted_exception("Uploading file #{@file_name} failed!") status = { errors: 1, status: 500} ensure File.delete @original_zip_file if File.exists?(@original_zip_file) end status end
validate_record(errors_count, row_hash, index)
click to toggle source
Validates each record and returns error count if record is invalid
# File lib/beam/upload.rb, line 82 def validate_record(errors_count, row_hash, index) record = new(row_hash) unless record.valid? errors_count += 1 [nil, errors_count, log_and_return_validation_error(record, row_hash, index)] else [record, errors_count, nil] end end