class Sp2db::BaseTable
Attributes
client[RW]
find_columns[RW]
name[RW]
sheet_name[RW]
spreadsheet_id[RW]
worksheet[RW]
Public Class Methods
new(opts={})
click to toggle source
# File lib/sp2db/base_table.rb, line 11 def initialize opts={} if opts[:name].blank? && opts[:sheet_name].blank? raise "Must specify at least one of name or sheet name" end opts.each do |k, v| self.send "#{k}=", v end self.sheet_name ||= opts[:sheet_name] = config[:sheet_name] || worksheet.try(:title) end
Protected Class Methods
all_tables()
click to toggle source
# File lib/sp2db/base_table.rb, line 219 def all_tables ModelTable.all_tables + NonModelTable.all_tables end
model_table_class()
click to toggle source
# File lib/sp2db/base_table.rb, line 239 def model_table_class ModelTable end
sp_to_csv(*table_names)
click to toggle source
# File lib/sp2db/base_table.rb, line 235 def sp_to_csv *table_names table_by_names(*table_names).map(&__method__) end
table_by_names(*names)
click to toggle source
# File lib/sp2db/base_table.rb, line 224 def table_by_names *names all_tables = self.all_tables if names.blank? all_tables else names.map do |n| all_tables.find {|tb| tb.name == n.to_sym} || raise("Not found: #{n}") end end end
Public Instance Methods
active_record?()
click to toggle source
# File lib/sp2db/base_table.rb, line 24 def active_record? false end
config()
click to toggle source
Global config
# File lib/sp2db/base_table.rb, line 128 def config {}.with_indifferent_access end
csv_data()
click to toggle source
# File lib/sp2db/base_table.rb, line 80 def csv_data raw_data = CSV.parse File.open(csv_file) data = process_data raw_data, source: :csv data end
csv_file()
click to toggle source
# File lib/sp2db/base_table.rb, line 97 def csv_file "#{csv_folder}/#{name}.csv" end
csv_folder()
click to toggle source
# File lib/sp2db/base_table.rb, line 91 def csv_folder folder = "#{Sp2db.config.export_location}/csv" FileUtils.mkdir_p folder folder end
data_transform(raw_data, opts={})
click to toggle source
Tranform data to standard csv format
# File lib/sp2db/base_table.rb, line 141 def data_transform raw_data, opts={} if config[:data_transform].present? config[:data_transform].call *args, &block else raw_data end end
header_row()
click to toggle source
# File lib/sp2db/base_table.rb, line 86 def header_row # @header_row ||= config[:header_row] || 0 0 end
name=(n)
click to toggle source
# File lib/sp2db/base_table.rb, line 37 def name=n @name = n&.to_sym end
process_data(raw_data, opts={})
click to toggle source
# File lib/sp2db/base_table.rb, line 132 def process_data raw_data, opts={} raw_data = data_transform raw_data, opts unless opts[:skip_data_transform] raw_data = raw_filter raw_data, opts unless opts[:skip_data_filter] data = call_process_data raw_data, opts data end
required_columns()
click to toggle source
# File lib/sp2db/base_table.rb, line 45 def required_columns @required_columns ||= config[:required_columns] || [] end
sp_data()
click to toggle source
# File lib/sp2db/base_table.rb, line 65 def sp_data retries = 2 begin raw_data = CSV.parse worksheet.export_as_string rescue Google::Apis::RateLimitError => e retries -= 1 sleep(5) retry if retries >= 0 raise e end data = process_data raw_data, source: :sp data end
sp_to_csv(opts={})
click to toggle source
# File lib/sp2db/base_table.rb, line 101 def sp_to_csv opts={} write_csv to_csv(sp_data) end
spreadsheet()
click to toggle source
# File lib/sp2db/base_table.rb, line 53 def spreadsheet client.spreadsheet spreadsheet_id end
to_csv(data)
click to toggle source
Array of hash data to csv format
# File lib/sp2db/base_table.rb, line 113 def to_csv data attributes = data.first&.keys || [] CSV.generate(headers: true) do |csv| csv << attributes data.each do |row| csv << attributes.map do |att| row[att] end end end end
write_csv(data)
click to toggle source
# File lib/sp2db/base_table.rb, line 105 def write_csv data File.open csv_file, "wb" do |f| f.write data end csv_file end
Protected Instance Methods
call_process_data(raw_data, opts={})
click to toggle source
# File lib/sp2db/base_table.rb, line 167 def call_process_data raw_data, opts={} data = raw_data if (data_proc = config[:process_data]).present? data = data_proc.call raw_data end data end
raw_filter(raw_data, opts={})
click to toggle source
Remove uncessary columns and invalid rows from csv format data
# File lib/sp2db/base_table.rb, line 176 def raw_filter raw_data, opts={} raw_header = raw_data[header_row].map.with_index do |h, idx| is_valid = valid_header?(h) { idx: idx, is_remove: !is_valid, is_required: require_header?(h), name: is_valid && h.gsub(/\s*/, '').gsub(/!/, '').downcase } end rows = raw_data[(header_row + 1)..-1].map.with_index do |raw, rdx| row = {}.with_indifferent_access raw_header.each do |h| val = raw[h[:idx]] next if h[:is_remove] if h[:is_required] && val.blank? row = {} break end row[h[:name]] = standardize_cell_val val end next if row.values.all?(&:blank?) row[:id] = rdx + 1 if find_columns.include?(:id) && row[:id].blank? row end.compact .reject(&:blank?) rows = rows.select do |row| if required_columns.present? required_columns.all? {|col| row[col].present? } else true end end rows end
require_header?(h)
click to toggle source
Header with “!” at the beginning or ending is required
# File lib/sp2db/base_table.rb, line 156 def require_header? h h.present? && (h.match("^!.*") || h.match(".*?!$")) end
standardize_cell_val(v)
click to toggle source
Convert number string to number
# File lib/sp2db/base_table.rb, line 161 def standardize_cell_val v v = ((float = Float(v)) && (float % 1.0 == 0) ? float.to_i : float) rescue v v = v.force_encoding("UTF-8") if v.is_a?(String) v end
valid_header?(h)
click to toggle source
Remove header which starts with “#”
# File lib/sp2db/base_table.rb, line 151 def valid_header? h h.present? && !h.match("^#.*") end