class Sp2db::BaseTable

Attributes

client[RW]
find_columns[RW]
name[RW]
sheet_name[RW]
spreadsheet_id[RW]
worksheet[RW]

Public Class Methods

new(opts={}) click to toggle source
# File lib/sp2db/base_table.rb, line 11
def initialize opts={}

  if opts[:name].blank? && opts[:sheet_name].blank?
    raise "Must specify at least one of name or sheet name"
  end

  opts.each do |k, v|
    self.send "#{k}=", v
  end

  self.sheet_name ||= opts[:sheet_name] = config[:sheet_name] || worksheet.try(:title)
end

Protected Class Methods

all_tables() click to toggle source
# File lib/sp2db/base_table.rb, line 219
def all_tables
  ModelTable.all_tables + NonModelTable.all_tables
end
model_table_class() click to toggle source
# File lib/sp2db/base_table.rb, line 239
def model_table_class
  ModelTable
end
sp_to_csv(*table_names) click to toggle source
# File lib/sp2db/base_table.rb, line 235
def sp_to_csv *table_names
  table_by_names(*table_names).map(&__method__)
end
table_by_names(*names) click to toggle source
# File lib/sp2db/base_table.rb, line 224
def table_by_names *names
  all_tables = self.all_tables
  if names.blank?
    all_tables
  else
    names.map do |n|
      all_tables.find {|tb| tb.name == n.to_sym} || raise("Not found: #{n}")
    end
  end
end

Public Instance Methods

active_record?() click to toggle source
# File lib/sp2db/base_table.rb, line 24
def active_record?
  false
end
config() click to toggle source

Global config

# File lib/sp2db/base_table.rb, line 128
def config
  {}.with_indifferent_access
end
csv_data() click to toggle source
# File lib/sp2db/base_table.rb, line 80
def csv_data
  raw_data = CSV.parse File.open(csv_file)
  data = process_data raw_data, source: :csv
  data
end
csv_file() click to toggle source
# File lib/sp2db/base_table.rb, line 97
def csv_file
  "#{csv_folder}/#{name}.csv"
end
csv_folder() click to toggle source
# File lib/sp2db/base_table.rb, line 91
def csv_folder
  folder = "#{Sp2db.config.export_location}/csv"
  FileUtils.mkdir_p folder
  folder
end
data_transform(raw_data, opts={}) click to toggle source

Tranform data to standard csv format

# File lib/sp2db/base_table.rb, line 141
def data_transform raw_data, opts={}
  if config[:data_transform].present?
    config[:data_transform].call *args, &block
  else
    raw_data
  end
end
header_row() click to toggle source
# File lib/sp2db/base_table.rb, line 86
def header_row
  # @header_row ||= config[:header_row] || 0
  0
end
name=(n) click to toggle source
# File lib/sp2db/base_table.rb, line 37
def name=n
  @name = n&.to_sym
end
process_data(raw_data, opts={}) click to toggle source
# File lib/sp2db/base_table.rb, line 132
def process_data raw_data, opts={}
  raw_data = data_transform raw_data, opts unless opts[:skip_data_transform]
  raw_data = raw_filter raw_data, opts unless opts[:skip_data_filter]
  data = call_process_data raw_data, opts
  data
end
required_columns() click to toggle source
# File lib/sp2db/base_table.rb, line 45
def required_columns
  @required_columns ||= config[:required_columns] || []
end
sp_data() click to toggle source
# File lib/sp2db/base_table.rb, line 65
def sp_data
  retries = 2
  begin
    raw_data = CSV.parse worksheet.export_as_string
  rescue Google::Apis::RateLimitError => e
    retries -= 1
    sleep(5)
    retry if retries >= 0
    raise e
  end

  data = process_data raw_data, source: :sp
  data
end
sp_to_csv(opts={}) click to toggle source
# File lib/sp2db/base_table.rb, line 101
def sp_to_csv opts={}
  write_csv to_csv(sp_data)
end
spreadsheet() click to toggle source
# File lib/sp2db/base_table.rb, line 53
def spreadsheet
  client.spreadsheet spreadsheet_id
end
to_csv(data) click to toggle source

Array of hash data to csv format

# File lib/sp2db/base_table.rb, line 113
def to_csv data
  attributes = data.first&.keys || []

  CSV.generate(headers: true) do |csv|
    csv << attributes

    data.each do |row|
      csv << attributes.map do |att|
        row[att]
      end
    end
  end
end
write_csv(data) click to toggle source
# File lib/sp2db/base_table.rb, line 105
def write_csv data
  File.open csv_file, "wb" do |f|
    f.write data
  end
  csv_file
end

Protected Instance Methods

call_process_data(raw_data, opts={}) click to toggle source
# File lib/sp2db/base_table.rb, line 167
def call_process_data raw_data, opts={}
  data = raw_data
  if (data_proc = config[:process_data]).present?
    data = data_proc.call raw_data
  end
  data
end
raw_filter(raw_data, opts={}) click to toggle source

Remove uncessary columns and invalid rows from csv format data

# File lib/sp2db/base_table.rb, line 176
def raw_filter raw_data, opts={}
  raw_header = raw_data[header_row].map.with_index do |h, idx|
    is_valid = valid_header?(h)
    {
      idx: idx,
      is_remove: !is_valid,
      is_required: require_header?(h),
      name: is_valid && h.gsub(/\s*/, '').gsub(/!/, '').downcase
    }
  end

  rows = raw_data[(header_row + 1)..-1].map.with_index do |raw, rdx|
    row = {}.with_indifferent_access
    raw_header.each do |h|
      val = raw[h[:idx]]
      next if h[:is_remove]
      if h[:is_required] && val.blank?
        row = {}
        break
      end

      row[h[:name]] = standardize_cell_val val
    end

    next if row.values.all?(&:blank?)

    row[:id] = rdx + 1 if find_columns.include?(:id) && row[:id].blank?
    row
  end.compact
     .reject(&:blank?)
  rows = rows.select do |row|
    if required_columns.present?
      required_columns.all? {|col| row[col].present? }
    else
      true
    end
  end

  rows
end
require_header?(h) click to toggle source

Header with “!” at the beginning or ending is required

# File lib/sp2db/base_table.rb, line 156
def require_header? h
  h.present? && (h.match("^!.*") || h.match(".*?!$"))
end
standardize_cell_val(v) click to toggle source

Convert number string to number

# File lib/sp2db/base_table.rb, line 161
def standardize_cell_val v
  v = ((float = Float(v)) && (float % 1.0 == 0) ? float.to_i : float) rescue v
  v = v.force_encoding("UTF-8") if v.is_a?(String)
  v
end
valid_header?(h) click to toggle source

Remove header which starts with “#”

# File lib/sp2db/base_table.rb, line 151
def valid_header? h
  h.present? && !h.match("^#.*")
end