class Fech::Table

Constants

AWS_URL

Public Class Methods

new(cycle, opts={}) click to toggle source
# File lib/fech-ftp/table.rb, line 5
def initialize(cycle, opts={})
  @cycle    = cycle
  @headers  = opts[:headers]
  @file     = opts[:file]
  @format   = opts[:format]
  @location = opts[:location]
  @passive  = opts[:passive]
  @receiver = opts[:connection] || receiver
  @parser   = parser
end

Public Instance Methods

create_table(row) click to toggle source
# File lib/fech-ftp/table.rb, line 47
def create_table(row)
  db, table = @receiver
  table = table.to_s.pluralize.to_sym
  db.create_table(table) { primary_key :id }

  row.each do |k,v|
    v = v.nil? ? String : v.class
    db.alter_table table do
      add_column k, v
    end
  end

  @receiver = db[table]
  @receiver << row
end
enter_row(row) click to toggle source
# File lib/fech-ftp/table.rb, line 29
def enter_row(row)
  case @format
  when :db
    table_exist? ? @receiver << row : create_table(row)
  when :csv
    @receiver << row.values
  else
    @receiver << row
  end
end
fetch_file(&blk) click to toggle source
# File lib/fech-ftp/table.rb, line 63
def fetch_file(&blk)
  filename = "#{@file}#{@cycle.to_s[2..3]}.zip"
  uri = URI("#{AWS_URL}/#{@cycle}/#{filename}")
  response = Net::HTTP.get_response(uri)

  if response.code == '200'
    unzip(response.body, &blk)
  end
end
format_row(line) click to toggle source
# File lib/fech-ftp/table.rb, line 87
def format_row(line)
  hash = {}
  line = line.encode('UTF-8', invalid: :replace, replace: ' ').chomp.split("|")

  @parser.each { |k,blk| hash[k] = blk.call(line) }

  return hash
end
parse_date(date) click to toggle source
# File lib/fech-ftp/table.rb, line 96
def parse_date(date)
  if date == '' && table_exist?
    if table_exist?
      return Date.new(@cycle, 1,1)
    else
      return ''
    end
  end

  if date.length == 8
    Date.strptime(date, "%m%d%Y")
  else
    Date.parse(date)
  end
end
parser() click to toggle source
# File lib/fech-ftp/table.rb, line 73
def parser
  @headers.map.with_index do |h,i|
    if h.to_s =~ /cash|amount|contributions|total|loan|transfer|debts|refund|expenditure/
      [h, ->(line) { line[i].to_f }]
    elsif h == :filing_id
      [h, ->(line) { line[i].to_i }]
    elsif h.to_s =~ /_date/
      [h, ->(line) { parse_date(line[i]) }]
    else
      [h, ->(line) { line[i] }]
    end
  end
end
receiver() click to toggle source
# File lib/fech-ftp/table.rb, line 16
def receiver
  if @format == :csv
    CSV.open("#{@file}#{@cycle.to_s[2..3]}.csv", 'a+', headers: @headers, write_headers: true)
  else
    []
  end
end
retrieve_data() click to toggle source
# File lib/fech-ftp/table.rb, line 24
def retrieve_data
  fetch_file { |row| enter_row(row) }
  return @receiver
end
table_exist?() click to toggle source

the @receiver obj is the database itself. This assumes the table needs to be created.

# File lib/fech-ftp/table.rb, line 43
def table_exist?
  @receiver.respond_to? :columns
end
unzip(zip_file, &blk) click to toggle source
# File lib/fech-ftp/table.rb, line 112
def unzip(zip_file, &blk)
  Zip::File.open_buffer(zip_file) do |zip|
    zip.each do |entry|
      path = @location.nil? ? entry.name : @location + entry.name
      entry.extract(path) if !File.file?(path)

      File.foreach(path) do |row|
        blk.call(format_row(row))
      end
    end
  end
end