class Fech::Table
Constants
- AWS_URL
Public Class Methods
new(cycle, opts={})
click to toggle source
# File lib/fech-ftp/table.rb, line 5 def initialize(cycle, opts={}) @cycle = cycle @headers = opts[:headers] @file = opts[:file] @format = opts[:format] @location = opts[:location] @passive = opts[:passive] @receiver = opts[:connection] || receiver @parser = parser end
Public Instance Methods
create_table(row)
click to toggle source
# File lib/fech-ftp/table.rb, line 47 def create_table(row) db, table = @receiver table = table.to_s.pluralize.to_sym db.create_table(table) { primary_key :id } row.each do |k,v| v = v.nil? ? String : v.class db.alter_table table do add_column k, v end end @receiver = db[table] @receiver << row end
enter_row(row)
click to toggle source
# File lib/fech-ftp/table.rb, line 29 def enter_row(row) case @format when :db table_exist? ? @receiver << row : create_table(row) when :csv @receiver << row.values else @receiver << row end end
fetch_file(&blk)
click to toggle source
# File lib/fech-ftp/table.rb, line 63 def fetch_file(&blk) filename = "#{@file}#{@cycle.to_s[2..3]}.zip" uri = URI("#{AWS_URL}/#{@cycle}/#{filename}") response = Net::HTTP.get_response(uri) if response.code == '200' unzip(response.body, &blk) end end
format_row(line)
click to toggle source
# File lib/fech-ftp/table.rb, line 87 def format_row(line) hash = {} line = line.encode('UTF-8', invalid: :replace, replace: ' ').chomp.split("|") @parser.each { |k,blk| hash[k] = blk.call(line) } return hash end
parse_date(date)
click to toggle source
# File lib/fech-ftp/table.rb, line 96 def parse_date(date) if date == '' && table_exist? if table_exist? return Date.new(@cycle, 1,1) else return '' end end if date.length == 8 Date.strptime(date, "%m%d%Y") else Date.parse(date) end end
parser()
click to toggle source
# File lib/fech-ftp/table.rb, line 73 def parser @headers.map.with_index do |h,i| if h.to_s =~ /cash|amount|contributions|total|loan|transfer|debts|refund|expenditure/ [h, ->(line) { line[i].to_f }] elsif h == :filing_id [h, ->(line) { line[i].to_i }] elsif h.to_s =~ /_date/ [h, ->(line) { parse_date(line[i]) }] else [h, ->(line) { line[i] }] end end end
receiver()
click to toggle source
# File lib/fech-ftp/table.rb, line 16 def receiver if @format == :csv CSV.open("#{@file}#{@cycle.to_s[2..3]}.csv", 'a+', headers: @headers, write_headers: true) else [] end end
retrieve_data()
click to toggle source
# File lib/fech-ftp/table.rb, line 24 def retrieve_data fetch_file { |row| enter_row(row) } return @receiver end
table_exist?()
click to toggle source
the @receiver obj is the database itself. This assumes the table needs to be created.
# File lib/fech-ftp/table.rb, line 43 def table_exist? @receiver.respond_to? :columns end
unzip(zip_file, &blk)
click to toggle source
# File lib/fech-ftp/table.rb, line 112 def unzip(zip_file, &blk) Zip::File.open_buffer(zip_file) do |zip| zip.each do |entry| path = @location.nil? ? entry.name : @location + entry.name entry.extract(path) if !File.file?(path) File.foreach(path) do |row| blk.call(format_row(row)) end end end end