require 'csv' require 'net/http' require 'zip/zipfilesystem'
namespace :pin_point do
task :import => :environment do TIME = Time.now.to_i ZIP_FILE = "/tmp/pin_point_data_#{TIME}.zip" BLOCK_FILE = "/tmp/pin_point_block_#{TIME}.csv" LOCATION_FILE = "/tmp/pin_point_location_#{TIME}.csv" REMOTE_DATA_DOMAIN = 'geolite.maxmind.com' REMOTE_DATA_PATH = '/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.zip' puts "Sourcing data..." Net::HTTP.start REMOTE_DATA_DOMAIN do |http| response = http.get REMOTE_DATA_PATH open ZIP_FILE, 'wb' do |file| file.write response.body end end puts "Unzipping..." Zip::ZipFile.open ZIP_FILE do |zipfile| zipfile.each do |file| case file.name when /block/i zipfile.extract file.name, BLOCK_FILE when /location/i zipfile.extract file.name, LOCATION_FILE end end end puts "Removing old IP Block data..." PinPoint::IpBlock.delete_all puts "Dropping existing indexes..." PinPoint::IpBlock.remove_indexes puts "Generating new indexes..." PinPoint::IpBlock.create_indexes puts "Importing IP Block data. This will take a while..." import_start = ip_block_start = Time.now open BLOCK_FILE, 'r' do |file| while line = file.gets if $. > 2 begin row = CSV.parse_line( line.force_encoding('UTF-8').chars.select{|i| i.valid_encoding?}.join, col_sep: ',' ) rescue CSV::MalformedCSVError Rails.logger.debug "Failed to parse: #{line}\n#{$!.message}" else PinPoint::IpBlock.create({ range_low: row[0].to_i, range_high: row[1].to_i, location: row[2].to_i }) end if ($. - 2) % 100000 == 0 now = Time.now current_count = ($. - 2) per_second = ($. - 2) / ( ( now - ip_block_start ) / 1.second ) seconds = ( now - ip_block_start ) % 1.minute minutes = ( now - ip_block_start ) / 1.minute puts "%d blocks imported in %d minutes, %d seconds. %.2f blocks/s ..." % [ current_count, minutes, seconds, per_second ] end end end end puts "Mapping location data to IP Blocks..." location_start = Time.now open LOCATION_FILE, 'r' do |file| while line = file.gets if $. > 2 begin row = CSV.parse_line( line.force_encoding('UTF-8').chars.select{|i| i.valid_encoding?}.join, col_sep: ',' ) rescue CSV::MalformedCSVError Rails.logger.debug "Failed to parse: #{line}\n#{$!.message}" else PinPoint::IpBlock.where( location: row[0].to_i ).update_all({ country: row[1], state: row[2], city: row[3], coordinates: [row[6].to_f, row[5].to_f] }) end if ($. - 2) % 100000 == 0 now = Time.now current_count = ($. - 2) per_second = ($. - 2) / ( ( now - location_start ) / 1.second ) seconds = ( now - location_start ) % 1.minute minutes = ( now - location_start ) / 1.minute puts "%d locations mapped in %d minutes, %d seconds. %.2f locations/s ..." % [ current_count, minutes, seconds, per_second ] end end end end end
end