module Apollo::Helper::Mongo
Public Class Methods
connect(conn, opts={})
click to toggle source
# File lib/apollo_crawler/helper/mongo_helper.rb, line 29 def self.connect(conn, opts={}) if(opts[:verbose]) puts "MongoDB connecting - '#{conn.inspect}" end res = ::Mongo::Connection.new(conn['host']) if(opts[:verbose]) puts "MongoDB connected: #{res.inspect}" end return res end
csv_bulk_insert(path, model, bulk_size, validate=false) { |row| ... }
click to toggle source
# File lib/apollo_crawler/helper/mongo_helper.rb, line 43 def self.csv_bulk_insert(path, model, bulk_size, validate=false, &block) batch = [] CSV.foreach(path) do |row| res = nil if block_given? res = yield row end if res.nil? == false if(!validate || model.where(res).length == 0) batch << res end end if((batch.length % bulk_size) == 0) # puts "Inserting batch '#{batch.inspect}'" model.collection.insert(batch) batch.clear end end if batch.empty? == false model.collection.insert(batch) batch.clear end end