module Apollo::Helper::Mongo

Public Class Methods

connect(conn, opts={}) click to toggle source
# File lib/apollo_crawler/helper/mongo_helper.rb, line 29
def self.connect(conn, opts={})
        if(opts[:verbose])
                puts "MongoDB connecting - '#{conn.inspect}"
        end

        res = ::Mongo::Connection.new(conn['host'])

        if(opts[:verbose])
                puts "MongoDB connected: #{res.inspect}"
        end

        return res
end
csv_bulk_insert(path, model, bulk_size, validate=false) { |row| ... } click to toggle source
# File lib/apollo_crawler/helper/mongo_helper.rb, line 43
def self.csv_bulk_insert(path, model, bulk_size, validate=false, &block)
        batch = []
        
        CSV.foreach(path) do |row|
                res = nil
                if block_given?
                        res = yield row
                end

                if res.nil? == false
                        if(!validate || model.where(res).length == 0)
                                batch << res
                        end
                end

                if((batch.length % bulk_size) == 0)
                        # puts "Inserting batch '#{batch.inspect}'"
                        
                        model.collection.insert(batch)
                        batch.clear
                end
        end

        if batch.empty? == false
                model.collection.insert(batch)
                batch.clear
        end
end