class Apollo::BaseProgram

Constants

CONFIG_DIR
DEFAULT_OPTIONS

Attributes

amqp[RW]
config[RW]
mongo[RW]
options[RW]
optparser[RW]

Public Class Methods

get_config_path(config) click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 50
def self.get_config_path(config)
        return File.join(CONFIG_DIR, "#{config}.yml")
end
new() click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 41
def initialize
        self.config = {}
        self.options = DEFAULT_OPTIONS
        self.optparser = nil

        self.amqp = nil
        self.mongo = nil
end
require_files(files = []) click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 86
def self.require_files(files = [])
        Dir.glob(files).each do |file|
                require file
        end
end

Public Instance Methods

init_amqp() click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 118
def init_amqp()
        conn_opts = self.config["amqp"]
        if(conn_opts)
                self.amqp = Apollo::Helper::Amqp::connect(conn_opts, self.options)
        end

        return self.amqp
end
init_mongo() click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 127
def init_mongo()
        conn_opts = self.config["mongo"]
        if(conn_opts)
                self.mongo = Apollo::Helper::Mongo::connect(conn_opts, self.options)

                # Init Mongoid
                path = File.join(Apollo::BASE_DIR, "config/mongoid.yml")
                Mongoid.load!(path, @options[:env])
        end

        return self.mongo
end
init_options() click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 114
def init_options()
        return nil
end
init_program(args) click to toggle source

Init program

# File lib/apollo_crawler/program/base_program.rb, line 168
def init_program(args)
        res = nil
        res = init_options()

        begin
                res = parse_options(args)
        rescue Exception => e
                puts "Unable to parse options"
                return res unless res.nil?
        end

        res = process_options(args)
        return res unless res.nil?

        confs = load_configs(BaseProgram::CONFIG_DIR, @options)
        puts "Loaded configs => #{confs.inspect}" if @options[:verbose]

        # Init Mongo Connection
        init_mongo()

        # Init AMQP
        init_amqp()

        # Init Seed data
        init_seeds(@options)

        return nil
end
init_seeds(opts={}) click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 163
def init_seeds(opts={})
        init_seeds_crawlers(opts)
end
init_seeds_crawlers(opts={}) click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 140
def init_seeds_crawlers(opts={})
        objs = Apollo::Crawler::BaseCrawler.subclasses
        objs.each do |o|
                crawler = Apollo::Model::Crawler.new
                i = o.new
                crawler.name = i.name
                crawler.class_name = o.to_s
                
                res = Apollo::Model::Crawler.where(class_name: crawler.class_name)
                # puts "RES: '#{res.inspect}'"
                if(res.nil? || res.count < 1)
                        crawler.save
                        if(opts[:verbose])
                                puts "Adding new crawler - '#{crawler.inspect}'"
                        end
                else
                        if(opts[:verbose])
                                puts "Using crawler - '#{res[0].inspect}'"
                        end
                end
        end
end
load_config(config_name, options = DEFAULT_OPTIONS) click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 81
def load_config(config_name, options = DEFAULT_OPTIONS)
        path = BaseProgram.get_config_path(config_name)
        return load_config_file(path, options)
end
load_config_file(config_file, options = DEFAULT_OPTIONS) click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 63
def load_config_file(config_file, options = DEFAULT_OPTIONS)
        return nil unless File.exists?(config_file)

        config_name = File.basename(config_file.downcase, ".yml")

        res = YAML.load(File.read(config_file))
        return nil unless res

        res = {config_name => res[options[:env]]}
        self.config.merge! res
        
        if(options[:verbose])
                puts "Loaded config '#{config_file}' => #{res[config_name]}"
        end

        return res[config_name]
end
load_configs(dir = CONFIG_DIR, options = DEFAULT_OPTIONS) click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 54
def load_configs(dir = CONFIG_DIR, options = DEFAULT_OPTIONS)
        Dir[File.join(dir, "**/*.yml")].each do |c|
                puts "Loading config #{c}" if options[:verbose]
                self.load_config_file(c, options)
        end

        return self.config
end
parse_options(args = ARGV) click to toggle source

Parse the options passed to command-line

# File lib/apollo_crawler/program/base_program.rb, line 93
def parse_options(args = ARGV)
        res = []

        # Parse the command-line. Remember there are two forms
        # of the parse method. The 'parse' method simply parses
        # ARGV, while the 'parse!' method parses ARGV and removes
        # any options found there, as well as any parameters for
        # the options. What's left is the list of files to resize.
        begin
                optparser.parse!(args)
        rescue Exception => e
                return nil
        end

        return res
end
process_options(args) click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 110
def process_options(args)
        return nil
end
request_exit(code = 0) click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 205
def request_exit(code = 0)
        puts "Exiting app"
        begin
                exit(code)
        rescue SystemExit => e
                # puts "rescued a SystemExit exception, reason: '#{e.to_s}'"
        end

        return code
end
run(args = ARGV) click to toggle source

Run Program

# File lib/apollo_crawler/program/base_program.rb, line 198
def run(args = ARGV)
        res = init_program(args)
        return res unless res.nil?

        puts "Running environment '#{@options[:env]}'" if @options[:verbose]
end