class Apollo::BaseProgram
Constants
- CONFIG_DIR
- DEFAULT_OPTIONS
Attributes
amqp[RW]
config[RW]
mongo[RW]
options[RW]
optparser[RW]
Public Class Methods
get_config_path(config)
click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 50 def self.get_config_path(config) return File.join(CONFIG_DIR, "#{config}.yml") end
new()
click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 41 def initialize self.config = {} self.options = DEFAULT_OPTIONS self.optparser = nil self.amqp = nil self.mongo = nil end
require_files(files = [])
click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 86 def self.require_files(files = []) Dir.glob(files).each do |file| require file end end
Public Instance Methods
init_amqp()
click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 118 def init_amqp() conn_opts = self.config["amqp"] if(conn_opts) self.amqp = Apollo::Helper::Amqp::connect(conn_opts, self.options) end return self.amqp end
init_mongo()
click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 127 def init_mongo() conn_opts = self.config["mongo"] if(conn_opts) self.mongo = Apollo::Helper::Mongo::connect(conn_opts, self.options) # Init Mongoid path = File.join(Apollo::BASE_DIR, "config/mongoid.yml") Mongoid.load!(path, @options[:env]) end return self.mongo end
init_options()
click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 114 def init_options() return nil end
init_program(args)
click to toggle source
Init program
# File lib/apollo_crawler/program/base_program.rb, line 168 def init_program(args) res = nil res = init_options() begin res = parse_options(args) rescue Exception => e puts "Unable to parse options" return res unless res.nil? end res = process_options(args) return res unless res.nil? confs = load_configs(BaseProgram::CONFIG_DIR, @options) puts "Loaded configs => #{confs.inspect}" if @options[:verbose] # Init Mongo Connection init_mongo() # Init AMQP init_amqp() # Init Seed data init_seeds(@options) return nil end
init_seeds(opts={})
click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 163 def init_seeds(opts={}) init_seeds_crawlers(opts) end
init_seeds_crawlers(opts={})
click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 140 def init_seeds_crawlers(opts={}) objs = Apollo::Crawler::BaseCrawler.subclasses objs.each do |o| crawler = Apollo::Model::Crawler.new i = o.new crawler.name = i.name crawler.class_name = o.to_s res = Apollo::Model::Crawler.where(class_name: crawler.class_name) # puts "RES: '#{res.inspect}'" if(res.nil? || res.count < 1) crawler.save if(opts[:verbose]) puts "Adding new crawler - '#{crawler.inspect}'" end else if(opts[:verbose]) puts "Using crawler - '#{res[0].inspect}'" end end end end
load_config(config_name, options = DEFAULT_OPTIONS)
click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 81 def load_config(config_name, options = DEFAULT_OPTIONS) path = BaseProgram.get_config_path(config_name) return load_config_file(path, options) end
load_config_file(config_file, options = DEFAULT_OPTIONS)
click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 63 def load_config_file(config_file, options = DEFAULT_OPTIONS) return nil unless File.exists?(config_file) config_name = File.basename(config_file.downcase, ".yml") res = YAML.load(File.read(config_file)) return nil unless res res = {config_name => res[options[:env]]} self.config.merge! res if(options[:verbose]) puts "Loaded config '#{config_file}' => #{res[config_name]}" end return res[config_name] end
load_configs(dir = CONFIG_DIR, options = DEFAULT_OPTIONS)
click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 54 def load_configs(dir = CONFIG_DIR, options = DEFAULT_OPTIONS) Dir[File.join(dir, "**/*.yml")].each do |c| puts "Loading config #{c}" if options[:verbose] self.load_config_file(c, options) end return self.config end
parse_options(args = ARGV)
click to toggle source
Parse the options passed to command-line
# File lib/apollo_crawler/program/base_program.rb, line 93 def parse_options(args = ARGV) res = [] # Parse the command-line. Remember there are two forms # of the parse method. The 'parse' method simply parses # ARGV, while the 'parse!' method parses ARGV and removes # any options found there, as well as any parameters for # the options. What's left is the list of files to resize. begin optparser.parse!(args) rescue Exception => e return nil end return res end
process_options(args)
click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 110 def process_options(args) return nil end
request_exit(code = 0)
click to toggle source
# File lib/apollo_crawler/program/base_program.rb, line 205 def request_exit(code = 0) puts "Exiting app" begin exit(code) rescue SystemExit => e # puts "rescued a SystemExit exception, reason: '#{e.to_s}'" end return code end
run(args = ARGV)
click to toggle source
Run Program
# File lib/apollo_crawler/program/base_program.rb, line 198 def run(args = ARGV) res = init_program(args) return res unless res.nil? puts "Running environment '#{@options[:env]}'" if @options[:verbose] end