class RRImm::Fetcher
Attributes
config[RW]
options[RW]
Public Class Methods
new(config, options)
click to toggle source
# File lib/rrimm/fetcher.rb, line 12 def initialize(config, options) @config = config @options = options @quiet = options['quiet'] @category = options['category'] end
Public Instance Methods
debug(message)
click to toggle source
# File lib/rrimm/fetcher.rb, line 44 def debug(message) puts message if options['verbose'] end
feeds()
click to toggle source
# File lib/rrimm/fetcher.rb, line 40 def feeds @config.feeds.select { |f,conf| @category.nil? || conf.category == @category } end
fetch()
click to toggle source
# File lib/rrimm/fetcher.rb, line 19 def fetch() if options['concurrency'] parallel_fetch else linear_fetch end end
fetch_feed(name, feed_config)
click to toggle source
# File lib/rrimm/fetcher.rb, line 48 def fetch_feed(name, feed_config) timeout = 12 * 60 Timeout::timeout(timeout) do begin fetch_feed_no_timeout(name, feed_config) rescue => e puts "Received #{e.class.name} #{e.message} for #{name}" raise end end rescue Timeout::Error puts "#{name} timeout after #{timeout} seconds" end
fetch_feed_no_timeout(name, feed_config)
click to toggle source
# File lib/rrimm/fetcher.rb, line 62 def fetch_feed_no_timeout(name, feed_config) debug "-> #{name}: reading cache" last_read = Time.at(@config.get_cache.read(feed_config)) print name unless @quiet debug "-> #{name}: fetching and parsing" feed = Feedjira::Feed.fetch_and_parse(feed_config.uri) debug "-> #{name}: fetched and parsed (#{feed.class})" if feed.respond_to? :entries items = feed.entries.select { |item| item.published > last_read } last_read = items.collect { |item| item.published }.max unless items.empty? feed_config.massage(items).each do |item| debug "-> #{name}: format an item" feed_config.format(feed, item) end debug "-> #{name}: saving cache" @config.get_cache.save(feed_config, last_read.to_i, false) end puts " (#{items.size rescue nil})" unless @quiet end
linear_fetch()
click to toggle source
# File lib/rrimm/fetcher.rb, line 34 def linear_fetch feeds.map do |name,feed_config| fetch_feed(name, feed_config) end end
parallel_fetch()
click to toggle source
# File lib/rrimm/fetcher.rb, line 27 def parallel_fetch Parallel.map(feeds, :in_threads => options['concurrency'], :progress => "fetching") do |name,feed_config| @quiet = true fetch_feed(name, feed_config) end end