class RRImm::Fetcher

Attributes

config[RW]
options[RW]

Public Class Methods

new(config, options) click to toggle source
# File lib/rrimm/fetcher.rb, line 12
def initialize(config, options)
  @config  = config
  @options = options
  @quiet = options['quiet']
  @category = options['category']
end

Public Instance Methods

debug(message) click to toggle source
# File lib/rrimm/fetcher.rb, line 44
def debug(message)
  puts message if options['verbose']
end
feeds() click to toggle source
# File lib/rrimm/fetcher.rb, line 40
def feeds
  @config.feeds.select { |f,conf| @category.nil? || conf.category == @category }
end
fetch() click to toggle source
# File lib/rrimm/fetcher.rb, line 19
def fetch()
  if options['concurrency']
    parallel_fetch
  else
    linear_fetch
  end
end
fetch_feed(name, feed_config) click to toggle source
# File lib/rrimm/fetcher.rb, line 48
def fetch_feed(name, feed_config)
  timeout = 12 * 60
  Timeout::timeout(timeout) do
    begin
      fetch_feed_no_timeout(name, feed_config)
    rescue => e
      puts "Received #{e.class.name} #{e.message} for #{name}"
      raise
    end
  end
rescue Timeout::Error
  puts "#{name} timeout after #{timeout} seconds"
end
fetch_feed_no_timeout(name, feed_config) click to toggle source
# File lib/rrimm/fetcher.rb, line 62
def fetch_feed_no_timeout(name, feed_config)
  debug "-> #{name}: reading cache"
  last_read = Time.at(@config.get_cache.read(feed_config))
  print name unless @quiet
  debug "-> #{name}: fetching and parsing"
  feed = Feedjira::Feed.fetch_and_parse(feed_config.uri)
  debug "-> #{name}: fetched and parsed (#{feed.class})"
  if feed.respond_to? :entries
    items = feed.entries.select { |item| item.published > last_read }
    last_read = items.collect { |item| item.published }.max unless items.empty?
    feed_config.massage(items).each do |item|
      debug "-> #{name}: format an item"
      feed_config.format(feed, item)
    end
    debug "-> #{name}: saving cache"
    @config.get_cache.save(feed_config, last_read.to_i, false)
  end
  puts " (#{items.size rescue nil})" unless @quiet
end
linear_fetch() click to toggle source
# File lib/rrimm/fetcher.rb, line 34
def linear_fetch
  feeds.map do |name,feed_config|
    fetch_feed(name, feed_config)
  end
end
parallel_fetch() click to toggle source
# File lib/rrimm/fetcher.rb, line 27
def parallel_fetch
  Parallel.map(feeds, :in_threads => options['concurrency'], :progress => "fetching") do |name,feed_config|
    @quiet = true
    fetch_feed(name, feed_config)
  end
end