class Crawler
Attributes
data[R]
Public Class Methods
new(data)
click to toggle source
# File lib/contentar/crawler.rb, line 3 def initialize(data) @data = data end
Public Instance Methods
get_data()
click to toggle source
# File lib/contentar/crawler.rb, line 7 def get_data data.each_with_index.inject([]) do |updated_data, (page_data, index)| updated_data << get_page_data(page_data, index) updated_data end end
Private Instance Methods
get_page_data(page_data, index)
click to toggle source
# File lib/contentar/crawler.rb, line 16 def get_page_data(page_data, index) title = page_data.fetch(:title) { '' } progress_message(index, title) page_stats = PageStats.new(page_data.fetch(:url)) page_data.merge(get_page_stats(page_stats)) end
get_page_stats(page_stats)
click to toggle source
# File lib/contentar/crawler.rb, line 27 def get_page_stats(page_stats) begin page_stats.data rescue Exception => e { error: e.message } end end
progress_message(index, title)
click to toggle source
# File lib/contentar/crawler.rb, line 23 def progress_message(index, title) print "Fetching page #{ index + 1 }: \t\t#{ title.to_s.strip }\n" end