class Crawler

Attributes

data[R]

Public Class Methods

new(data) click to toggle source
# File lib/contentar/crawler.rb, line 3
def initialize(data)
  @data = data
end

Public Instance Methods

get_data() click to toggle source
# File lib/contentar/crawler.rb, line 7
def get_data
  data.each_with_index.inject([]) do |updated_data, (page_data, index)|
    updated_data << get_page_data(page_data, index)
    updated_data
  end
end

Private Instance Methods

get_page_data(page_data, index) click to toggle source
# File lib/contentar/crawler.rb, line 16
def get_page_data(page_data, index)
  title = page_data.fetch(:title) { '' }
  progress_message(index, title)
  page_stats = PageStats.new(page_data.fetch(:url))
  page_data.merge(get_page_stats(page_stats))
end
get_page_stats(page_stats) click to toggle source
# File lib/contentar/crawler.rb, line 27
def get_page_stats(page_stats)
  begin
    page_stats.data
  rescue Exception => e
    { error: e.message }
  end
end
progress_message(index, title) click to toggle source
# File lib/contentar/crawler.rb, line 23
def progress_message(index, title)
  print "Fetching page #{ index + 1 }: \t\t#{ title.to_s.strip }\n"
end