class Anaximander::Crawler
Attributes
root[R]
url[R]
Public Class Methods
new(url)
click to toggle source
# File lib/anaximander/crawler.rb, line 5 def initialize(url) @url = url.chomp("/") @root = Page.new(url) @visited = [url] end
Public Instance Methods
crawl(page=self.root)
click to toggle source
# File lib/anaximander/crawler.rb, line 11 def crawl(page=self.root) page.children = page.links.map { |link| visit(link.chomp("/")) }.compact page.children.each { |child| crawl(child) } end
logger()
click to toggle source
# File lib/anaximander/crawler.rb, line 27 def logger Anaximander.logger end
visit(link)
click to toggle source
# File lib/anaximander/crawler.rb, line 16 def visit(link) return if @visited.include?(link) logger.debug(link) @visited << link Page.new(link) rescue Anaximander::PageNotAccessibleError nil end