class WebRobots::RobotsTxt::Record
Attributes
Public Class Methods
Source
# File lib/webrobots/robotstxt.rb, line 617 def initialize(agentlines, rulelines) @patterns = agentlines.map { |agentline| agentline.pattern } @acls = [] @delay = nil @options = {} rulelines.each { |ruleline| case ruleline when AccessControlLine @acls << ruleline when CrawlDelayLine @delay = ruleline.delay else @options[ruleline.token.downcase] = ruleline.value end } if rulelines @acls.replace @acls.sort_by { |x| [-x.value.length, x.is_a?(AllowLine) ? -1 : 0] } end
Public Instance Methods
Source
# File lib/webrobots/robotstxt.rb, line 649 def allow?(request_uri) @acls.each { |acl| if acl.match?(request_uri) return acl.allow? end } return true end
Source
# File lib/webrobots/robotstxt.rb, line 645 def default? @patterns.include?(//) end
Source
# File lib/webrobots/robotstxt.rb, line 639 def match?(user_agent) @patterns.any? { |pattern| pattern.match(user_agent) } end