module Spidr
Constants
- VERSION
Spidr
version
Public Class Methods
host(name,options={},&block)
click to toggle source
@see Agent.host
# File lib/spidr/spidr.rb, line 46 def self.host(name,options={},&block) Agent.host(name,options,&block) end
robots()
click to toggle source
@abstract
# File lib/spidr/spidr.rb, line 60 def self.robots end
robots=(mode)
click to toggle source
Enables or disables `robots.txt` globally.
@param [Boolean] mode
@return [Boolean]
@since 0.5.0
# File lib/spidr/spidr.rb, line 32 def self.robots=(mode) @robots = mode end
robots?()
click to toggle source
Specifies whether `robots.txt` should be honored globally.
@return [Boolean]
@since 0.5.0
# File lib/spidr/spidr.rb, line 18 def self.robots? @robots ||= false @robots end
site(url,options={},&block)
click to toggle source
@see Agent.site
# File lib/spidr/spidr.rb, line 53 def self.site(url,options={},&block) Agent.site(url,options,&block) end
start_at(url,options={},&block)
click to toggle source
@see Agent.start_at
# File lib/spidr/spidr.rb, line 39 def self.start_at(url,options={},&block) Agent.start_at(url,options,&block) end