module CrawlerRocks::DSL

Attributes

current_url[R]
html[R]

Public Instance Methods

get_view_state() click to toggle source
# File lib/crawler_rocks.rb, line 57
def get_view_state
  Hash[@doc.css('input[type="hidden"]').map {|d| [d[:name], d[:value]]}]
end
post(url, opt = {}) click to toggle source
# File lib/crawler_rocks.rb, line 48
def post url, opt = {}
  handle_response RestClient.post url, opt.merge({cookies: @cookies})
  # @curl.url = url
  # @curl.http_post(opt)
  # binding.pry
  # handle_response
  @current_url = url
end
setup(opts = {}) click to toggle source
# File lib/crawler_rocks.rb, line 10
def setup opts = {}
  opts = Hash[opts.map{ |k, v| [k.to_sym, v] }]
  # @curl = Curl::Easy.new
  # @curl.follow_location = true
  # @curl.enable_cookies = true
  @cookies = nil
  # @encoding = opts.fetch(:encoding, 'utf-8')
end
submit(submit_name=nil, form_data={}) click to toggle source
# File lib/crawler_rocks.rb, line 27
def submit submit_name=nil, form_data={}
  submit_selector = "input[type=\"submit\"][value=\"#{submit_name}\"]"

  if submit_name.nil?
    post_hash = get_view_state.merge(form_data)
  else
    post_hash = Hash[@doc.css(submit_selector).map{|node| [node[:name], node[:value]]}].merge(get_view_state).merge(form_data)
  end

  post_path = @doc.css(submit_selector).xpath('ancestor::form[1]//@action')[0].value

  uri = URI.parse(@current_url)
  if post_path[0] == '/'
    post_path = "#{uri.scheme}://#{uri.host}/"
  else
    post_path = URI.join("#{File.dirname(uri.to_s)}/", post_path).to_s
  end

  post post_path, post_hash
end
visit(url) click to toggle source
# File lib/crawler_rocks.rb, line 19
def visit url
  # @curl.url = url
  # @curl.http_get
  # handle_response
  handle_response RestClient.get url
  @current_url = url
end

Private Instance Methods

handle_response(response) click to toggle source
# File lib/crawler_rocks.rb, line 62
def handle_response response
  # @doc = Nokogiri::HTML @curl.body_str.force_encoding(@curl.body_str.encoding)
  @doc = Nokogiri::HTML response.force_encoding(response.encoding)
  @html = response
  @cookies ||= response.cookies
end