module CrawlerRocks::DSL
Attributes
current_url[R]
html[R]
Public Instance Methods
get_view_state()
click to toggle source
# File lib/crawler_rocks.rb, line 57 def get_view_state Hash[@doc.css('input[type="hidden"]').map {|d| [d[:name], d[:value]]}] end
post(url, opt = {})
click to toggle source
# File lib/crawler_rocks.rb, line 48 def post url, opt = {} handle_response RestClient.post url, opt.merge({cookies: @cookies}) # @curl.url = url # @curl.http_post(opt) # binding.pry # handle_response @current_url = url end
setup(opts = {})
click to toggle source
# File lib/crawler_rocks.rb, line 10 def setup opts = {} opts = Hash[opts.map{ |k, v| [k.to_sym, v] }] # @curl = Curl::Easy.new # @curl.follow_location = true # @curl.enable_cookies = true @cookies = nil # @encoding = opts.fetch(:encoding, 'utf-8') end
submit(submit_name=nil, form_data={})
click to toggle source
# File lib/crawler_rocks.rb, line 27 def submit submit_name=nil, form_data={} submit_selector = "input[type=\"submit\"][value=\"#{submit_name}\"]" if submit_name.nil? post_hash = get_view_state.merge(form_data) else post_hash = Hash[@doc.css(submit_selector).map{|node| [node[:name], node[:value]]}].merge(get_view_state).merge(form_data) end post_path = @doc.css(submit_selector).xpath('ancestor::form[1]//@action')[0].value uri = URI.parse(@current_url) if post_path[0] == '/' post_path = "#{uri.scheme}://#{uri.host}/" else post_path = URI.join("#{File.dirname(uri.to_s)}/", post_path).to_s end post post_path, post_hash end
visit(url)
click to toggle source
# File lib/crawler_rocks.rb, line 19 def visit url # @curl.url = url # @curl.http_get # handle_response handle_response RestClient.get url @current_url = url end
Private Instance Methods
handle_response(response)
click to toggle source
# File lib/crawler_rocks.rb, line 62 def handle_response response # @doc = Nokogiri::HTML @curl.body_str.force_encoding(@curl.body_str.encoding) @doc = Nokogiri::HTML response.force_encoding(response.encoding) @html = response @cookies ||= response.cookies end