class Sunbro::DynamicHTTP

Attributes

session[R]

Public Class Methods

new(opts = {}) click to toggle source
# File lib/sunbro/dynamic_http.rb, line 6
def initialize(opts = {})
  @opts = opts
  Retryable.retryable { new_session }
end

Public Instance Methods

close() click to toggle source
# File lib/sunbro/dynamic_http.rb, line 11
def close
  @session.driver.quit
rescue IOError
  nil
end
fetch_page(url, opts={}) click to toggle source

Create new Pages from the response of an HTTP request to url, including redirects

# File lib/sunbro/dynamic_http.rb, line 63
def fetch_page(url, opts={})
  begin
    tries ||= 5
    get_page(url, opts)
  rescue IOError, Capybara::Poltergeist::DeadClient, Errno::EPIPE, NoMethodError, Capybara::Poltergeist::BrowserError => e
    restart_session
    retry unless (tries -= 1).zero?
    close
    raise e
  end
end
get_page(url, opts) click to toggle source
# File lib/sunbro/dynamic_http.rb, line 75
def get_page(url, opts)
  reset = opts.fetch(:reset) rescue true
  start = Time.current.to_i
  session.visit(url.to_s)
  page = create_page_from_session(url, session, opts)
  page.response_time = ((Time.now - start) * 1000).round
  session.reset! if reset
  page
rescue Capybara::Poltergeist::TimeoutError => e
  restart_session
  return Page.new(url, :error => e)
end
new_session() click to toggle source
# File lib/sunbro/dynamic_http.rb, line 17
def new_session
  Capybara.register_driver :poltergeist do |app|
    Capybara::Poltergeist::Driver.new(
      app,
      timeout: 10,
      js_errors: false,
      phantomjs_options: phantomjs_options,
    )
  end
  Capybara.default_driver = :poltergeist
  Capybara.javascript_driver = :poltergeist
  Capybara.run_server = false
  @session = Capybara::Session.new(:poltergeist)
  @session.driver.headers = {
    'User-Agent' => user_agent
  }
  @session
end
phantomjs_options() click to toggle source
# File lib/sunbro/dynamic_http.rb, line 36
def phantomjs_options
  @phantomjs_options ||= begin
    opts = [ '--load-images=no', '--ignore-ssl-errors=yes' ]
    if Sunbro::Settings.proxy_host
      if Sunbro::Settings.proxy_port
        opts << "--proxy=#{Sunbro::Settings.proxy_host}:#{Sunbro::Settings.proxy_port}"
      else
        opts << "--proxy=#{Sunbro::Settings.proxy_host}"
      end
    end
    opts
  end
end
restart_session() click to toggle source
# File lib/sunbro/dynamic_http.rb, line 54
def restart_session
  close
  Retryable.retryable { new_session }
end
user_agent() click to toggle source
# File lib/sunbro/dynamic_http.rb, line 50
def user_agent
  @opts[:agent] || Settings.phantomjs_user_agent
end

Private Instance Methods

create_page_from_session(url, session, opts) click to toggle source
# File lib/sunbro/dynamic_http.rb, line 90
def create_page_from_session(url, session, opts)
  url = url.to_s
  if url == session.current_url
    Page.new(
        session.current_url,
        :body => session.html.dup,
        :code => session.status_code,
        :headers => session.response_headers,
        :force_format => (opts[:force_format] || default_page_format)
    )
  else
    Page.new(
        session.current_url,
        :body => session.html.dup,
        :code => 301,
        :redirect_from => url,
        :headers => session.response_headers,
        :force_format => (opts[:force_format] || default_page_format)
    )
  end
end
default_page_format() click to toggle source
# File lib/sunbro/dynamic_http.rb, line 112
def default_page_format
  # Don't force the page format if the default format is set to :any
  return unless [:xml, :html].include? Settings.page_format
  Settings.page_format
end