class Mechanize::HTTP::Agent
An HTTP
(and local disk access) user agent. This class is an implementation detail and is subject to change at any time.
Constants
- COOKIE_HEADERS
- CREDENTIAL_HEADERS
- POST_HEADERS
Attributes
Responses larger than this will be written to a Tempfile instead of stored in memory. Setting this to nil disables creation of Tempfiles.
Public Class Methods
Source
# File lib/mechanize/http/agent.rb, line 140 def initialize(connection_name = 'mechanize') @allowed_error_codes = [] @conditional_requests = true @context = nil @content_encoding_hooks = [] @cookie_jar = Mechanize::CookieJar.new @follow_meta_refresh = false @follow_meta_refresh_self = false @gzip_enabled = true @history = Mechanize::History.new @ignore_bad_chunking = false @keep_alive = true @max_file_buffer = 100_000 # 5MB for response bodies @open_timeout = nil @post_connect_hooks = [] @pre_connect_hooks = [] @read_timeout = nil @redirect_ok = true @redirection_limit = 20 @request_headers = {} @robots = false @robots_mutex = Mutex.new @user_agent = nil @webrobots = nil # HTTP Authentication @auth_store = Mechanize::HTTP::AuthStore.new @authenticate_parser = Mechanize::HTTP::WWWAuthenticateParser.new @authenticate_methods = Hash.new do |methods, uri| methods[uri] = Hash.new do |realms, auth_scheme| realms[auth_scheme] = [] end end @digest_auth = Net::HTTP::DigestAuth.new @digest_challenges = {} # SSL @pass = nil @scheme_handlers = Hash.new { |h, scheme| h[scheme] = lambda { |link, page| raise Mechanize::UnsupportedSchemeError.new(scheme, link) } } @scheme_handlers['http'] = lambda { |link, page| link } @scheme_handlers['https'] = @scheme_handlers['http'] @scheme_handlers['relative'] = @scheme_handlers['http'] @scheme_handlers['file'] = @scheme_handlers['http'] @http = if defined?(Net::HTTP::Persistent::DEFAULT_POOL_SIZE) Net::HTTP::Persistent.new(name: connection_name) else # net-http-persistent < 3.0 Net::HTTP::Persistent.new(connection_name) end @http.idle_timeout = 5 @http.keep_alive = 300 end
The connection_name can be used to segregate SSL connections. Agents with different names will not share the same persistent connection.
Public Instance Methods
Source
# File lib/mechanize/http/agent.rb, line 209 def add_auth uri, user, password, realm = nil, domain = nil @auth_store.add_auth uri, user, password, realm, domain end
Adds credentials user
, pass
for uri
. If realm
is set the credentials are used only for that realm. If realm
is not set the credentials become the default for any realm on that URI.
domain
and realm
are exclusive as NTLM does not follow RFC 2617. If domain
is given it is only used for NTLM authentication.
Source
# File lib/mechanize/http/agent.rb, line 240 def fetch uri, method = :get, headers = {}, params = [], referer = current_page, redirects = 0 referer_uri = referer ? referer.uri : nil uri = resolve uri, referer uri, params = resolve_parameters uri, method, params request = http_request uri, method, params connection = connection_for uri request_auth request, uri disable_keep_alive request enable_gzip request request_language_charset request request_cookies request, uri request_host request, uri request_referer request, uri, referer_uri request_user_agent request request_add_headers request, headers pre_connect request # Consult robots.txt if robots && uri.is_a?(URI::HTTP) robots_allowed?(uri) or raise Mechanize::RobotsDisallowedError.new(uri) end # Add If-Modified-Since if page is in history if page = visited_page(uri) and last_modified = page.response['Last-Modified'] request['If-Modified-Since'] = last_modified end if @conditional_requests # Specify timeouts if supplied and our connection supports them if @open_timeout && connection.respond_to?(:open_timeout=) connection.open_timeout = @open_timeout end if @read_timeout && connection.respond_to?(:read_timeout=) connection.read_timeout = @read_timeout end request_log request response_body_io = nil # Send the request begin response = connection.request(uri, request) { |res| response_log res response_body_io = response_read res, request, uri res } rescue Mechanize::ChunkedTerminationError => e raise unless @ignore_bad_chunking response = e.response response_body_io = e.body_io end hook_content_encoding response, uri, response_body_io response_body_io = response_content_encoding response, response_body_io if request.response_body_permitted? post_connect uri, response, response_body_io page = response_parse response, response_body_io, uri response_cookies response, uri, page meta = response_follow_meta_refresh response, uri, page, redirects return meta if meta if robots && page.is_a?(Mechanize::Page) page.parser.noindex? and raise Mechanize::RobotsDisallowedError.new(uri) end case response when Net::HTTPSuccess page when Mechanize::FileResponse page when Net::HTTPNotModified log.debug("Got cached page") if log visited_page(uri) || page when Net::HTTPRedirection response_redirect response, method, page, redirects, headers, referer when Net::HTTPUnauthorized response_authenticate(response, page, uri, request, headers, params, referer) else if @allowed_error_codes.any? {|code| code.to_s == page.code} then page else raise Mechanize::ResponseCodeError.new(page, 'unhandled response') end end end
Retrieves uri
and parses it into a page or other object according to PluggableParser. If the URI is an HTTP
or HTTPS scheme URI the given HTTP
method
is used to retrieve it, along with the HTTP
headers
, request params
and HTTP
referer
.
The final URI to access is built with uri
and params
, the latter of which is formatted into a string using Mechanize::Util.build_query_string
, which see.
redirects
tracks the number of redirects experienced when retrieving the page. If it is over the redirection_limit
an error will be raised.
Source
# File lib/mechanize/http/agent.rb, line 340 def proxy_uri @http.proxy_uri end
URI for a proxy connection
Source
# File lib/mechanize/http/agent.rb, line 345 def retry_change_requests @http.retry_change_requests end
Retry non-idempotent requests?
Source
# File lib/mechanize/http/agent.rb, line 351 def retry_change_requests= retri @http.retry_change_requests = retri end
Retry non-idempotent requests
Allowed error codes
↑ topAttributes
List of error codes (in String or Integer) to handle without raising Mechanize::ResponseCodeError
, defaulted to an empty array. Note that 2xx, 3xx and 401 status codes will be handled without checking this list.
HTTP Authentication
↑ topHeaders
↑ topAttributes
Disables If-Modified-Since conditional requests (enabled by default)
Is gzip compression of requests enabled?
A hash of request headers to be used for every request
The User-Agent header to send
Public Instance Methods
Source
# File lib/mechanize/http/agent.rb, line 357 def user_agent= user_agent @webrobots = nil if user_agent != @user_agent @user_agent = user_agent end
History
↑ topAttributes
history of requests made
Public Instance Methods
Source
# File lib/mechanize/http/agent.rb, line 366 def back @history.pop end
Equivalent to the browser back button. Returns the most recent page visited.
Source
# File lib/mechanize/http/agent.rb, line 373 def current_page @history.last end
Returns the latest page loaded by the agent
Source
# File lib/mechanize/http/agent.rb, line 378 def max_history @history.max_size end
Returns the maximum size for the history stack.
Source
# File lib/mechanize/http/agent.rb, line 383 def max_history=(length) @history.max_size = length end
Set the maximum size for the history stack.
Source
# File lib/mechanize/http/agent.rb, line 388 def visited_page url @history.visited_page resolve url end
Returns a visited page for the url passed in, otherwise nil
Hooks
↑ topAttributes
A list of hooks to call to handle the content-encoding of a request.
A list of hooks to call after retrieving a response. Hooks are called with the agent and the response returned.
A list of hooks to call before making a request. Hooks are called with the agent and the request to be performed.
Public Instance Methods
Source
# File lib/mechanize/http/agent.rb, line 394 def hook_content_encoding response, uri, response_body_io @content_encoding_hooks.each do |hook| hook.call self, uri, response, response_body_io end end
Source
# File lib/mechanize/http/agent.rb, line 407 def post_connect uri, response, body_io # :yields: agent, uri, response, body @post_connect_hooks.each do |hook| begin hook.call self, uri, response, body_io.read ensure body_io.rewind end end end
Invokes hooks added to post_connect_hooks
after a response
is returned and the response body
is handled.
Yields the context
, the uri
for the request, the response
and the response body
.
Source
# File lib/mechanize/http/agent.rb, line 421 def pre_connect request # :yields: agent, request @pre_connect_hooks.each do |hook| hook.call self, request end end
Invokes hooks added to pre_connect_hooks
before a request
is made. Yields the agent
and the request
that will be performed to each hook.
Redirection
↑ topAttributes
Follow HTML meta refresh and HTTP
Refresh. If set to :anywhere
meta refresh tags outside of the head element will be followed.
Follow an HTML meta refresh that has no “url=” in the content attribute.
Defaults to false to prevent infinite refresh loops.
Controls how this agent deals with redirects. The following values are allowed:
- :all, true
-
All 3xx redirects are followed (default)
- :permanent
-
Only 301 Moved Permanently redirects are followed
- false
-
No redirects are followed
Maximum number of redirects to follow
Request
↑ topPublic Instance Methods
Source
# File lib/mechanize/http/agent.rb, line 429 def connection_for uri case uri.scheme.downcase when 'http', 'https' then return @http when 'file' then return Mechanize::FileConnection.new end end
Source
# File lib/mechanize/http/agent.rb, line 509 def content_encoding_brotli(body_io) log.debug('deflate brotli body') if log unless defined?(::Brotli) raise Mechanize::Error, "cannot deflate brotli-encoded response. Please install and require the 'brotli' gem." end begin return StringIO.new(Brotli.inflate(body_io.read)) rescue Brotli::Error log.error("unable to brotli-inflate response") if log raise Mechanize::Error, "error inflating brotli-encoded response." end ensure body_io.close end
Decodes a Brotli-encoded body_io
(Experimental, CRuby only) Although Mechanize
will never request a Brotli-encoded response via ‘accept-encoding`, buggy servers may return brotli-encoded responses anyway. Let’s try to handle that case if the Brotli gem is loaded.
If you need to handle Brotli-encoded responses, install the ‘brotli’ gem and require it in your application. If the ‘Brotli` constant is defined, Mechanize
will attempt to use it to inflate the response.
Source
# File lib/mechanize/http/agent.rb, line 447 def content_encoding_gunzip body_io log.debug('gzip response') if log zio = Zlib::GzipReader.new body_io out_io = auto_io 'mechanize-gunzip', 16384, zio zio.finish return out_io rescue Zlib::Error => gz_error log.warn "unable to gunzip response: #{gz_error} (#{gz_error.class})" if log body_io.rewind body_io.read 10 begin log.warn "trying raw inflate on response" if log return inflate body_io, -Zlib::MAX_WBITS rescue Zlib::Error => e log.error "unable to inflate response: #{e} (#{e.class})" if log raise end ensure # do not close a second time if we failed the first time zio.close if zio and !(zio.closed? or gz_error) body_io.close unless body_io.closed? end
Decodes a gzip-encoded body_io
. If it cannot be decoded, inflate is tried followed by raising an error.
Source
# File lib/mechanize/http/agent.rb, line 479 def content_encoding_inflate body_io log.debug('deflate body') if log return inflate body_io rescue Zlib::Error log.error('unable to inflate response, trying raw deflate') if log body_io.rewind begin return inflate body_io, -Zlib::MAX_WBITS rescue Zlib::Error => e log.error("unable to inflate response: #{e}") if log raise end ensure body_io.close end
Decodes a deflate-encoded body_io
. If it cannot be decoded, raw inflate is tried followed by raising an error.
Source
# File lib/mechanize/http/agent.rb, line 538 def content_encoding_zstd(body_io) log.debug('deflate zstd body') if log unless defined?(::Zstd) raise Mechanize::Error, "cannot deflate zstd-encoded response. Please install and require the 'zstd-ruby' gem." end begin return StringIO.new(Zstd.decompress(body_io.read)) rescue StandardError log.error("unable to zstd#decompress response") if log raise Mechanize::Error, "error decompressing zstd-encoded response." end ensure body_io.close end
Decodes a Zstd-encoded body_io
(Experimental, CRuby only) Although Mechanize
will never request a zstd-encoded response via ‘accept-encoding`, buggy servers may return zstd-encoded responses, or you might need to inform the zstd keyword on your Accept-Encoding headers. Let’s try to handle those cases if the Zstd gem is loaded.
If you need to handle Zstd-encoded responses, install the ‘zstd-ruby’ gem and require it in your application. If the ‘Zstd` constant is defined, Mechanize
will attempt to use it to inflate the response.
Source
# File lib/mechanize/http/agent.rb, line 555 def disable_keep_alive request request['connection'] = 'close' unless @keep_alive end
Source
# File lib/mechanize/http/agent.rb, line 559 def enable_gzip request request['accept-encoding'] = if @gzip_enabled 'gzip,deflate,identity' else 'identity' end end
Source
# File lib/mechanize/http/agent.rb, line 567 def http_request uri, method, params = nil case uri.scheme.downcase when 'http', 'https' then klass = Net::HTTP.const_get(method.to_s.capitalize) request ||= klass.new(uri.request_uri) request.body = params.first if params request when 'file' then Mechanize::FileRequest.new uri end end
Source
# File lib/mechanize/http/agent.rb, line 581 def request_add_headers request, headers = {} @request_headers.each do |k,v| request[k] = v end headers.each do |field, value| case field when :etag then request["ETag"] = value when :if_modified_since then request["If-Modified-Since"] = value when Symbol then raise ArgumentError, "unknown header symbol #{field}" else request[field] = value end end end
Source
# File lib/mechanize/http/agent.rb, line 598 def request_auth request, uri base_uri = uri + '/' base_uri.user &&= nil base_uri.password &&= nil schemes = @authenticate_methods[base_uri] if realm = schemes[:digest].find { |r| r.uri == base_uri } then request_auth_digest request, uri, realm, base_uri, false elsif realm = schemes[:iis_digest].find { |r| r.uri == base_uri } then request_auth_digest request, uri, realm, base_uri, true elsif realm = schemes[:basic].find { |r| r.uri == base_uri } then user, password, = @auth_store.credentials_for uri, realm.realm request.basic_auth user, password end end
Source
# File lib/mechanize/http/agent.rb, line 614 def request_auth_digest request, uri, realm, base_uri, iis challenge = @digest_challenges[realm] uri.user, uri.password, = @auth_store.credentials_for uri, realm.realm auth = @digest_auth.auth_header uri, challenge.to_s, request.method, iis request['Authorization'] = auth end
Source
# File lib/mechanize/http/agent.rb, line 633 def request_host request, uri port = [80, 443].include?(uri.port.to_i) ? nil : uri.port host = uri.host request['Host'] = [host, port].compact.join ':' end
Source
# File lib/mechanize/http/agent.rb, line 640 def request_language_charset request request['accept-language'] = 'en-us,en;q=0.5' end
Source
# File lib/mechanize/http/agent.rb, line 645 def request_log request return unless log log.info("#{request.class}: #{request.path}") request.each_header do |k, v| log.debug("request-header: #{k} => #{v}") end end
Log specified headers for the request
Source
# File lib/mechanize/http/agent.rb, line 658 def request_referer request, uri, referer return unless referer return if 'https'.casecmp(referer.scheme) == 0 and 'https'.casecmp(uri.scheme) != 0 if referer.fragment || referer.user || referer.password referer = referer.dup referer.fragment = referer.user = referer.password = nil end request['Referer'] = referer end
Sets a Referer header. Fragment part is removed as demanded by RFC 2616 14.36, and user information part is removed just like major browsers do.
Source
# File lib/mechanize/http/agent.rb, line 669 def request_user_agent request request['User-Agent'] = @user_agent if @user_agent end
Source
# File lib/mechanize/http/agent.rb, line 673 def resolve(uri, referer = current_page) referer_uri = referer && referer.uri if uri.is_a?(URI) uri = uri.dup elsif uri.nil? if referer_uri return referer_uri end raise ArgumentError, "absolute URL needed (not nil)" else url = uri.to_s.strip if url.empty? if referer_uri return referer_uri.dup.tap { |u| u.fragment = nil } end raise ArgumentError, "absolute URL needed (not #{uri.inspect})" end url.gsub!(/[^#{0.chr}-#{126.chr}]/o) { |match| Mechanize::Util.uri_escape(match) } escaped_url = Mechanize::Util.html_unescape( url.split(/((?:%[0-9A-Fa-f]{2})+|#)/).each_slice(2).map { |x, y| "#{WEBrick::HTTPUtils.escape(x)}#{y}" }.join('') ) begin uri = URI.parse(escaped_url) rescue uri = URI.parse(WEBrick::HTTPUtils.escape(escaped_url)) end end uri.host = referer_uri.host if referer_uri && URI::HTTP === uri && uri.host.nil? scheme = uri.relative? ? 'relative' : uri.scheme.downcase uri = @scheme_handlers[scheme].call(uri, referer) if uri.relative? raise ArgumentError, "absolute URL needed (not #{uri})" unless referer_uri if referer.respond_to?(:bases) && referer.parser && (lbase = referer.bases.last) && lbase.uri && lbase.uri.absolute? base = lbase else base = nil end base = referer_uri + (base ? base.uri : referer_uri) # Workaround for URI's bug in that it squashes consecutive # slashes. See #304. if uri.path.match(%r{\A(.*?/)(?!/\.\.?(?!/))(/.*)\z}i) uri = URI((base + $1).to_s + $2) else uri = base + uri end # Strip initial "/.." bits from the path uri.path.sub!(/^(\/\.\.)+(?=\/)/, '') end unless ['http', 'https', 'file'].include?(uri.scheme.downcase) raise ArgumentError, "unsupported scheme: #{uri.scheme}" end case uri.path when nil raise ArgumentError, "hierarchical URL needed (not #{uri})" when ''.freeze uri.path = '/' end uri end
Source
# File lib/mechanize/http/agent.rb, line 764 def resolve_parameters uri, method, parameters case method when :head, :get, :delete, :trace then if parameters and parameters.length > 0 uri.query ||= '' uri.query << '&' if uri.query.length > 0 uri.query << Mechanize::Util.build_query_string(parameters) end return uri, nil end return uri, parameters end
Source
# File lib/mechanize/http/agent.rb, line 752 def secure_resolve!(uri, referer = current_page) new_uri = resolve(uri, referer) if (referer_uri = referer && referer.uri) && referer_uri.scheme != 'file'.freeze && new_uri.scheme == 'file'.freeze raise Mechanize::Error, "insecure redirect to a file URI" end new_uri end
Source
# File lib/mechanize/http/agent.rb, line 439 def shutdown http.shutdown end
Closes all open connections for this agent.
Response
↑ topPublic Instance Methods
Source
# File lib/mechanize/http/agent.rb, line 781 def get_meta_refresh response, uri, page return nil unless @follow_meta_refresh if page.respond_to?(:meta_refresh) and (redirect = page.meta_refresh.first) then [redirect.delay, redirect.href] unless not @follow_meta_refresh_self and redirect.link_self elsif refresh = response['refresh'] delay, href, link_self = Mechanize::Page::MetaRefresh.parse refresh, uri raise Mechanize::Error, 'Invalid refresh http header' unless delay [delay.to_f, href] unless not @follow_meta_refresh_self and link_self end end
Source
# File lib/mechanize/http/agent.rb, line 796 def response_authenticate(response, page, uri, request, headers, params, referer) www_authenticate = response['www-authenticate'] unless www_authenticate = response['www-authenticate'] then message = 'WWW-Authenticate header missing in response' raise Mechanize::UnauthorizedError.new(page, nil, message) end challenges = @authenticate_parser.parse www_authenticate unless @auth_store.credentials? uri, challenges then message = "no credentials found, provide some with #add_auth" raise Mechanize::UnauthorizedError.new(page, challenges, message) end if challenge = challenges.find { |c| c.scheme =~ /^Digest$/i } then realm = challenge.realm uri auth_scheme = if response['server'] =~ /Microsoft-IIS/ then :iis_digest else :digest end existing_realms = @authenticate_methods[realm.uri][auth_scheme] if existing_realms.include? realm message = 'Digest authentication failed' raise Mechanize::UnauthorizedError.new(page, challenges, message) end existing_realms << realm @digest_challenges[realm] = challenge =begin elsif challenge = challenges.find { |c| c.scheme == 'NTLM' } then existing_realms = @authenticate_methods[uri + '/'][:ntlm] if existing_realms.include?(realm) and not challenge.params then message = 'NTLM authentication failed' raise Mechanize::UnauthorizedError.new(page, challenges, message) end existing_realms << realm if challenge.params then type_2 = Net::NTLM::Message.decode64 challenge.params user, password, domain = @auth_store.credentials_for uri, nil type_3 = type_2.response({ :user => user, :password => password, :domain => domain }, { :ntlmv2 => true }).encode64 headers['Authorization'] = "NTLM #{type_3}" else type_1 = Net::NTLM::Message::Type1.new.encode64 headers['Authorization'] = "NTLM #{type_1}" end =end elsif challenge = challenges.find { |c| c.scheme == 'Basic' } then realm = challenge.realm uri existing_realms = @authenticate_methods[realm.uri][:basic] if existing_realms.include? realm then message = 'Basic authentication failed' raise Mechanize::UnauthorizedError.new(page, challenges, message) end existing_realms << realm else message = 'unsupported authentication scheme' raise Mechanize::UnauthorizedError.new(page, challenges, message) end fetch uri, request.method.downcase.to_sym, headers, params, referer end
Source
# File lib/mechanize/http/agent.rb, line 875 def response_content_encoding response, body_io length = response.content_length || case body_io when Tempfile, IO then body_io.stat.size else body_io.length end return body_io if length.zero? out_io = case response['Content-Encoding'] when nil, 'none', '7bit', 'identity', "" then body_io when 'deflate' then content_encoding_inflate body_io when 'gzip', 'x-gzip' then content_encoding_gunzip body_io when 'br' then content_encoding_brotli body_io when 'zstd' then content_encoding_zstd body_io else raise Mechanize::Error, "unsupported content-encoding: #{response['Content-Encoding']}" end out_io.flush out_io.rewind out_io rescue Zlib::Error => e message = String.new("error handling content-encoding #{response['Content-Encoding']}:") message << " #{e.message} (#{e.class})" raise Mechanize::Error, message ensure begin if Tempfile === body_io and (StringIO === out_io or (out_io and out_io.path != body_io.path)) then body_io.close! end rescue IOError # HACK ruby 1.8 raises IOError when closing the stream end end
Source
# File lib/mechanize/http/agent.rb, line 949 def response_follow_meta_refresh response, uri, page, redirects delay, new_url = get_meta_refresh(response, uri, page) return nil unless delay new_url = new_url ? secure_resolve!(new_url, page) : uri raise Mechanize::RedirectLimitReachedError.new(page, redirects) if redirects + 1 > @redirection_limit sleep delay @history.push(page, page.uri) fetch new_url, :get, {}, [], Mechanize::Page.new, redirects + 1 end
Source
# File lib/mechanize/http/agent.rb, line 963 def response_log response return unless log log.info("status: #{response.class} #{response.http_version} " \ "#{response.code} #{response.message}") response.each_header do |k, v| log.debug("response-header: #{k} => #{v}") end end
Source
# File lib/mechanize/http/agent.rb, line 974 def response_parse response, body_io, uri @context.parse uri, response, body_io end
Source
# File lib/mechanize/http/agent.rb, line 978 def response_read response, request, uri content_length = response.content_length if use_tempfile? content_length then body_io = make_tempfile 'mechanize-raw' else body_io = StringIO.new.set_encoding(Encoding::BINARY) end total = 0 begin response.read_body { |part| total += part.length if StringIO === body_io and use_tempfile? total then new_io = make_tempfile 'mechanize-raw' new_io.write body_io.string body_io = new_io end body_io.write(part) log.debug("Read #{part.length} bytes (#{total} total)") if log } rescue EOFError => e # terminating CRLF might be missing, let the user check the document raise unless response.chunked? and total.nonzero? body_io.rewind raise Mechanize::ChunkedTerminationError.new(e, response, body_io, uri, @context) rescue Net::HTTP::Persistent::Error, Errno::ECONNRESET => e body_io.rewind raise Mechanize::ResponseReadError.new(e, response, body_io, uri, @context) end body_io.flush body_io.rewind raise Mechanize::ResponseCodeError.new(response, uri) if Net::HTTPUnknownResponse === response content_length = response.content_length unless Net::HTTP::Head === request or Net::HTTPRedirection === response then if content_length and content_length != body_io.length err = EOFError.new("Content-Length (#{content_length}) does not " \ "match response body length (#{body_io.length})") raise Mechanize::ResponseReadError.new(err, response, body_io, uri, @context) end end body_io end
Source
# File lib/mechanize/http/agent.rb, line 1037 def response_redirect(response, method, page, redirects, headers, referer = current_page) case @redirect_ok when true, :all # shortcut when false, nil return page when :permanent return page unless Net::HTTPMovedPermanently === response end log.info("follow redirect to: #{response['Location']}") if log raise Mechanize::RedirectLimitReachedError.new(page, redirects) if redirects + 1 > @redirection_limit redirect_method = method == :head ? :head : :get new_uri = secure_resolve!(response['Location'].to_s, page) @history.push(page, page.uri) # Make sure we are not copying over the POST headers from the original request POST_HEADERS.each do |key| headers.delete_if { |h| h.casecmp?(key) } end # Make sure we clear credential headers if being redirected to another site if new_uri.host == page.uri.host if new_uri.port != page.uri.port # https://datatracker.ietf.org/doc/html/rfc6265#section-8.5 # cookies are OK to be shared across ports on the same host CREDENTIAL_HEADERS.each { |ch| headers.delete_if { |h| h.casecmp?(ch) } } end else (COOKIE_HEADERS + CREDENTIAL_HEADERS).each { |ch| headers.delete_if { |h| h.casecmp?(ch) } } end fetch new_uri, redirect_method, headers, [], referer, redirects + 1 end
Robots
↑ topConstants
- RobotsKey
Attributes
When true, this agent will consult the site’s robots.txt for each access.
Mutex used when fetching robots.txt
Public Instance Methods
Source
# File lib/mechanize/http/agent.rb, line 1101 def robots= value require 'webrobots' if value @webrobots = nil if value != @robots @robots = value end
Source
# File lib/mechanize/http/agent.rb, line 1111 def robots_allowed? uri return true if Thread.current[RobotsKey] webrobots.allowed? uri end
Tests if this agent is allowed to access url
, consulting the site’s robots.txt.
Source
# File lib/mechanize/http/agent.rb, line 1119 def robots_disallowed? url !robots_allowed? url end
Opposite of robots_allowed?
Source
# File lib/mechanize/http/agent.rb, line 1125 def robots_error(url) webrobots.error(url) end
Returns an error object if there is an error in fetching or parsing robots.txt of the site url
.
Source
# File lib/mechanize/http/agent.rb, line 1131 def robots_error!(url) webrobots.error!(url) end
Raises the error if there is an error in fetching or parsing robots.txt of the site url
.
Source
# File lib/mechanize/http/agent.rb, line 1136 def robots_reset(url) webrobots.reset(url) end
Removes robots.txt cache for the site url
.
Source
# File lib/mechanize/http/agent.rb, line 1140 def webrobots @webrobots ||= WebRobots.new(@user_agent, :http_get => method(:get_robots)) end
SSL
↑ topAttributes
OpenSSL key password
Public Instance Methods
Source
# File lib/mechanize/http/agent.rb, line 1147 def ca_file @http.ca_file end
Path to an OpenSSL CA certificate file
Source
# File lib/mechanize/http/agent.rb, line 1152 def ca_file= ca_file @http.ca_file = ca_file end
Sets the path to an OpenSSL CA certificate file
Source
# File lib/mechanize/http/agent.rb, line 1157 def cert_store @http.cert_store end
The SSL certificate store used for validating connections
Source
# File lib/mechanize/http/agent.rb, line 1162 def cert_store= cert_store @http.cert_store = cert_store end
Sets the SSL certificate store used for validating connections
Source
# File lib/mechanize/http/agent.rb, line 1167 def certificate @http.certificate end
The client X509 certificate
Source
# File lib/mechanize/http/agent.rb, line 1173 def certificate= certificate certificate = if OpenSSL::X509::Certificate === certificate then certificate else OpenSSL::X509::Certificate.new File.read certificate end @http.certificate = certificate end
Sets the client certificate to given X509 certificate. If a path is given the certificate will be loaded and set.
Source
# File lib/mechanize/http/agent.rb, line 1184 def private_key @http.private_key end
An OpenSSL private key or the path to a private key
Source
# File lib/mechanize/http/agent.rb, line 1189 def private_key= private_key private_key = if OpenSSL::PKey::PKey === private_key then private_key else OpenSSL::PKey::RSA.new File.read(private_key), @pass end @http.private_key = private_key end
Sets the client’s private key
Source
# File lib/mechanize/http/agent.rb, line 1200 def ssl_version @http.ssl_version end
SSL version to use
Source
# File lib/mechanize/http/agent.rb, line 1205 def ssl_version= ssl_version @http.ssl_version = ssl_version end
Sets the SSL version to use
Source
# File lib/mechanize/http/agent.rb, line 1215 def verify_callback @http.verify_callback end
A callback for additional certificate verification. See OpenSSL::SSL::SSLContext#verify_callback
The callback can be used for debugging or to ignore errors by always returning true
. Specifying nil uses the default method that was valid when the SSLContext was created
Source
# File lib/mechanize/http/agent.rb, line 1220 def verify_callback= verify_callback @http.verify_callback = verify_callback end
Sets the certificate verify callback
Source
# File lib/mechanize/http/agent.rb, line 1225 def verify_mode @http.verify_mode end
How to verify SSL connections. Defaults to VERIFY_PEER
Source
# File lib/mechanize/http/agent.rb, line 1230 def verify_mode= verify_mode @http.verify_mode = verify_mode end
Sets the mode for verifying SSL connections
Timeouts
↑ topAttributes
Set to false to disable HTTP/1.1 keep-alive requests
Length of time to wait until a connection is opened in seconds
Length of time to attempt to read data from the server
Public Instance Methods
Source
# File lib/mechanize/http/agent.rb, line 1237 def idle_timeout @http.idle_timeout end
Reset connections that have not been used in this many seconds
Source
# File lib/mechanize/http/agent.rb, line 1242 def idle_timeout= timeout @http.idle_timeout = timeout end
Sets the connection idle timeout for persistent connections
Utility
↑ topAttributes
The context parses responses into pages
When set to true mechanize will ignore an EOF during chunked transfer encoding so long as at least one byte was received. Be careful when enabling this as it may cause data loss.
Handlers for various URI schemes
Public Instance Methods
Source
# File lib/mechanize/http/agent.rb, line 1256 def auto_io name, read_size, input_io out_io = StringIO.new.set_encoding(Encoding::BINARY) until input_io.eof? do if StringIO === out_io and use_tempfile? out_io.size then new_io = make_tempfile name new_io.write out_io.string out_io = new_io end chunk = input_io.read read_size chunk = yield chunk if block_given? out_io.write chunk end out_io.rewind out_io end
Creates a new output IO by reading input_io
in read_size
chunks. If the output is over the max_file_buffer
size a Tempfile with name
is created.
If a block is provided, each chunk of input_io
is yielded for further processing.
Source
# File lib/mechanize/http/agent.rb, line 1277 def inflate compressed, window_bits = nil inflate = Zlib::Inflate.new window_bits out_io = auto_io 'mechanize-inflate', 1024, compressed do |chunk| inflate.inflate chunk end inflate.finish out_io ensure inflate.close if inflate.finished? end
Source
# File lib/mechanize/http/agent.rb, line 1335 def make_tempfile name io = Tempfile.new name io.unlink io.binmode io end
Source
# File lib/mechanize/http/agent.rb, line 1349 def reset @cookie_jar.clear @history.clear end
Source
# File lib/mechanize/http/agent.rb, line 1300 def set_proxy addr, port = nil, user = nil, pass = nil case addr when URI::HTTP proxy_uri = addr.dup when %r{\Ahttps?://}i proxy_uri = URI addr when String proxy_uri = URI "http://#{addr}" when nil @http.proxy = nil return end case port when Integer proxy_uri.port = port when nil else begin proxy_uri.port = Socket.getservbyname port rescue SocketError begin proxy_uri.port = Integer port rescue ArgumentError raise ArgumentError, "invalid value for port: #{port.inspect}" end end end proxy_uri.user = user if user proxy_uri.password = pass if pass @http.proxy = proxy_uri end
Sets the proxy address, port, user, and password. addr
may be an HTTP
URL/URI or a host name, port
may be a port number, service name or port number string.
Source
# File lib/mechanize/http/agent.rb, line 1342 def use_tempfile? size return false unless @max_file_buffer return false unless size size >= @max_file_buffer end