class Sites::Base
Attributes
doc[R]
jobs_count[R]
rows_count[R]
url[R]
Public Class Methods
new()
click to toggle source
# File lib/sites/base.rb, line 6 def initialize @url = "#{self.class::HOST}#{self.class::PATH}" @doc = Nokogiri::HTML(open_page(@url)) @current_time = Time.now @timestamp = @current_time.strftime("%Y%m%d%H%M%S") @rows_count = 0 @jobs_count = get_jobs_count end
Private Instance Methods
delay_range()
click to toggle source
# File lib/sites/base.rb, line 23 def delay_range RemoteJobScraper.configuration.delay_range end
filepath()
click to toggle source
# File lib/sites/base.rb, line 31 def filepath return test_filepath if ENV["RAILS_ENV"] == 'test' "#{self.class::STORE_DIR}/#{@timestamp}.csv" end
open_page(url)
click to toggle source
# File lib/sites/base.rb, line 17 def open_page(url) sleep(rand(delay_range)) unless ENV['RAILS_ENV'] == 'test' # less mechanical behaviour options = ENV['RAILS_ENV'] == 'test' ? {} : { 'User-Agent' => user_agent } open(url, options) end
test_filepath()
click to toggle source
# File lib/sites/base.rb, line 36 def test_filepath "spec/fixtures/data/#{underscore(self.class.name.split('::').last)}/#{@timestamp}.csv" end
underscore(camel_cased_word)
click to toggle source
# File lib/sites/base.rb, line 41 def underscore(camel_cased_word) word = camel_cased_word.dup word.gsub!(/::/, '/') word.gsub!(/([A-Z]+)([A-Z][a-z])/,'\1_\2') word.gsub!(/([a-z\d])([A-Z])/,'\1_\2') word.tr!("-", "_") word.downcase! word end
user_agent()
click to toggle source
# File lib/sites/base.rb, line 27 def user_agent Support::UserAgent::LIST.sample end