class Sites::Base

Attributes

doc[R]
jobs_count[R]
rows_count[R]
url[R]

Public Class Methods

new() click to toggle source
# File lib/sites/base.rb, line 6
def initialize
  @url = "#{self.class::HOST}#{self.class::PATH}"
  @doc = Nokogiri::HTML(open_page(@url))
  @current_time = Time.now
  @timestamp = @current_time.strftime("%Y%m%d%H%M%S")
  @rows_count = 0
  @jobs_count = get_jobs_count
end

Private Instance Methods

delay_range() click to toggle source
# File lib/sites/base.rb, line 23
def delay_range
  RemoteJobScraper.configuration.delay_range
end
filepath() click to toggle source
# File lib/sites/base.rb, line 31
def filepath
  return test_filepath if ENV["RAILS_ENV"] == 'test'
  "#{self.class::STORE_DIR}/#{@timestamp}.csv"
end
open_page(url) click to toggle source
# File lib/sites/base.rb, line 17
def open_page(url)
  sleep(rand(delay_range)) unless ENV['RAILS_ENV'] == 'test' # less mechanical behaviour
  options = ENV['RAILS_ENV'] == 'test' ? {} : { 'User-Agent' => user_agent }
  open(url, options)
end
test_filepath() click to toggle source
# File lib/sites/base.rb, line 36
def test_filepath
  "spec/fixtures/data/#{underscore(self.class.name.split('::').last)}/#{@timestamp}.csv"
end
underscore(camel_cased_word) click to toggle source

stackoverflow.com/a/5622585

# File lib/sites/base.rb, line 41
def underscore(camel_cased_word)
  word = camel_cased_word.dup
  word.gsub!(/::/, '/')
  word.gsub!(/([A-Z]+)([A-Z][a-z])/,'\1_\2')
  word.gsub!(/([a-z\d])([A-Z])/,'\1_\2')
  word.tr!("-", "_")
  word.downcase!
  word
end
user_agent() click to toggle source
# File lib/sites/base.rb, line 27
def user_agent
  Support::UserAgent::LIST.sample
end