class WonderScrape::Scrapers::MFC::Scraper

Constants

BASE_URL
DEFAULT_DELAY_BETWEEN_REQUESTS
DEFAULT_MAX_PAGES
DEFAULT_SEARCH_CATEGORY
DEFAULT_START_PAGE
FIELDS
NAME
RESULTS_PER_PAGE
SEARCH_PATH
SEARCH_RESULT_ITEM_SELECTOR

Attributes

options[R]
recorder[R]
writer[R]

Public Class Methods

new(writer, recorder, options = {}) click to toggle source
# File lib/wonder_scrape/scrapers/mfc/scraper.rb, line 22
def initialize(writer, recorder, options = {})
  @writer = writer
  @recorder = recorder 
  @options = options
end

Public Instance Methods

scrape() click to toggle source
# File lib/wonder_scrape/scrapers/mfc/scraper.rb, line 28
def scrape
  scraper.scrape(&ItemParser.parse(writer, recorder))
end

Private Instance Methods

build_scraper() click to toggle source
# File lib/wonder_scrape/scrapers/mfc/scraper.rb, line 40
def build_scraper
  new_scraper = Upton::Scraper.new(
    search_url,
    SEARCH_RESULT_ITEM_SELECTOR
  )

  new_scraper.paginated = true
  new_scraper.pagination_start_index = options[:start_page] || DEFAULT_START_PAGE
  new_scraper.pagination_max_pages = options[:num_pages] || DEFAULT_MAX_PAGES
  new_scraper.verbose = options[:verbose] || false
  new_scraper.sleep_time_between_requests = options[:request_delay] || DEFAULT_DELAY_BETWEEN_REQUESTS

  new_scraper
end
build_search_query_params() click to toggle source
# File lib/wonder_scrape/scrapers/mfc/scraper.rb, line 63
def build_search_query_params
  URI.encode_www_form({
                        'mode': 'search',
                        'categoryId': DEFAULT_SEARCH_CATEGORY,
                        'sort': 'date',
                        'order': 'desc'
                      })
end
scraper() click to toggle source
# File lib/wonder_scrape/scrapers/mfc/scraper.rb, line 36
def scraper
  @scraper ||= build_scraper
end
search_url() click to toggle source
# File lib/wonder_scrape/scrapers/mfc/scraper.rb, line 55
def search_url
  URI::HTTPS.build(
    host: BASE_URL,
    path: SEARCH_PATH,
    query: build_search_query_params
  ).to_s
end