class WonderScrape::Scrapers::MFC::Scraper
Constants
- BASE_URL
- DEFAULT_DELAY_BETWEEN_REQUESTS
- DEFAULT_MAX_PAGES
- DEFAULT_SEARCH_CATEGORY
- DEFAULT_START_PAGE
- FIELDS
- NAME
- RESULTS_PER_PAGE
- SEARCH_PATH
- SEARCH_RESULT_ITEM_SELECTOR
Attributes
options[R]
recorder[R]
writer[R]
Public Class Methods
new(writer, recorder, options = {})
click to toggle source
# File lib/wonder_scrape/scrapers/mfc/scraper.rb, line 22 def initialize(writer, recorder, options = {}) @writer = writer @recorder = recorder @options = options end
Public Instance Methods
scrape()
click to toggle source
# File lib/wonder_scrape/scrapers/mfc/scraper.rb, line 28 def scrape scraper.scrape(&ItemParser.parse(writer, recorder)) end
Private Instance Methods
build_scraper()
click to toggle source
# File lib/wonder_scrape/scrapers/mfc/scraper.rb, line 40 def build_scraper new_scraper = Upton::Scraper.new( search_url, SEARCH_RESULT_ITEM_SELECTOR ) new_scraper.paginated = true new_scraper.pagination_start_index = options[:start_page] || DEFAULT_START_PAGE new_scraper.pagination_max_pages = options[:num_pages] || DEFAULT_MAX_PAGES new_scraper.verbose = options[:verbose] || false new_scraper.sleep_time_between_requests = options[:request_delay] || DEFAULT_DELAY_BETWEEN_REQUESTS new_scraper end
build_search_query_params()
click to toggle source
# File lib/wonder_scrape/scrapers/mfc/scraper.rb, line 63 def build_search_query_params URI.encode_www_form({ 'mode': 'search', 'categoryId': DEFAULT_SEARCH_CATEGORY, 'sort': 'date', 'order': 'desc' }) end
scraper()
click to toggle source
# File lib/wonder_scrape/scrapers/mfc/scraper.rb, line 36 def scraper @scraper ||= build_scraper end
search_url()
click to toggle source
# File lib/wonder_scrape/scrapers/mfc/scraper.rb, line 55 def search_url URI::HTTPS.build( host: BASE_URL, path: SEARCH_PATH, query: build_search_query_params ).to_s end