class Gogdb::Engine
Public Class Methods
new(options={})
click to toggle source
# File lib/gogdb/engine.rb, line 4 def initialize(options={}) @logger = Logger.new(options) @utils = Utils.new(@logger) end
Public Instance Methods
fetch(options)
click to toggle source
Downloads data from source and updates database with changes
@param [Hash] options Additional options such as :limit and :type
# File lib/gogdb/engine.rb, line 12 def fetch(options) # If user passed type parameter, we need to figure it out which urls we # need actually to fetch sources = [] sources.push(GOG_GAMES_URL) if options[:type] == "games" sources.push(GOG_MOVIES_URL) if options[:type] == "movies" sources = [GOG_GAMES_URL, GOG_MOVIES_URL] if options[:type] == "all" || options[:type] == nil sleep_time = 60 / options[:requestLimit] sources.each do |source| pages_number = get_pages_number(source) @logger.debug "Pages received: #{pages_number}" nil if pages_number < 1 # Do nothing if there are no pages # Loop through pages count = 0; catch :limitReached do for i in 0..pages_number do # Let's get products number and URL's for current page items_data = get_data(source)['products'] items_number = items_data.length # Loop through products and get data of every one for i in 0..items_number do item_url = items_data[i]['url'] item_data = get_data("#{GOG_URL}#{item_url}")['gameProductData'] @logger.debug "Item received: #{item_data['title']}" count += 1 # Break loop if limit is reached throw :limitReached if count >= options[:limit] && options[:limit] != 0 sleep(sleep_time) end end end end end
get_data(url)
click to toggle source
Gets data from source and parses global javascript object.
@param [string] url Page URL @return [hash]
# File lib/gogdb/engine.rb, line 71 def get_data(url) begin page = Net::HTTP.get(URI(url)) JSON.parse(page[/(?<=var gogData = )(.*)(?=;)/,1]) rescue => e @logger.warning "Cannot retrieve or parse data from gog.com" @logger.error e.message # In case crawler cannot access gog.com, let's check if gog.com is # actually online. ph = Net::Ping::HTTP.new(url) unless ph.ping? @logger.warning "Cannot establish connection to gog.com. Retrying..." # Retry connection until gog.com comes online. After that - retry # getting data. @utils.retryConnection(url) retry else @logger.warning "Connection to gog.com established successfully. Retrying..." sleep(15) retry end end end
get_pages_number(url)
click to toggle source
Gets pages number.
@param [string] url Page URL @return [integer] number of pages
# File lib/gogdb/engine.rb, line 57 def get_pages_number(url) begin get_data(url)["totalPages"] rescue => e @logger.error "Cannot retrieve pages number." @logger.debug e.message 0 end end