class Finviz::Crawler
Async way to fetch the data
Attributes
paths[R]
Public Class Methods
call(*args)
click to toggle source
# File lib/finviz/crawler.rb, line 12 def call(*args) new(*args).tap(&:fetch).result end
new(paths: [])
click to toggle source
# File lib/finviz/crawler.rb, line 17 def initialize(paths: []) @paths = Array(paths) end
Public Instance Methods
fetch()
click to toggle source
# File lib/finviz/crawler.rb, line 23 def fetch async do paths.each do |path| response = internet.get(path.to_s, headers) result << OpenStruct.new(path: path, html: Nokogiri::HTML(response.read)) end end end
result()
click to toggle source
# File lib/finviz/crawler.rb, line 32 def result @result ||= [] end
Private Instance Methods
async() { |task| ... }
click to toggle source
# File lib/finviz/crawler.rb, line 38 def async Async do |task| # Spawn an asynchronous task for each topic: task.with_timeout(Finviz.config.timeout) do yield task # Ensure we wait for all requests to complete before continuing: barrier.wait end ensure internet&.close end end
barrier()
click to toggle source
# File lib/finviz/crawler.rb, line 56 def barrier @barrier ||= Async::Barrier.new end
headers()
click to toggle source
# File lib/finviz/crawler.rb, line 60 def headers [ [ "User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:#{rand(47..80)}.0) Gecko/20100101 Firefox/#{rand(47..80)}.0" ] ] end
internet()
click to toggle source
# File lib/finviz/crawler.rb, line 52 def internet @internet ||= Async::HTTP::Internet.new end