class Finviz::Crawler

Async way to fetch the data

Attributes

paths[R]

Public Class Methods

call(*args) click to toggle source
# File lib/finviz/crawler.rb, line 12
def call(*args)
  new(*args).tap(&:fetch).result
end
new(paths: []) click to toggle source
# File lib/finviz/crawler.rb, line 17
def initialize(paths: [])
  @paths = Array(paths)
end

Public Instance Methods

fetch() click to toggle source
# File lib/finviz/crawler.rb, line 23
def fetch
  async do
    paths.each do |path|
      response = internet.get(path.to_s, headers)
      result << OpenStruct.new(path: path, html: Nokogiri::HTML(response.read))
    end
  end
end
result() click to toggle source
# File lib/finviz/crawler.rb, line 32
def result
  @result ||= []
end

Private Instance Methods

async() { |task| ... } click to toggle source
# File lib/finviz/crawler.rb, line 38
def async
  Async do |task|
    # Spawn an asynchronous task for each topic:
    task.with_timeout(Finviz.config.timeout) do
      yield task

      # Ensure we wait for all requests to complete before continuing:
      barrier.wait
    end
  ensure
    internet&.close
  end
end
barrier() click to toggle source
# File lib/finviz/crawler.rb, line 56
def barrier
  @barrier ||= Async::Barrier.new
end
headers() click to toggle source
# File lib/finviz/crawler.rb, line 60
def headers
  [
    [
      "User-Agent",
      "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:#{rand(47..80)}.0) Gecko/20100101 Firefox/#{rand(47..80)}.0"
    ]
  ]
end
internet() click to toggle source
# File lib/finviz/crawler.rb, line 52
def internet
  @internet ||= Async::HTTP::Internet.new
end