class DaimonSkycrawlers::Storage::File
Storage
for files
Constants
- Page
Page
for file storage
Public Class Methods
new(base_dir)
click to toggle source
Calls superclass method
DaimonSkycrawlers::ConfigMixin::new
# File lib/daimon_skycrawlers/storage/file.rb, line 9 def initialize(base_dir) super() @base_dir = Pathname(base_dir) end
Public Instance Methods
read(message)
click to toggle source
Read data from files under base directory
@param message [Hash] this hash can include `:url`, `:key` to find page @return [DaimonSkycrawlers::Storage::File::Page]
# File lib/daimon_skycrawlers/storage/file.rb, line 45 def read(message) url = message[:url] key = message[:key] headers = JSON.parse(headers_path(url, key).read) body = body_path(url, key).read Page.new(url, key, headers, body, headers["last-modified"], headers["etag"]) end
save(data)
click to toggle source
Save data to files under base directory
@param data [Hash] data has following keys
* `:url`: URL * `:message`: Given message * `:response`: HTTP response
# File lib/daimon_skycrawlers/storage/file.rb, line 22 def save(data) url = data[:url] message = data[:message] key = message[:key] response = data[:response] headers = response.headers body = response.body @base_dir.mkpath body_path(url, key).dirname.mkpath body_path(url, key).open("wb+") do |file| file.write(body) end headers_path(url, key).open("wb+") do |file| file.write(JSON.generate(headers)) end end
Private Instance Methods
body_path(url, key)
click to toggle source
# File lib/daimon_skycrawlers/storage/file.rb, line 60 def body_path(url, key) url = URI(url) if key @base_dir + key else @base_dir + ".#{url.path}" end end
headers_path(url, key)
click to toggle source
# File lib/daimon_skycrawlers/storage/file.rb, line 69 def headers_path(url, key) url = URI(url) Pathname("#{body_path(url, key)}-headers.json") end