class ByeFlickr::PhotoDownloader
Constants
- Download
Attributes
errors[R]
Public Class Methods
new(dir, workers: 2)
click to toggle source
# File lib/bye_flickr/photo_downloader.rb, line 14 def initialize(dir, workers: 2) @basedir = dir @lock = Mutex.new @http = Net::HTTP::Persistent.new @images = Concurrent::Array.new @errors = Concurrent::Array.new @tempdir = @basedir.join 'tmp' FileUtils.mkdir_p @tempdir @running = true @workers = 1.upto(workers).map do |i| Thread.new{ Worker.new(self).run } end end
Public Instance Methods
add_failure(dl)
click to toggle source
# File lib/bye_flickr/photo_downloader.rb, line 50 def add_failure(dl) @errors << dl end
add_image(url, path)
click to toggle source
# File lib/bye_flickr/photo_downloader.rb, line 38 def add_image(url, path) @images << Download.new(url, path, 0) end
download(dl)
click to toggle source
# File lib/bye_flickr/photo_downloader.rb, line 54 def download(dl) response = @http.request dl.url f = Tempfile.create('bye-flickr-download', @tempdir) f << response.body f.close @lock.synchronize do i = 0 path = dl.path while File.readable?(path) i = i+1 path = "#{dl.path.sub(/\.jpg$/i, '')}_#{i}.jpg" end FileUtils.mv f, path end rescue puts "#{$!}:\n#{dl.url} => #{dl.path}" dl.tries += 1 if dl.tries > 2 puts "giving up on this one" add_failure dl else @images << dl end end
next()
click to toggle source
# File lib/bye_flickr/photo_downloader.rb, line 46 def next @images.shift end
running?()
click to toggle source
# File lib/bye_flickr/photo_downloader.rb, line 42 def running? !!@running end
wait()
click to toggle source
# File lib/bye_flickr/photo_downloader.rb, line 30 def wait @running = false @workers.each{|t|t.join} if @errors.any? (File.open(@basedir.join('errors.json'), 'wb') << @errors.to_json).close end end