class ByeFlickr::PhotoDownloader

Constants

Download

Attributes

errors[R]

Public Class Methods

new(dir, workers: 2) click to toggle source
# File lib/bye_flickr/photo_downloader.rb, line 14
def initialize(dir, workers: 2)
  @basedir = dir
  @lock = Mutex.new
  @http = Net::HTTP::Persistent.new

  @images = Concurrent::Array.new
  @errors = Concurrent::Array.new

  @tempdir = @basedir.join 'tmp'
  FileUtils.mkdir_p @tempdir
  @running = true
  @workers = 1.upto(workers).map do |i|
    Thread.new{ Worker.new(self).run }
  end
end

Public Instance Methods

add_failure(dl) click to toggle source
# File lib/bye_flickr/photo_downloader.rb, line 50
def add_failure(dl)
  @errors << dl
end
add_image(url, path) click to toggle source
# File lib/bye_flickr/photo_downloader.rb, line 38
def add_image(url, path)
  @images << Download.new(url, path, 0)
end
download(dl) click to toggle source
# File lib/bye_flickr/photo_downloader.rb, line 54
def download(dl)
  response = @http.request dl.url
  f = Tempfile.create('bye-flickr-download', @tempdir)
  f << response.body
  f.close

  @lock.synchronize do
    i = 0
    path = dl.path
    while File.readable?(path)
      i = i+1
      path = "#{dl.path.sub(/\.jpg$/i, '')}_#{i}.jpg"
    end
    FileUtils.mv f, path
  end

rescue
  puts "#{$!}:\n#{dl.url} => #{dl.path}"
  dl.tries += 1
  if dl.tries > 2
    puts "giving up on this one"
    add_failure dl
  else
    @images << dl
  end
end
next() click to toggle source
# File lib/bye_flickr/photo_downloader.rb, line 46
def next
  @images.shift
end
running?() click to toggle source
# File lib/bye_flickr/photo_downloader.rb, line 42
def running?
  !!@running
end
wait() click to toggle source
# File lib/bye_flickr/photo_downloader.rb, line 30
def wait
  @running = false
  @workers.each{|t|t.join}
  if @errors.any?
    (File.open(@basedir.join('errors.json'), 'wb') << @errors.to_json).close
  end
end