class S3Toolkit::Fetcher
Public Class Methods
call(region:, s3_url:, dest_dir:, request_concurrency:, overwrite:, client: nil)
click to toggle source
# File lib/s3-toolkit/fetcher.rb, line 5 def self.call(region:, s3_url:, dest_dir:, request_concurrency:, overwrite:, client: nil) files = 0 FileUtils.mkdir_p(dest_dir) queue = Queue.new bucket, key = S3Toolkit::UrlParser.call(s3_url) s3 = client || Aws::S3::Client.new(region: region) resp = s3.list_objects(bucket: bucket, prefix: key) resp.contents.each { |content| queue.push(content.key) } puts "Found #{queue.size} files" request_concurrency.times.map do Thread.new do until queue.empty? download = queue.pop base_name = File.basename(download) destination = "#{dest_dir}/#{base_name}" next unless !File.exist?(destination) || overwrite puts "download #{s3_url}/#{base_name} to #{destination}" File.open(destination, "wb") do |file| resp = s3.get_object({bucket: bucket, key: download}, target: file) files += 1 end end Thread.exit end end.each(&:join) files end