class NightcrawlerSwift::MultithreadSync
Constants
- DEFAULT_POOL_SIZE
Public Class Methods
new()
click to toggle source
# File lib/nightcrawler_swift/commands/multithread_sync.rb, line 5 def initialize require 'concurrent/utilities' require 'concurrent/executors' @logger = NightcrawlerSwift.logger end
Public Instance Methods
execute(args = {})
click to toggle source
# File lib/nightcrawler_swift/commands/multithread_sync.rb, line 11 def execute args = {} pool_size = args[:pool_size] || DEFAULT_POOL_SIZE dir_path = args[:dir_path] @logger.info "[NightcrawlerSwift] dir_path: #{dir_path}" @logger.info "[NightcrawlerSwift] multithread sync, #{Concurrent.processor_count} processors" assets = Dir["#{dir_path}/**/**"]. reject {|fullpath| File.directory?(fullpath)}. map {|fullpath| path = fullpath.gsub("#{dir_path}/", "") OpenStruct.new(path: path, fullpath: fullpath) } pool = Concurrent::FixedThreadPool.new pool_size assets.each do |asset| pool.post do @logger.info "[NightcrawlerSwift] #{asset.path}" upload = Upload.new upload.execute asset.path, File.open(asset.fullpath, "r") end end sleep(1) while pool.queue_length > 0 @logger.info "[NightcrawlerSwift] shutting down" pool.shutdown pool.wait_for_termination end