class NightcrawlerSwift::MultithreadSync

Constants

DEFAULT_POOL_SIZE

Public Class Methods

new() click to toggle source
# File lib/nightcrawler_swift/commands/multithread_sync.rb, line 5
def initialize
  require 'concurrent/utilities'
  require 'concurrent/executors'
  @logger = NightcrawlerSwift.logger
end

Public Instance Methods

execute(args = {}) click to toggle source
# File lib/nightcrawler_swift/commands/multithread_sync.rb, line 11
def execute args = {}
  pool_size = args[:pool_size] || DEFAULT_POOL_SIZE
  dir_path = args[:dir_path]

  @logger.info "[NightcrawlerSwift] dir_path: #{dir_path}"
  @logger.info "[NightcrawlerSwift] multithread sync, #{Concurrent.processor_count} processors"

  assets = Dir["#{dir_path}/**/**"].
    reject {|fullpath| File.directory?(fullpath)}.
    map {|fullpath|
      path = fullpath.gsub("#{dir_path}/", "")
      OpenStruct.new(path: path, fullpath: fullpath)
    }

  pool = Concurrent::FixedThreadPool.new pool_size

  assets.each do |asset|
    pool.post do
      @logger.info "[NightcrawlerSwift] #{asset.path}"

      upload = Upload.new
      upload.execute asset.path, File.open(asset.fullpath, "r")
    end
  end

  sleep(1) while pool.queue_length > 0

  @logger.info "[NightcrawlerSwift] shutting down"
  pool.shutdown
  pool.wait_for_termination
end