class Lockbox::Migrator

Public Class Methods

new(relation, batch_size:) click to toggle source
# File lib/lockbox/migrator.rb, line 3
def initialize(relation, batch_size:)
  @relation = relation
  @transaction = @relation.respond_to?(:transaction)
  @batch_size = batch_size
end

Public Instance Methods

migrate(restart:) click to toggle source

TODO add attributes option

# File lib/lockbox/migrator.rb, line 26
def migrate(restart:)
  fields = model.respond_to?(:lockbox_attributes) ? model.lockbox_attributes.select { |k, v| v[:migrating] } : {}

  # need blind indexes for building relation
  blind_indexes = model.respond_to?(:blind_indexes) ? model.blind_indexes.select { |k, v| v[:migrating] } : {}

  attachments = model.respond_to?(:lockbox_attachments) ? model.lockbox_attachments.select { |k, v| v[:migrating] } : {}

  perform(fields: fields, blind_indexes: blind_indexes, restart: restart) if fields.any? || blind_indexes.any?
  perform_attachments(attachments: attachments, restart: restart) if attachments.any?
end
model() click to toggle source
# File lib/lockbox/migrator.rb, line 9
def model
  @model ||= @relation
end
rotate(attributes:) click to toggle source
# File lib/lockbox/migrator.rb, line 13
def rotate(attributes:)
  fields = {}
  attributes.each do |a|
    # use key instead of v[:attribute] to make it more intuitive when migrating: true
    field = model.lockbox_attributes[a]
    raise ArgumentError, "Bad attribute: #{a}" unless field
    fields[a] = field
  end

  perform(fields: fields, rotate: true)
end

Private Instance Methods

ar_relation?(relation) click to toggle source
# File lib/lockbox/migrator.rb, line 183
def ar_relation?(relation)
  defined?(ActiveRecord::Relation) && relation.is_a?(ActiveRecord::Relation)
end
base_relation() click to toggle source
# File lib/lockbox/migrator.rb, line 171
def base_relation
  relation = @relation

  # unscope if passed a model
  unless ar_relation?(relation) || mongoid_relation?(relation)
    relation = relation.unscoped
  end

  # convert from possible class to ActiveRecord::Relation or Mongoid::Criteria
  relation.all
end
each_batch(relation) { |records| ... } click to toggle source
# File lib/lockbox/migrator.rb, line 99
def each_batch(relation)
  if relation.respond_to?(:find_in_batches)
    relation.find_in_batches(batch_size: @batch_size) do |records|
      yield records
    end
  else
    # https://github.com/karmi/tire/blob/master/lib/tire/model/import.rb
    # use cursor for Mongoid
    records = []
    relation.all.each do |record|
      records << record
      if records.length == @batch_size
        yield records
        records = []
      end
    end
    yield records if records.any?
  end
end
migrate_records(records, fields:, blind_indexes:, restart:, rotate:) click to toggle source

there's a small chance for this process to read data, another process to update the data, and this process to write the now stale data this time window can be reduced with smaller batch sizes locking individual records could eliminate this one option is: relation.in_batches { |batch| batch.lock } which runs SELECT … FOR UPDATE in Postgres

# File lib/lockbox/migrator.rb, line 126
def migrate_records(records, fields:, blind_indexes:, restart:, rotate:)
  # do computation outside of transaction
  # especially expensive blind index computation
  if rotate
    records.each do |record|
      fields.each do |k, v|
        # update encrypted attribute directly to skip blind index computation
        record.send("lockbox_direct_#{k}=", record.send(k))
      end
    end
  else
    records.each do |record|
      if restart
        fields.each do |k, v|
          record.send("#{v[:encrypted_attribute]}=", nil)
        end

        blind_indexes.each do |k, v|
          record.send("#{v[:bidx_attribute]}=", nil)
        end
      end

      fields.each do |k, v|
        record.send("#{v[:attribute]}=", record.send(k)) unless record.send(v[:encrypted_attribute])
      end

      # with Blind Index 2.0, bidx_attribute should be already set for each record
      blind_indexes.each do |k, v|
        record.send("compute_#{k}_bidx") unless record.send(v[:bidx_attribute])
      end
    end
  end

  # don't need to save records that went from nil => nil
  records.select! { |r| r.changed? }

  if records.any?
    with_transaction do
      records.each do |record|
        record.save!(validate: false)
      end
    end
  end
end
mongoid_relation?(relation) click to toggle source
# File lib/lockbox/migrator.rb, line 187
def mongoid_relation?(relation)
  defined?(Mongoid::Criteria) && relation.is_a?(Mongoid::Criteria)
end
perform(fields:, blind_indexes: [], restart: true, rotate: false) click to toggle source
# File lib/lockbox/migrator.rb, line 68
def perform(fields:, blind_indexes: [], restart: true, rotate: false)
  relation = base_relation

  unless restart
    attributes = fields.map { |_, v| v[:encrypted_attribute] }
    attributes += blind_indexes.map { |_, v| v[:bidx_attribute] }

    if ar_relation?(relation)
      base_relation = relation.unscoped
      or_relation = relation.unscoped

      attributes.each_with_index do |attribute, i|
        or_relation =
          if i == 0
            base_relation.where(attribute => nil)
          else
            or_relation.or(base_relation.where(attribute => nil))
          end
      end

      relation = relation.merge(or_relation)
    else
      relation.merge(relation.unscoped.or(attributes.map { |a| {a => nil} }))
    end
  end

  each_batch(relation) do |records|
    migrate_records(records, fields: fields, blind_indexes: blind_indexes, restart: restart, rotate: rotate)
  end
end
perform_attachments(attachments:, restart:) click to toggle source
# File lib/lockbox/migrator.rb, line 40
def perform_attachments(attachments:, restart:)
  relation = base_relation

  # eager load attachments
  attachments.each_key do |k|
    relation = relation.send("with_attached_#{k}")
  end

  each_batch(relation) do |records|
    records.each do |record|
      attachments.each_key do |k|
        attachment = record.send(k)
        if attachment.attached?
          if attachment.is_a?(ActiveStorage::Attached::One)
            unless attachment.metadata["encrypted"]
              attachment.rotate_encryption!
            end
          else
            unless attachment.all? { |a| a.metadata["encrypted"] }
              attachment.rotate_encryption!
            end
          end
        end
      end
    end
  end
end
with_transaction() { || ... } click to toggle source
# File lib/lockbox/migrator.rb, line 191
def with_transaction
  if @transaction
    @relation.transaction do
      yield
    end
  else
    yield
  end
end