class DNN::Optimizers::AdaBound

Attributes

final_lr[RW]
gamma[RW]

Public Class Methods

new(alpha: 0.001, beta1: 0.9, beta2: 0.999, final_lr: 0.1, gamma: 0.001, eps: 1e-7, amsgrad: false, clip_norm: nil) click to toggle source

@param [Float] final_lr Final learning rate. @param [Float] gamma Lower and upper range value.

Calls superclass method DNN::Optimizers::Adam::new
# File lib/dnn/core/optimizers.rb, line 314
def initialize(alpha: 0.001, beta1: 0.9, beta2: 0.999, final_lr: 0.1, gamma: 0.001, eps: 1e-7, amsgrad: false, clip_norm: nil)
  super(alpha: alpha, beta1: beta1, beta2: beta2, eps: eps, amsgrad: amsgrad, clip_norm: clip_norm)
  @final_lr = final_lr
  @gamma = gamma
end

Public Instance Methods

load_hash(hash) click to toggle source
# File lib/dnn/core/optimizers.rb, line 354
def load_hash(hash)
  initialize(alpha: hash[:alpha], beta1: hash[:beta1], beta2: hash[:beta2],
             final_lr: hash[:final_lr], gamma: hash[:gamma], eps: hash[:eps], amsgrad: hash[:amsgrad], clip_norm: hash[:clip_norm])
end
to_hash() click to toggle source
# File lib/dnn/core/optimizers.rb, line 320
def to_hash
  {
    class: self.class.name, alpha: @alpha, beta1: @beta1, beta2: @beta2,
    final_lr: @final_lr, gamma: @gamma, eps: @eps, amsgrad: amsgrad, clip_norm: @clip_norm
  }
end

Private Instance Methods

clip_lr(lr, lower_bound, upper_bound) click to toggle source
# File lib/dnn/core/optimizers.rb, line 348
        def clip_lr(lr, lower_bound, upper_bound)
  lr[lr < lower_bound] = lower_bound
  lr[lr > upper_bound] = upper_bound
  lr
end
update_params(params) click to toggle source
# File lib/dnn/core/optimizers.rb, line 327
        def update_params(params)
  @t += 1
  lr = @alpha * Math.sqrt(1 - @beta2**@t) / (1 - @beta1**@t)
  final_lr = @final_lr * lr / @alpha
  lower_bound = final_lr * (1 - 1 / (@gamma * @t + 1))
  upper_bound = final_lr * (1 + 1 / (@gamma * @t))
  params.each do |param|
    @m[param] ||= Xumo::SFloat.zeros(*param.data.shape)
    @v[param] ||= Xumo::SFloat.zeros(*param.data.shape)
    @m[param] += (1 - @beta1) * (param.grad - @m[param])
    @v[param] += (1 - @beta2) * (param.grad**2 - @v[param])
    if @amsgrad
      @s[param] ||= Xumo::SFloat.zeros(*param.data.shape)
      @s[param] = Xumo::SFloat.maximum(@s[param], @v[param])
      param.data -= clip_lr(lr / (Xumo::NMath.sqrt(@s[param]) + @eps), lower_bound, upper_bound) * @m[param]
    else
      param.data -= clip_lr(lr / (Xumo::NMath.sqrt(@v[param]) + @eps), lower_bound, upper_bound) * @m[param]
    end
  end
end