class DNN::Optimizers::AdaBound
Attributes
final_lr[RW]
gamma[RW]
Public Class Methods
new(alpha: 0.001, beta1: 0.9, beta2: 0.999, final_lr: 0.1, gamma: 0.001, eps: 1e-7, amsgrad: false, clip_norm: nil)
click to toggle source
@param [Float] final_lr
Final learning rate. @param [Float] gamma Lower and upper range value.
Calls superclass method
DNN::Optimizers::Adam::new
# File lib/dnn/core/optimizers.rb, line 314 def initialize(alpha: 0.001, beta1: 0.9, beta2: 0.999, final_lr: 0.1, gamma: 0.001, eps: 1e-7, amsgrad: false, clip_norm: nil) super(alpha: alpha, beta1: beta1, beta2: beta2, eps: eps, amsgrad: amsgrad, clip_norm: clip_norm) @final_lr = final_lr @gamma = gamma end
Public Instance Methods
load_hash(hash)
click to toggle source
# File lib/dnn/core/optimizers.rb, line 354 def load_hash(hash) initialize(alpha: hash[:alpha], beta1: hash[:beta1], beta2: hash[:beta2], final_lr: hash[:final_lr], gamma: hash[:gamma], eps: hash[:eps], amsgrad: hash[:amsgrad], clip_norm: hash[:clip_norm]) end
to_hash()
click to toggle source
# File lib/dnn/core/optimizers.rb, line 320 def to_hash { class: self.class.name, alpha: @alpha, beta1: @beta1, beta2: @beta2, final_lr: @final_lr, gamma: @gamma, eps: @eps, amsgrad: amsgrad, clip_norm: @clip_norm } end
Private Instance Methods
clip_lr(lr, lower_bound, upper_bound)
click to toggle source
# File lib/dnn/core/optimizers.rb, line 348 def clip_lr(lr, lower_bound, upper_bound) lr[lr < lower_bound] = lower_bound lr[lr > upper_bound] = upper_bound lr end
update_params(params)
click to toggle source
# File lib/dnn/core/optimizers.rb, line 327 def update_params(params) @t += 1 lr = @alpha * Math.sqrt(1 - @beta2**@t) / (1 - @beta1**@t) final_lr = @final_lr * lr / @alpha lower_bound = final_lr * (1 - 1 / (@gamma * @t + 1)) upper_bound = final_lr * (1 + 1 / (@gamma * @t)) params.each do |param| @m[param] ||= Xumo::SFloat.zeros(*param.data.shape) @v[param] ||= Xumo::SFloat.zeros(*param.data.shape) @m[param] += (1 - @beta1) * (param.grad - @m[param]) @v[param] += (1 - @beta2) * (param.grad**2 - @v[param]) if @amsgrad @s[param] ||= Xumo::SFloat.zeros(*param.data.shape) @s[param] = Xumo::SFloat.maximum(@s[param], @v[param]) param.data -= clip_lr(lr / (Xumo::NMath.sqrt(@s[param]) + @eps), lower_bound, upper_bound) * @m[param] else param.data -= clip_lr(lr / (Xumo::NMath.sqrt(@v[param]) + @eps), lower_bound, upper_bound) * @m[param] end end end