class NN

Constants

VERSION

Attributes

activation[RW]
batch_size[RW]
betas[RW]
biases[RW]
dropout_ratio[RW]
gammas[RW]
learning_rate[RW]
momentum[RW]
training[R]
weight_decay[RW]
weights[RW]

Public Class Methods

load(file_name) click to toggle source
# File lib/nn.rb, line 46
def self.load(file_name)
  Marshal.load(File.binread(file_name))
end
load_json(file_name) click to toggle source
# File lib/nn.rb, line 50
def self.load_json(file_name)
  json = JSON.parse(File.read(file_name))
  nn = self.new(json["num_nodes"],
    learning_rate: json["learning_rate"],
    batch_size: json["batch_size"],
    activation: json["activation"].map(&:to_sym),
    momentum: json["momentum"],
    weight_decay: json["weight_decay"],
    use_dropout: json["use_dropout"],
    dropout_ratio: json["dropout_ratio"],
    use_batch_norm: json["use_batch_norm"],
  )
  nn.weights = json["weights"].map{|weight| SFloat.cast(weight)}
  nn.biases = json["biases"].map{|bias| SFloat.cast(bias)}
  if json["use_batch_norm"]
    nn.gammas = json["gammas"].map{|gamma| SFloat.cast(gamma)}
    nn.betas = json["betas"].map{|beta| SFloat.cast(beta)}
  end
  nn
end
new(num_nodes, learning_rate: 0.01, batch_size: 1, activation: %i(relu identity), momentum: 0, weight_decay: 0, use_dropout: false, dropout_ratio: 0.5, use_batch_norm: false) click to toggle source
# File lib/nn.rb, line 21
def initialize(num_nodes,
               learning_rate: 0.01,
               batch_size: 1,
               activation: %i(relu identity),
               momentum: 0,
               weight_decay: 0,
               use_dropout: false,
               dropout_ratio: 0.5,
               use_batch_norm: false)
  SFloat.srand(rand(2 ** 64))
  @num_nodes = num_nodes
  @learning_rate = learning_rate
  @batch_size = batch_size
  @activation = activation
  @momentum = momentum
  @weight_decay = weight_decay
  @use_dropout = use_dropout
  @dropout_ratio = dropout_ratio
  @use_batch_norm = use_batch_norm
  init_weight_and_bias
  init_gamma_and_beta if @use_batch_norm
  @training = true
  init_layers
end

Public Instance Methods

accurate(x_test, y_test, tolerance = 0.5, &block) click to toggle source
# File lib/nn.rb, line 93
def accurate(x_test, y_test, tolerance = 0.5, &block)
  correct = 0
  num_test_data = x_test.is_a?(SFloat) ? x_test.shape[0] : x_test.length
  (num_test_data.to_f / @batch_size).ceil.times do |i|
    x = SFloat.zeros(@batch_size, @num_nodes.first)
    y = SFloat.zeros(@batch_size, @num_nodes.last)
    @batch_size.times do |j|
      k = i * @batch_size + j
      break if k >= num_test_data
      if x_test.is_a?(SFloat)
        x[j, true] = x_test[k, true]
        y[j, true] = y_test[k, true]
      else
        x[j, true] = SFloat.cast(x_test[k])
        y[j, true] = SFloat.cast(y_test[k])
      end
    end
    x, y = block.call(x, y) if block
    out = forward(x, false)
    @batch_size.times do |j|
      vout = out[j, true]
      vy = y[j, true]
      case @activation[1]
      when :identity
        correct += 1 unless (NMath.sqrt((vout - vy) ** 2) < tolerance).to_a.include?(0)
      when :softmax
        correct += 1 if vout.max_index == vy.max_index
      end
    end
  end
  correct.to_f / num_test_data
end
learn(x_train, y_train, &block) click to toggle source
# File lib/nn.rb, line 126
def learn(x_train, y_train, &block)
  if x_train.is_a?(SFloat)
    indexes = (0...x_train.shape[0]).to_a.sample(@batch_size)
    x = x_train[indexes, true]
    y = y_train[indexes, true]
  else
    indexes = (0...x_train.length).to_a.sample(@batch_size)
    x = SFloat.zeros(@batch_size, @num_nodes.first)
    y = SFloat.zeros(@batch_size, @num_nodes.last)
    @batch_size.times do |i|
      x[i, true] = SFloat.cast(x_train[indexes[i]])
      y[i, true] = SFloat.cast(y_train[indexes[i]])
    end
  end
  x, y = block.call(x, y) if block
  forward(x)
  backward(y)
  update_weight_and_bias
  update_gamma_and_beta if @use_batch_norm
  @layers[-1].loss(y)
end
run(x) click to toggle source
# File lib/nn.rb, line 148
def run(x)
  if x.is_a?(Array)
    forward(SFloat.cast(x), false).to_a
  else
    forward(x, false)
  end
end
save(file_name) click to toggle source
# File lib/nn.rb, line 156
def save(file_name)
  File.binwrite(file_name, Marshal.dump(self))
end
save_json(file_name) click to toggle source
# File lib/nn.rb, line 160
def save_json(file_name)
  json = {
    "version" => VERSION,
    "num_nodes" => @num_nodes,
    "learning_rate" => @learning_rate,
    "batch_size" => @batch_size,
    "activation" => @activation,
    "momentum" => @momentum,
    "weight_decay" => @weight_decay,
    "use_dropout" => @use_dropout,
    "dropout_ratio" => @dropout_ratio,
    "use_batch_norm" => @use_batch_norm,
    "weights" => @weights.map(&:to_a),
    "biases" => @biases.map(&:to_a),
  }
  if @use_batch_norm
    json_batch_norm = {
      "gammas" => @gammas,
      "betas" => @betas
    }
    json.merge!(json_batch_norm)
  end
  File.write(file_name, JSON.dump(json))
end
test(x_test, y_test, tolerance = 0.5, &block) click to toggle source
# File lib/nn.rb, line 87
def test(x_test, y_test, tolerance = 0.5, &block)
  acc = accurate(x_test, y_test, tolerance, &block)
  puts "accurate: #{acc}"
  acc
end
train(x_train, y_train, epochs, func = nil, &block) click to toggle source
# File lib/nn.rb, line 71
def train(x_train, y_train, epochs, func = nil, &block)
  num_train_data = x_train.is_a?(SFloat) ? x_train.shape[0] : x_train.length
  (1..epochs).each do |epoch|
    loss = nil
    (num_train_data.to_f / @batch_size).ceil.times do
      loss = learn(x_train, y_train, &func)
      if loss.nan?
        puts "loss is nan"
        return
      end
    end
    puts "epoch #{epoch}/#{epochs} loss: #{loss}"
    block.call(epoch) if block
  end
end

Private Instance Methods

backward(y) click to toggle source
# File lib/nn.rb, line 242
def backward(y)
  dout = @layers[-1].backward(y)
  @layers[0...-1].reverse.each do |layer|
    dout = layer.backward(dout)
  end
end
forward(x, training = true) click to toggle source
# File lib/nn.rb, line 234
def forward(x, training = true)
  @training = training
  @layers.each do |layer|
    x = layer.forward(x)
  end
  x
end
init_gamma_and_beta() click to toggle source
# File lib/nn.rb, line 205
def init_gamma_and_beta
  @gammas = Array.new(@num_nodes.length - 2, 1)
  @betas = Array.new(@num_nodes.length - 2, 0)
  @gamma_amounts = Array.new(@num_nodes.length - 2, 0)
  @beta_amounts = Array.new(@num_nodes.length - 2, 0)
end
init_layers() click to toggle source
# File lib/nn.rb, line 212
def init_layers
  @layers = []
  @num_nodes[0...-2].each_index do |i|
    @layers << Affine.new(self, i)
    @layers << BatchNorm.new(self, i) if @use_batch_norm
    @layers << case @activation[0]
    when :sigmoid
      Sigmoid.new
    when :relu
      ReLU.new
    end
    @layers << Dropout.new(self) if @use_dropout
  end
  @layers << Affine.new(self, -1)
  @layers << case @activation[1]
  when :identity
    Identity.new(self)
  when :softmax
    Softmax.new(self)
  end
end
init_weight_and_bias() click to toggle source
# File lib/nn.rb, line 187
def init_weight_and_bias
  @weights = Array.new(@num_nodes.length - 1)
  @biases = Array.new(@num_nodes.length - 1)
  @weight_amounts = Array.new(@num_nodes.length - 1, 0)
  @bias_amounts = Array.new(@num_nodes.length - 1, 0)
  @num_nodes[0...-1].each_index do |i|
    weight = SFloat.new(@num_nodes[i], @num_nodes[i + 1]).rand_norm
    bias = SFloat.new(@num_nodes[i + 1]).rand_norm
    if @activation[0] == :relu
      @weights[i] = weight / Math.sqrt(@num_nodes[i]) * Math.sqrt(2)
      @biases[i] = bias / Math.sqrt(@num_nodes[i]) * Math.sqrt(2)
    else
      @weights[i] = weight / Math.sqrt(@num_nodes[i])
      @biases[i] = bias / Math.sqrt(@num_nodes[i])
    end
  end
end
update_gamma_and_beta() click to toggle source
# File lib/nn.rb, line 264
def update_gamma_and_beta
  @layers.select{|layer| layer.is_a?(BatchNorm)}.each.with_index do |layer, i|
    gamma_amount = layer.d_gamma * @learning_rate
    beta_amount = layer.d_beta * @learning_rate
    if @momentum > 0
      gamma_amount += @momentum * @gamma_amounts[i]
      @gamma_amounts[i] = gamma_amount
      beta_amount += @momentum * @beta_amounts[i]
      @beta_amounts[i] = beta_amount
    end
    @gammas[i] -= gamma_amount
    @betas[i] -= gamma_amount
  end
end
update_weight_and_bias() click to toggle source
# File lib/nn.rb, line 249
def update_weight_and_bias
  @layers.select{|layer| layer.is_a?(Affine)}.each.with_index do |layer, i|
    weight_amount = layer.d_weight * @learning_rate
    bias_amount = layer.d_bias * @learning_rate
    if @momentum > 0
      weight_amount += @momentum * @weight_amounts[i]
      @weight_amounts[i] = weight_amount
      bias_amount += @momentum * @bias_amounts[i]
      @bias_amounts[i] = bias_amount
    end
    @weights[i] -= weight_amount
    @biases[i] -= bias_amount
  end
end