module LoadingData

Constants

CONVERT_RECORD
REGEX_APACHE
REGEX_KEYS

Your code goes here…

REGEX_NGINX

Public Class Methods

convert_nginx_logs(log) click to toggle source
# File lib/log_analysis/loading_data.rb, line 38
def self.convert_nginx_logs(log)
  o = log.split(REGEX_NGINX)

  return false if o.size <= 1
  o.delete('')

  {}.tap do |p|
    p['host']      = o[0]
    p['user']      = o[2]
    p['time']      = o[3]
    p['method']    = o[4]
    p['uri']       = o[5]
    p['status']    = o[6]
    p['size']      = o[7]
    p['referer']   = o[8]
    p['ua']        = o[9]
    p['forwarded'] = o[10]
    p['user']      = save_user(o)
  end
end
input(file_path, type) click to toggle source
# File lib/log_analysis/loading_data.rb, line 15
def self.input(file_path, type)
  @users    = []
  text_file = File.readlines(file_path)

  text_file.each_with_object([]).with_index do |(line, arr), i|
    preprocessed_log = type.nil? ? line.gsub(/[\t]/i, ' ').chomp! : line
    record_params    = send(CONVERT_RECORD[type.nil? ? 'nginx' : type], preprocessed_log)
    record           = Record.new(record_params) if record_params && preprocessed_log

    system('clear')
    puts "#{((i.to_f / text_file.size) * 100).round}/100"
    arr.push(record) if record
  end
end
save_user(log) click to toggle source
# File lib/log_analysis/loading_data.rb, line 63
def self.save_user(log)
  user = @users.find { |i| i.host == log[0] && i.user_agent.to_s == log[9] }
  return user unless user.nil?

  @users.push(UserIdentity.new(host: IPAddr.new(log[0]), user_agent: UserAgent.parse(log[9])))
  @users.last
end
to_json(pair) click to toggle source
# File lib/log_analysis/loading_data.rb, line 59
def self.to_json(pair)
  { pair.first.delete(':') => pair.last }
end
to_record(log) click to toggle source
# File lib/log_analysis/loading_data.rb, line 30
def self.to_record(log)
  o = log.gsub!('\t', ' ')
  o = log.split(REGEX_KEYS)
  o = o.map(&:strip)
  o.delete('')
  o.each_slice(2).to_a.each_with_object({}) { |pair, log_obj| log_obj.merge!(to_json(pair)) }
end