class TnS3FileUploader::FilePathGenerator

[ec2-user@ip-10-185-180-243 tmp]$ ./time.sh partition = y=2014/m=06/d=18/h=18 minute_partition = 55 file_timestamp = 20140618185540 date = Wed Jun 18 19:03:40 UTC 2014

Constants

EC2_INSTANCE_METADATA_IP

Public Class Methods

new(options) click to toggle source
# File lib/tn_s3_file_uploader/file_path_generator.rb, line 21
def initialize(options)
  #Find the last rotation window
  @options = options
end

Public Instance Methods

dest_full_path_for(time, file) click to toggle source

Makes datetime and macro substitutions for input file 'file', based on the s3_output_pattern option Assumes input file 'file' and s3_output_pattern option are both valid. This method removes the bucket (everything until the first '/', including the '/') from the s3_output_pattern while applying the datetime/macro/substitutions

# File lib/tn_s3_file_uploader/file_path_generator.rb, line 31
def dest_full_path_for(time, file)
  time = previous_rotation_window(time)
  output_file_pattern = remove_bucket(@options[:s3_output_pattern])

  # Time#strftime is removing '%' characters on our macros. Our macro substitution must run first
  subs = build_substitutions(file, time)
  replace_macros!(output_file_pattern, subs)

  substitute_datetime_macros(time, output_file_pattern)
end

Private Instance Methods

build_substitutions(file, time) click to toggle source
# File lib/tn_s3_file_uploader/file_path_generator.rb, line 102
def build_substitutions(file, time)
  file_components = file.split('/').last.split('.')

  if file_components.size == 1
    file_name = file_components[0]
    file_extension = ''
  else
    file_name = file_components[0..-2].join('.')
    file_extension = file_components.last
  end
  
  ip_address = local_ip.gsub('.', '-')
  
  file_timestamp = generate_file_timestamp(time)

  {
      '%{file-name}' => file_name,
      '%{file-timestamp}' => file_timestamp,
      '%{file-extension}' => file_extension,
      '%{ip-address}' => ip_address
  }
end
ec2_instance_metadata_local_ip() click to toggle source
# File lib/tn_s3_file_uploader/file_path_generator.rb, line 90
def ec2_instance_metadata_local_ip
  Net::HTTP.get(EC2_INSTANCE_METADATA_IP, '/latest/meta-data/local-ipv4') rescue nil
end
generate_file_timestamp(time) click to toggle source

Generates rounded off timestamp based on rotation_seconds

# File lib/tn_s3_file_uploader/file_path_generator.rb, line 58
def generate_file_timestamp(time)
  time.strftime('%Y%m%d%H%M%S')
end
hostname_resolve_ip() click to toggle source
# File lib/tn_s3_file_uploader/file_path_generator.rb, line 94
def hostname_resolve_ip
  IPSocket.getaddress(Socket.gethostname)
end
local_ip() click to toggle source

First tries to find local IP using the EC2 instance metadata endpoint. In the event of a failure, we will revert to using the old method of local ip retrieval. In the event that both techniques fail, we return a default value

# File lib/tn_s3_file_uploader/file_path_generator.rb, line 72
def local_ip
  ip_address = nil

  if @options[:use_ec2_metadata_flag]
    ip_address = ec2_instance_metadata_local_ip
  end

  unless ip_address.nil? or valid_ip?(ip_address)
    ip_address = hostname_resolve_ip
  end

  unless valid_ip?(ip_address)
    ip_address = '0.0.0.0'
  end

  ip_address
end
previous_rotation_window(time) click to toggle source
# File lib/tn_s3_file_uploader/file_path_generator.rb, line 125
def previous_rotation_window(time)
  rotation_seconds = @options[:file_timestamp_resolution]
  t = time - rotation_seconds
  floored_seconds = (t.to_f / rotation_seconds).floor * rotation_seconds
  Time.at(floored_seconds).utc
end
remove_bucket(output_file_pattern) click to toggle source
# File lib/tn_s3_file_uploader/file_path_generator.rb, line 44
def remove_bucket(output_file_pattern)
  output_file_pattern.split('/')[1..-1].join('/')
end
replace_macros!(output_file_pattern, subs) click to toggle source
# File lib/tn_s3_file_uploader/file_path_generator.rb, line 62
def replace_macros!(output_file_pattern, subs)
  subs.each do |macro, sub|
    output_file_pattern.gsub!(macro, sub)
  end
end
substitute_datetime_macros(time, output_pattern) click to toggle source

Makes the datetime substitutions on the give s3_output_pattern option For example:

Given s3_output_pattern y=%Y/m=%m/d=%d/h=%H
and time: Thu Jun 12 23:57:49 UTC 2014
it will produce the following folder structure: y=2014/m=06/d=12/h=23
# File lib/tn_s3_file_uploader/file_path_generator.rb, line 53
def substitute_datetime_macros(time, output_pattern)
  time.strftime(output_pattern)
end
valid_ip?(resolve_ip) click to toggle source
# File lib/tn_s3_file_uploader/file_path_generator.rb, line 98
def valid_ip?(resolve_ip)
  resolve_ip =~ /\d+\.\d+\.\d+\.\d+/
end