class TnS3FileUploader::FilePathGenerator
[ec2-user@ip-10-185-180-243 tmp]$ ./time.sh partition = y=2014/m=06/d=18/h=18 minute_partition = 55 file_timestamp = 20140618185540 date = Wed Jun 18 19:03:40 UTC 2014
Constants
- EC2_INSTANCE_METADATA_IP
Public Class Methods
# File lib/tn_s3_file_uploader/file_path_generator.rb, line 21 def initialize(options) #Find the last rotation window @options = options end
Public Instance Methods
Makes datetime and macro substitutions for input file 'file', based on the s3_output_pattern option Assumes input file 'file' and s3_output_pattern option are both valid. This method removes the bucket (everything until the first '/', including the '/') from the s3_output_pattern while applying the datetime/macro/substitutions
# File lib/tn_s3_file_uploader/file_path_generator.rb, line 31 def dest_full_path_for(time, file) time = previous_rotation_window(time) output_file_pattern = remove_bucket(@options[:s3_output_pattern]) # Time#strftime is removing '%' characters on our macros. Our macro substitution must run first subs = build_substitutions(file, time) replace_macros!(output_file_pattern, subs) substitute_datetime_macros(time, output_file_pattern) end
Private Instance Methods
# File lib/tn_s3_file_uploader/file_path_generator.rb, line 102 def build_substitutions(file, time) file_components = file.split('/').last.split('.') if file_components.size == 1 file_name = file_components[0] file_extension = '' else file_name = file_components[0..-2].join('.') file_extension = file_components.last end ip_address = local_ip.gsub('.', '-') file_timestamp = generate_file_timestamp(time) { '%{file-name}' => file_name, '%{file-timestamp}' => file_timestamp, '%{file-extension}' => file_extension, '%{ip-address}' => ip_address } end
# File lib/tn_s3_file_uploader/file_path_generator.rb, line 90 def ec2_instance_metadata_local_ip Net::HTTP.get(EC2_INSTANCE_METADATA_IP, '/latest/meta-data/local-ipv4') rescue nil end
Generates rounded off timestamp based on rotation_seconds
# File lib/tn_s3_file_uploader/file_path_generator.rb, line 58 def generate_file_timestamp(time) time.strftime('%Y%m%d%H%M%S') end
# File lib/tn_s3_file_uploader/file_path_generator.rb, line 94 def hostname_resolve_ip IPSocket.getaddress(Socket.gethostname) end
First tries to find local IP using the EC2 instance metadata endpoint. In the event of a failure, we will revert to using the old method of local ip retrieval. In the event that both techniques fail, we return a default value
# File lib/tn_s3_file_uploader/file_path_generator.rb, line 72 def local_ip ip_address = nil if @options[:use_ec2_metadata_flag] ip_address = ec2_instance_metadata_local_ip end unless ip_address.nil? or valid_ip?(ip_address) ip_address = hostname_resolve_ip end unless valid_ip?(ip_address) ip_address = '0.0.0.0' end ip_address end
# File lib/tn_s3_file_uploader/file_path_generator.rb, line 125 def previous_rotation_window(time) rotation_seconds = @options[:file_timestamp_resolution] t = time - rotation_seconds floored_seconds = (t.to_f / rotation_seconds).floor * rotation_seconds Time.at(floored_seconds).utc end
# File lib/tn_s3_file_uploader/file_path_generator.rb, line 44 def remove_bucket(output_file_pattern) output_file_pattern.split('/')[1..-1].join('/') end
# File lib/tn_s3_file_uploader/file_path_generator.rb, line 62 def replace_macros!(output_file_pattern, subs) subs.each do |macro, sub| output_file_pattern.gsub!(macro, sub) end end
Makes the datetime substitutions on the give s3_output_pattern option For example:
Given s3_output_pattern y=%Y/m=%m/d=%d/h=%H and time: Thu Jun 12 23:57:49 UTC 2014 it will produce the following folder structure: y=2014/m=06/d=12/h=23
# File lib/tn_s3_file_uploader/file_path_generator.rb, line 53 def substitute_datetime_macros(time, output_pattern) time.strftime(output_pattern) end
# File lib/tn_s3_file_uploader/file_path_generator.rb, line 98 def valid_ip?(resolve_ip) resolve_ip =~ /\d+\.\d+\.\d+\.\d+/ end