class Fluent::SwiftOutput
Public Class Methods
new()
click to toggle source
Calls superclass method
# File lib/fluent/plugin/out_swift.rb, line 8 def initialize super require 'fog' require 'zlib' require 'time' require 'tempfile' require 'open3' end
Public Instance Methods
configure(conf)
click to toggle source
Calls superclass method
# File lib/fluent/plugin/out_swift.rb, line 48 def configure(conf) super if format_json = conf['format_json'] @format_json = true else @format_json = false end @ext, @mime_type = case @store_as when 'gzip' then ['gz', 'application/x-gzip'] when 'lzo' then begin Open3.capture3('lzop -V') rescue Errno::ENOENT raise ConfigError, "'lzop' utility must be in PATH for LZO compression" end ['lzo', 'application/x-lzop'] when 'json' then ['json', 'application/json'] else ['txt', 'text/plain'] end @timef = TimeFormatter.new(@time_format, @localtime) if @localtime @path_slicer = Proc.new {|path| Time.now.strftime(path) } else @path_slicer = Proc.new {|path| Time.now.utc.strftime(path) } end end
format(tag, time, record)
click to toggle source
# File lib/fluent/plugin/out_swift.rb, line 99 def format(tag, time, record) if @include_time_key || !@format_json time_str = @timef.format(time) end # copied from each mixin because current TimeSlicedOutput can't support mixins. if @include_tag_key record[@tag_key] = tag end if @include_time_key record[@time_key] = time_str end if @format_json Yajl.dump(record) + "\n" else "#{time_str}\t#{tag}\t#{Yajl.dump(record)}\n" end end
placeholders()
click to toggle source
# File lib/fluent/plugin/out_swift.rb, line 44 def placeholders [:percent] end
start()
click to toggle source
Calls superclass method
# File lib/fluent/plugin/out_swift.rb, line 83 def start super Excon.defaults[:ssl_verify_peer] = @ssl_verify @storage = Fog::Storage.new :provider => 'OpenStack', :openstack_auth_url => @auth_url, :openstack_username => @auth_user, :openstack_tenant => @auth_tenant, :openstack_api_key => @auth_api_key, :openstack_region => @auth_region @storage.change_account @swift_account if @swift_account check_container end
write(chunk)
click to toggle source
# File lib/fluent/plugin/out_swift.rb, line 119 def write(chunk) i = 0 begin path = @path_slicer.call(@path) values_for_swift_object_key = { "path" => path, "time_slice" => chunk.key, "file_extension" => @ext, "index" => i } swift_path = @swift_object_key_format.gsub(%r(%{[^}]+})) { |expr| values_for_swift_object_key[expr[2...expr.size-1]] } i += 1 end while check_object_exists(@swift_container, swift_path) tmp = Tempfile.new("swift-") begin if @store_as == "gzip" w = Zlib::GzipWriter.new(tmp) chunk.write_to(w) w.close elsif @store_as == "lzo" w = Tempfile.new("chunk-tmp") chunk.write_to(w) w.close tmp.close # We don't check the return code because we can't recover lzop failure. system "lzop -qf1 -o #{tmp.path} #{w.path}" else chunk.write_to(tmp) tmp.close end File.open(tmp.path) do |file| @storage.put_object(@swift_container, swift_path, file, {:content_type => @mime_type}) end $log.info "Put Log to Swift. container=#{@swift_container} object=#{swift_path}" ensure tmp.close(true) rescue nil w.close rescue nil w.unlink rescue nil end end
Private Instance Methods
check_container()
click to toggle source
# File lib/fluent/plugin/out_swift.rb, line 166 def check_container begin @storage.get_container(@swift_container) rescue Fog::Storage::OpenStack::NotFound if @auto_create_container $log.info "Creating container #{@swift_container} on #{@auth_url}, #{@swift_account}" @storage.put_container(@swift_container) else raise "The specified container does not exist: container = #{swift_container}" end end end
check_object_exists(container, object)
click to toggle source
# File lib/fluent/plugin/out_swift.rb, line 179 def check_object_exists(container, object) begin @storage.head_object(container, object) rescue Fog::Storage::OpenStack::NotFound return false end return true end