class SdbS3Interface::FilesMgr
Constants
- AccessKey
- BucketNamespacePrefix
- SecretKey
Attributes
bucket_name[RW]
Public Class Methods
new(glue_env, node_key_value)
click to toggle source
# File lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb, line 33 def initialize(glue_env, node_key_value) #@s3_connection = S3::Base.establish_connection!(:access_key_id => AccessKey, # :secret_access_key => SecretKey) @bucket_name = "#{BucketNamespacePrefix}_#{glue_env.user_datastore_location}" #@attachment_bucket = use_bucket(@bucket_name) #This can be stale!!! #verify bucket is ready #puts "Previous Response: #{S3::Service.response}" #puts "#{__LINE__} - #{ S3::Service.buckets(true).map{|b| b.name} }" #puts "This Bucket: #{@attachment_bucket.name}" #puts "Last Response: #{S3::Service.response}" #size = @attachment_bucket.size end
Public Instance Methods
add(node, file_datas)
click to toggle source
TODO: Move common file management functions from base node to here
# File lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb, line 47 def add(node, file_datas) filenames = [] file_datas.each do |file_data| filenames << file_data[:src_filename] end filenames.each do |filename| basename = File.basename(filename) esc_basename = TkEscape.escape(basename) begin S3::S3Object.store(esc_basename, open(filename), @bucket_name) rescue AWS::S3::NoSuchBucket puts "Rescued while adding files, retrying" retry_request { S3::S3Object.store(esc_basename, open(filename), @bucket_name) } end end #verify files are there files = self.list_attachments filenames.each do |f| bname = TkEscape.escape(File.basename(f)) esc_bname = TkEscape.escape(bname) retry_request { S3::S3Object.store(esc_bname, open(f), @bucket_name) } unless files.include?(esc_bname) end #update the metadata (have to wait until they're uploaded *sigh* filenames.each do |f| basename = File.basename(f) attach_name = TkEscape.escape(basename) begin s3_obj = S3::S3Object.find(attach_name, @bucket_name) rescue AWS::S3::NoSuchBucket puts "Rescued while finding bucket, retrying" retry_request { S3::S3Object.find(attach_name, @bucket_name) } end modified_at = File.mtime(f).to_s s3_obj.metadata[:modified_at] = modified_at s3_obj.store end filenames.map {|f| TkEscape.escape(File.basename(f))} #return basenames end
add_raw_data(node, file_name, content_type, raw_data, file_modified_at = nil)
click to toggle source
# File lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb, line 90 def add_raw_data(node, file_name, content_type, raw_data, file_modified_at = nil) attach_name = TkEscape.escape(file_name) options = {:content_type => content_type} #=begin begin resp = S3::S3Object.store(attach_name, raw_data, @bucket_name, options) rescue AWS::S3::NoSuchBucket puts "Rescued while adding raw data, retrying" retry_request { S3::S3Object.store(attach_name, raw_data, @bucket_name, options) } end obj = S3::S3Object.find(attach_name, @bucket_name) obj.metadata[:modified_at] = "2011-01-13" #CGI.escape(file_modified_at) obj.store #verify files are there max_wait_time = 20 now = Time.now until (S3::S3Object.exists?(attach_name, @bucket_name) ) || (Time.now > (now + max_wait_time)) puts "Retrying store for add raw data" sleep 2 S3::S3Object.store(attach_name, raw_data, @bucket_name, options) end [attach_name] end
destroy_file_container()
click to toggle source
# File lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb, line 203 def destroy_file_container this_bucket = use_bucket(@bucket_name) begin this_bucket.delete(:force => true) rescue AWS::S3::NoSuchBucket puts "Running sanity check" buckets = S3::Service.buckets(true).map{|b| b.name} if buckets.include?(@bucket_name) puts "AWS temporarily lost bucket before finding it so it can be deleted" retry_request { this_bucket.delete(:force => true) } end end end
get_attachments_metadata(node)
click to toggle source
todo change name to get_files_metadata
# File lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb, line 158 def get_attachments_metadata(node) files_md = {} begin this_bucket = use_bucket(@bucket_name) objects = this_bucket.objects rescue AWS::S3::NoSuchBucket puts "rescued while getting objects from bucket to check metadata" objects = retry_request{ this_bucket.objects } end objects.each do |object| begin obj_md = object.about.merge(object.metadata) rescue AWS::S3::NoSuchBucket puts "Rescued while getting metadata from object" obj_md = retry_request{object.about} end time_str = obj_md["x-amz-meta-modified-at"]||Time.parse(obj_md["last_modified"]).to_s obj_md_file_modified = time_str obj_md_content_type = obj_md["content-type"] new_md = {:content_type => obj_md_content_type, :file_modified => obj_md_file_modified} new_md.merge(obj_md) #where does the original metadata go? files_md[object.key] = new_md end files_md end
get_raw_data(node, basename)
click to toggle source
# File lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb, line 137 def get_raw_data(node, basename) rtn = nil attach_name = TkEscape.escape(basename) begin rtn = S3::S3Object.value(attach_name, @bucket_name) rescue AWS::S3::NoSuchBucket puts "Rescued while getting raw data, bucket name: #{@bucket_name}" begin rtn = retry_request(attach_name, @bucket_name){|obj, buck| puts "sdbs3: #{obj.inspect} - #{buck.inspect}"; S3::S3Object.value(obj, buck)} rescue AWS::S3::NoSuchKey rtn = nil end rescue AWS::S3::NoSuchKey rtn = nil end rtn end
list(node)
click to toggle source
# File lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb, line 119 def list(node) #conforming to base file mgr list_attachments end
list_attachments()
click to toggle source
# File lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb, line 197 def list_attachments objs = list_objects atts = objs.map{|o| o.key} if objs atts || [] end
list_objects()
click to toggle source
# File lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb, line 185 def list_objects list = nil this_bucket = use_bucket(@bucket_name) begin list = this_bucket.objects rescue AWS::S3::NoSuchBucket puts "Rescued while listing attachments" list = retry_request{this_bucket.objects} end list end
retry_request(*args) { |*args| ... }
click to toggle source
# File lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb, line 255 def retry_request(*args, &block) puts "RETRYING Request with block: #{block.inspect}" wait_time = 2 backoff_delay = 0.5 max_retries = 10 resp = nil 1.upto(max_retries) do |i| puts "Wating #{wait_time} secs to try again" sleep wait_time begin resp = yield *args raise TypeError, "Response was Nil, retrying" unless resp break rescue AWS::S3::NoSuchKey => e raise e #we want to raise this one" rescue AWS::S3::ResponseError => e puts "rescued #{e.inspect}" backoff_delay += backoff_delay# * i wait_time += backoff_delay if (wait_time > 3) && (e.class == AWS::S3::NoSuchBucket) puts "Attempting to reset bucket" @attachment_bucket = use_bucket(@bucket_name) end next end#begin-rescue end#upto end
subtract(node, file_basenames)
click to toggle source
# File lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb, line 124 def subtract(node, file_basenames) #conforming to base file mgr subtract_files(node, file_basenames) end
subtract_all()
click to toggle source
# File lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb, line 224 def subtract_all #Changed behavior to leave bucket (this is different than other FileMgrs) this_bucket = use_bucket(@bucket_name) begin this_bucket.delete_all rescue AWS::S3::NoSuchBucket puts "Bucket not found while deleting all. Maybe it's already been deleted?" return nil #aws_names = retry_request{@attachment_bucket.objects} end max_wait_time = 20 now = Time.now while Time.now < (now + max_wait_time) begin this_bucket.delete rescue AWS::S3::BucketNotEmpty sleep 1 puts "Bucket not empty yet, trying again" this_bucket.delete_all next end break end use_bucket(@bucket_name) #file_basenames = aws_names.map{|o| o.key} if aws_names #self.subtract_some(file_basenames) if file_basenames end
subtract_files(node, file_basenames)
click to toggle source
# File lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb, line 129 def subtract_files(node, file_basenames) if file_basenames == :all subtract_all else subtract_some(file_basenames) end end
subtract_some(file_basenames)
click to toggle source
# File lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb, line 217 def subtract_some(file_basenames) file_basenames.each do |basename| attach_name = TkEscape.escape(basename) S3::S3Object.delete(attach_name, @bucket_name) end end
use_bucket(bucket_name)
click to toggle source
# File lib/glue_envs/sdb_s3/sdb_s3_files_mgr.rb, line 287 def use_bucket(bucket_name) begin bucket = S3::Bucket.find(bucket_name) rescue (AWS::S3::NoSuchBucket||NilBucketError) => e begin puts "Rescued error in use_bucket: #{e.inspect}" S3::Bucket.create(bucket_name) bucket = S3::Bucket.find(bucket_name) rescue AWS::S3::NoSuchBucket #we just made it!! bucket = retry_request(bucket_name){|buck_name| S3::Bucket.find(buck_name)} end#inner begin-rescue end#outer begin-rescue #verify bucket exists found_buckets = S3::Service.buckets(true).map{|b| b.name} unless found_buckets.include?(bucket_name) #bucket = retry(:retry_block, bucket_name){|buck_name| S3::Bucket.find(buck_name)} end#unless unless bucket puts "NIL Bucket cannot be returned" retry_request(bucket_name){|buck_name| S3::Bucket.find(buck_name)} end raise(NilBucketError, "NIL Bucket cannot be returned",nil) unless bucket return bucket end