module Qiniu::Storage
Public Class Methods
batch(command, bucket, keys)
click to toggle source
# File lib/qiniu/management.rb, line 84 def batch(command, bucket, keys) execs = [] keys.each do |key| encoded_uri = encode_entry_uri(bucket, key) execs << "op=/#{command}/#{encoded_uri}" end url = Config.settings[:rs_host] + "/batch" return HTTP.management_post(url, execs.join("&")) end
batch_copy(*args)
click to toggle source
# File lib/qiniu/management.rb, line 102 def batch_copy(*args) _batch_cp_or_mv('copy', args) end
batch_delete(bucket, keys)
click to toggle source
# File lib/qiniu/management.rb, line 110 def batch_delete(bucket, keys) batch("delete", bucket, keys) end
batch_get(bucket, keys)
click to toggle source
# File lib/qiniu/management.rb, line 94 def batch_get(bucket, keys) batch("get", bucket, keys) end
batch_move(*args)
click to toggle source
# File lib/qiniu/management.rb, line 106 def batch_move(*args) _batch_cp_or_mv('move', args) end
batch_stat(bucket, keys)
click to toggle source
# File lib/qiniu/management.rb, line 98 def batch_stat(bucket, keys) batch("stat", bucket, keys) end
buckets()
click to toggle source
# File lib/qiniu/management.rb, line 49 def buckets url = Config.settings[:rs_host] + '/buckets' return HTTP.management_post(url) end
copy(source_bucket, source_key, target_bucket, target_key)
click to toggle source
# File lib/qiniu/management.rb, line 67 def copy(source_bucket, source_key, target_bucket, target_key) uri = _generate_cp_or_mv_opstr('copy', source_bucket, source_key, target_bucket, target_key) url = Config.settings[:rs_host] + uri return HTTP.management_post(url) end
delete(bucket, key)
click to toggle source
# File lib/qiniu/management.rb, line 79 def delete(bucket, key) url = Config.settings[:rs_host] + '/delete/' + encode_entry_uri(bucket, key) return HTTP.management_post(url) end
get(bucket, key, save_as = nil, expires_in = nil, version = nil)
click to toggle source
# File lib/qiniu/management.rb, line 59 def get(bucket, key, save_as = nil, expires_in = nil, version = nil) url = Config.settings[:rs_host] + '/get/' + encode_entry_uri(bucket, key) url += '/base/' + version unless version.nil? url += '/attName/' + Utils.urlsafe_base64_encode(save_as) unless save_as.nil? url += '/expires/' + expires_in.to_s if !expires_in.nil? && expires_in > 0 return HTTP.management_post(url) end
image_mogrify_save_as(bucket, key, source_image_url, options)
click to toggle source
# File lib/qiniu/management.rb, line 121 def image_mogrify_save_as(bucket, key, source_image_url, options) mogrify_params_string = Fop::Image.generate_mogrify_params_string(options) save_as(bucket, key, source_image_url, mogrify_params_string) end
list(list_policy)
click to toggle source
# File lib/qiniu/management.rb, line 126 def list(list_policy) url = Config.settings[:rsf_host] + '/list?' + list_policy.to_query_string() resp_code, resp_body, resp_headers = HTTP.management_post(url) if resp_code == 0 || resp_code > 299 then has_more = false return resp_code, resp_body, resp_headers, has_more, list_policy end has_more = (resp_body['marker'].is_a?(String) && resp_body['marker'] != '') if has_more then new_list_policy = list_policy.clone() new_list_policy.marker = resp_body['marker'] else new_list_policy = list_policy end return resp_code, resp_body, resp_headers, has_more, new_list_policy end
move(source_bucket, source_key, target_bucket, target_key)
click to toggle source
# File lib/qiniu/management.rb, line 73 def move(source_bucket, source_key, target_bucket, target_key) uri = _generate_cp_or_mv_opstr('move', source_bucket, source_key, target_bucket, target_key) url = Config.settings[:rs_host] + uri return HTTP.management_post(url) end
resumable_upload_with_token(uptoken, local_file, bucket, key = nil, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil)
click to toggle source
# File lib/qiniu/resumable_upload.rb, line 45 def resumable_upload_with_token(uptoken, local_file, bucket, key = nil, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil) begin ifile = File.open(local_file, 'rb') fh = FileData.new(ifile) fsize = fh.data_size key = Digest::SHA1.hexdigest(local_file + fh.mtime.to_s) if key.nil? if mime_type.nil? || mime_type.empty? mime = MIME::Types.type_for local_file mime_type = mime.empty? ? 'application/octet-stream' : mime[0].content_type end code, data = _resumable_upload(uptoken, fh, fsize, bucket, key, mime_type, custom_meta, customer, callback_params, rotate) [code, data] ensure ifile.close unless ifile.nil? end end
save_as(bucket, key, source_url, op_params_string)
click to toggle source
# File lib/qiniu/management.rb, line 114 def save_as(bucket, key, source_url, op_params_string) encoded_uri = encode_entry_uri(bucket, key) save_as_string = '/save-as/' + encoded_uri new_url = source_url + '?' + op_params_string + save_as_string return HTTP.management_post(new_url) end
stat(bucket, key)
click to toggle source
# File lib/qiniu/management.rb, line 54 def stat(bucket, key) url = Config.settings[:rs_host] + '/stat/' + encode_entry_uri(bucket, key) return HTTP.management_post(url) end
upload_with_put_policy(put_policy, local_file, key = nil, x_vars = nil, opts = {})
click to toggle source
授权举例
put_policy.bucket | put_policy.key | key | 语义 | 授权 :—————- | :————- | :—— | :— | :— trivial_bucket | <nil> | <nil> | 新增 | 允许,最终key为1)使用put_policy.save_key生成的值或2)资源内容的Hash值 trivial_bucket | <nil> | foo.txt | 新增 | 允许 trivial_bucket | <nil> | bar.jpg | 新增 | 允许 trivial_bucket | foo.txt | <nil> | 覆盖 | 允许,由SDK将put_policy.key赋值给key实现 trivial_bucket | foo.txt | foo.txt | 覆盖 | 允许 trivial_bucket | foo.txt | bar.jpg | 覆盖 | 禁止,put_policy.key与key不一致
# File lib/qiniu/upload.rb, line 97 def upload_with_put_policy(put_policy, local_file, key = nil, x_vars = nil, opts = {}) uptoken = Auth.generate_uptoken(put_policy) if key.nil? then key = put_policy.key end return upload_with_token_2(uptoken, local_file, key, x_vars, opts) end
upload_with_token(uptoken, local_file, bucket, key = nil, mime_type = nil, custom_meta = nil, callback_params = nil, enable_crc32_check = false, rotate = nil)
click to toggle source
# File lib/qiniu/upload.rb, line 8 def upload_with_token(uptoken, local_file, bucket, key = nil, mime_type = nil, custom_meta = nil, callback_params = nil, enable_crc32_check = false, rotate = nil) action_params = _generate_action_params( local_file, bucket, key, mime_type, custom_meta, enable_crc32_check, rotate ) if callback_params.nil? callback_params = {:bucket => bucket, :key => key, :mime_type => mime_type} end callback_query_string = HTTP.generate_query_string(callback_params) url = Config.settings[:up_host] + '/upload' post_data = { :params => callback_query_string, :action => action_params, :file => File.new(local_file, 'rb'), :multipart => true } if !uptoken.nil? then post_data[:auth] = uptoken unless uptoken.nil? end return HTTP.api_post(url, post_data) end
upload_with_token_2(uptoken, local_file, key = nil, x_vars = nil, opts = {})
click to toggle source
# File lib/qiniu/upload.rb, line 46 def upload_with_token_2(uptoken, local_file, key = nil, x_vars = nil, opts = {}) ### 构造URL url = Config.settings[:up_host] url[/\/*$/] = '' url += '/' ### 构造HTTP Body if local_file.is_a?(String) tmp = Tempfile.new('foo') tmp.write(local_file) tmp.close local_file = tmp end file = File.new(local_file, 'rb') if not opts[:content_type].nil? file.define_singleton_method("content_type") do opts[:content_type] end end post_data = { :file => file, :multipart => true, } if not uptoken.nil? post_data[:token] = uptoken end if not key.nil? post_data[:key] = key end if x_vars.is_a?(Hash) post_data.merge!(x_vars) end ### 发送请求 HTTP.api_post(url, post_data) end
Private Class Methods
_batch_cp_or_mv(command, *op_args)
click to toggle source
# File lib/qiniu/management.rb, line 154 def _batch_cp_or_mv(command, *op_args) execs = [] op_args.each do |e| execs << 'op=' + _generate_cp_or_mv_opstr(command, e[0], e[1], e[2], e[3]) if e.size == 4 end url = Config.settings[:rs_host] + "/batch" return HTTP.management_post(url, execs.join("&")) end
_block_count(fsize)
click to toggle source
# File lib/qiniu/resumable_upload.rb, line 212 def _block_count(fsize) ((fsize + Config.settings[:block_size] - 1) / Config.settings[:block_size]).to_i end
_call_binary_with_token(uptoken, url, data, content_type = nil, retry_times = 0)
click to toggle source
# File lib/qiniu/resumable_upload.rb, line 97 def _call_binary_with_token(uptoken, url, data, content_type = nil, retry_times = 0) options = { :headers => { :content_type => 'application/octet-stream', 'Authorization' => 'UpToken ' + uptoken } } if !content_type.nil? && !content_type.empty? then options[:headers][:content_type] = content_type end code, data, raw_headers = HTTP.api_post(url, data, options) unless HTTP.is_response_ok?(code) retry_times += 1 if Config.settings[:auto_reconnect] && retry_times < Config.settings[:max_retry_times] return _call_binary_with_token(uptoken, url, data, options[:content_type], retry_times) end end return code, data, raw_headers end
_generate_action_params(local_file, bucket, key = nil, mime_type = nil, custom_meta = nil, enable_crc32_check = false, rotate = nil)
click to toggle source
# File lib/qiniu/upload.rb, line 111 def _generate_action_params(local_file, bucket, key = nil, mime_type = nil, custom_meta = nil, enable_crc32_check = false, rotate = nil) raise NoSuchFileError, local_file unless File.exist?(local_file) if key.nil? key = Digest::SHA1.hexdigest(local_file + Time.now.to_s) end entry_uri = bucket + ':' + key if mime_type.nil? || mime_type.empty? mime = MIME::Types.type_for local_file mime_type = mime.empty? ? 'application/octet-stream' : mime[0].content_type end action_params = '/rs-put/' + Utils.urlsafe_base64_encode(entry_uri) + '/mimeType/' + Utils.urlsafe_base64_encode(mime_type) action_params += '/meta/' + Utils.urlsafe_base64_encode(custom_meta) unless custom_meta.nil? action_params += '/crc32/' + Utils.crc32checksum(local_file).to_s if enable_crc32_check action_params += '/rotate/' + rotate if !rotate.nil? && rotate.to_i >= 0 action_params end
_generate_cp_or_mv_opstr(command, source_bucket, source_key, target_bucket, target_key)
click to toggle source
# File lib/qiniu/management.rb, line 148 def _generate_cp_or_mv_opstr(command, source_bucket, source_key, target_bucket, target_key) source_encoded_entry_uri = encode_entry_uri(source_bucket, source_key) target_encoded_entry_uri = encode_entry_uri(target_bucket, target_key) %Q(/#{command}/#{source_encoded_entry_uri}/#{target_encoded_entry_uri}) end
_mkblock(uptoken, block_size, body)
click to toggle source
# File lib/qiniu/resumable_upload.rb, line 118 def _mkblock(uptoken, block_size, body) url = Config.settings[:up_host] + "/mkblk/#{block_size}" _call_binary_with_token(uptoken, url, body) end
_mkfile(uphost, uptoken, entry_uri, fsize, checksums, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil)
click to toggle source
# File lib/qiniu/resumable_upload.rb, line 254 def _mkfile(uphost, uptoken, entry_uri, fsize, checksums, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil) path = '/rs-mkfile/' + Utils.urlsafe_base64_encode(entry_uri) + "/fsize/#{fsize}" path += '/mimeType/' + Utils.urlsafe_base64_encode(mime_type) if !mime_type.nil? && !mime_type.empty? path += '/meta/' + Utils.urlsafe_base64_encode(custom_meta) if !custom_meta.nil? && !custom_meta.empty? path += '/customer/' + customer if !customer.nil? && !customer.empty? callback_query_string = HTTP.generate_query_string(callback_params) if !callback_params.nil? && !callback_params.empty? path += '/params/' + Utils.urlsafe_base64_encode(callback_query_string) if !callback_query_string.nil? && !callback_query_string.empty? path += '/rotate/' + rotate if !rotate.nil? && rotate.to_i >= 0 url = uphost + path #body = '' #checksums.each do |checksum| # body += Utils.urlsafe_base64_decode(checksum) #end body = checksums.join(',') _call_binary_with_token(uptoken, url, body, 'text/plain') end
_new_block_put_progress_data()
click to toggle source
# File lib/qiniu/resumable_upload.rb, line 93 def _new_block_put_progress_data {:ctx => nil, :offset => 0, :restsize => nil, :status_code => nil, :host => nil} end
_putblock(uphost, uptoken, ctx, offset, body)
click to toggle source
# File lib/qiniu/resumable_upload.rb, line 123 def _putblock(uphost, uptoken, ctx, offset, body) url = uphost + "/bput/#{ctx}/#{offset}" _call_binary_with_token(uptoken, url, body) end
_resumable_put(uptoken, fh, checksums, progresses, block_notifier = nil, chunk_notifier = nil)
click to toggle source
# File lib/qiniu/resumable_upload.rb, line 216 def _resumable_put(uptoken, fh, checksums, progresses, block_notifier = nil, chunk_notifier = nil) code, data = 0, {} fsize = fh.data_size block_count = _block_count(fsize) checksum_count = checksums.length progress_count = progresses.length if checksum_count != block_count || progress_count != block_count raise BlockCountNotMathchError.new(fh.path, block_count, checksum_count, progress_count) end 0.upto(block_count-1).each do |block_index| if checksums[block_index].nil? || checksums[block_index].empty? block_size = Config.settings[:block_size] if block_index == block_count - 1 block_size = fsize - block_index*Config.settings[:block_size] end if progresses[block_index].nil? progresses[block_index] = _new_block_put_progress_data end #code, data = _resumable_put_block(uptoken, fh, block_index, block_size, Config.settings[:chunk_size], progresses[block_index], Config.settings[:max_retry_times], chunk_notifier) # Put the whole block as a chunk code, data = _resumable_put_block(uptoken, fh, block_index, block_size, block_size, progresses[block_index], Config.settings[:max_retry_times], chunk_notifier) if HTTP.is_response_ok?(code) #checksums[block_index] = data["checksum"] checksums[block_index] = data["ctx"] if !block_notifier.nil? && block_notifier.respond_to?("notify") block_notifier.notify(block_index, checksums[block_index]) end end end end return [code, data] end
_resumable_put_block(uptoken, fh, block_index, block_size, chunk_size, progress, retry_times, notifier)
click to toggle source
# File lib/qiniu/resumable_upload.rb, line 128 def _resumable_put_block(uptoken, fh, block_index, block_size, chunk_size, progress, retry_times, notifier) code, data = 0, {} fpath = fh.path # this block has never been uploaded. if progress[:ctx] == nil || progress[:ctx].empty? progress[:offset] = 0 progress[:restsize] = block_size # choose the smaller one body_length = [block_size, chunk_size].min for i in 1..retry_times seek_pos = block_index*Config.settings[:block_size] body = fh.get_data(seek_pos, body_length) result_length = body.length if result_length != body_length raise FileSeekReadError.new(fpath, block_index, seek_pos, body_length, result_length) end code, data, raw_headers = _mkblock(uptoken, block_size, body) Utils.debug "Mkblk : #{code.inspect} #{data.inspect} #{raw_headers.inspect}" body_crc32 = Zlib.crc32(body) if HTTP.is_response_ok?(code) && data["crc32"] == body_crc32 progress[:ctx] = data["ctx"] progress[:offset] = body_length progress[:restsize] = block_size - body_length progress[:status_code] = code progress[:host] = data["host"] if !notifier.nil? && notifier.respond_to?("notify") notifier.notify(block_index, progress) end break elsif i == retry_times && data["crc32"] != body_crc32 Log.logger.error %Q(Uploading block error. Expected crc32: #{body_crc32}, but got: #{data["crc32"]}) return code, data, raw_headers end end elsif progress[:offset] + progress[:restsize] != block_size raise BlockSizeNotMathchError.new(fpath, block_index, progress[:offset], progress[:restsize], block_size) end # loop uploading other chunks except the first one while progress[:restsize].to_i > 0 && progress[:restsize] < block_size # choose the smaller one body_length = [progress[:restsize], chunk_size].min for i in 1..retry_times seek_pos = block_index*Config.settings[:block_size] + progress[:offset] body = fh.get_data(seek_pos, body_length) result_length = body.length if result_length != body_length raise FileSeekReadError.new(fpath, block_index, seek_pos, body_length, result_length) end code, data, raw_headers = _putblock(progress[:host], uptoken, progress[:ctx], progress[:offset], body) Utils.debug "Bput : #{code.inspect} #{data.inspect} #{raw_headers.inspect}" body_crc32 = Zlib.crc32(body) if HTTP.is_response_ok?(code) && data["crc32"] == body_crc32 progress[:ctx] = data["ctx"] progress[:offset] += body_length progress[:restsize] -= body_length progress[:status_code] = code progress[:host] = data["host"] if !notifier.nil? && notifier.respond_to?("notify") notifier.notify(block_index, progress) end break elsif i == retry_times && data["crc32"] != body_crc32 Log.logger.error %Q(Uploading block error. Expected crc32: #{body_crc32}, but got: #{data["crc32"]}) return code, data, raw_headers end end end # return return code, data, raw_headers end
_resumable_upload(uptoken, fh, fsize, bucket, key, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil)
click to toggle source
# File lib/qiniu/resumable_upload.rb, line 280 def _resumable_upload(uptoken, fh, fsize, bucket, key, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil) block_count = _block_count(fsize) chunk_notifier = ChunkProgressNotifier.new() block_notifier = BlockProgressNotifier.new() progresses = [] block_count.times{progresses << _new_block_put_progress_data} checksums = [] block_count.times{checksums << ''} code, data, raw_headers = _resumable_put(uptoken, fh, checksums, progresses, block_notifier, chunk_notifier) if HTTP.is_response_ok?(code) uphost = data["host"] entry_uri = bucket + ':' + key code, data, raw_headers = _mkfile(uphost, uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params, rotate) Utils.debug "Mkfile : #{code.inspect} #{data.inspect} #{raw_headers.inspect}" end if HTTP.is_response_ok?(code) Utils.debug "File #{fh.path} {size: #{fsize}} successfully uploaded." end return code, data, raw_headers end