module Qiniu::Storage

Public Class Methods

batch(command, bucket, keys) click to toggle source
# File lib/qiniu/management.rb, line 81
def batch(command, bucket, keys)
  execs = []
  keys.each do |key|
    encoded_uri = encode_entry_uri(bucket, key)
    execs << "op=/#{command}/#{encoded_uri}"
  end
  url = Config.settings[:rs_host] + "/batch"
  return HTTP.management_post(url, execs.join("&"))
end
batch_copy(*args) click to toggle source
# File lib/qiniu/management.rb, line 95
def batch_copy(*args)
  _batch_cp_or_mv('copy', *args)
end
batch_delete(bucket, keys) click to toggle source
# File lib/qiniu/management.rb, line 103
def batch_delete(bucket, keys)
  batch("delete", bucket, keys)
end
batch_move(*args) click to toggle source
# File lib/qiniu/management.rb, line 99
def batch_move(*args)
  _batch_cp_or_mv('move', *args)
end
batch_stat(bucket, keys) click to toggle source
# File lib/qiniu/management.rb, line 91
def batch_stat(bucket, keys)
  batch("stat", bucket, keys)
end
buckets() click to toggle source
# File lib/qiniu/management.rb, line 49
def buckets
  url = Config.settings[:rs_host] + '/buckets'
  return HTTP.management_post(url)
end
copy(source_bucket, source_key, target_bucket, target_key) click to toggle source
# File lib/qiniu/management.rb, line 59
def copy(source_bucket, source_key, target_bucket, target_key)
  uri = _generate_cp_or_mv_opstr('copy', source_bucket, source_key, target_bucket, target_key)
  url = Config.settings[:rs_host] + uri
  return HTTP.management_post(url)
end
delete(bucket, key) click to toggle source
# File lib/qiniu/management.rb, line 71
def delete(bucket, key)
  url = Config.settings[:rs_host] + '/delete/' + encode_entry_uri(bucket, key)
  return HTTP.management_post(url)
end
domains(bucket) click to toggle source
# File lib/qiniu/management.rb, line 139
def domains(bucket)
  url = Config.settings[:api_host] + '/v7/domain/list?tbl=' + bucket
  return HTTP.management_post(url)
end
fetch(bucket, target_url, key) click to toggle source
# File lib/qiniu/management.rb, line 76
def fetch(bucket, target_url, key)
  url = Config.fetch_host(bucket) + '/fetch/' + Utils.urlsafe_base64_encode(target_url) + '/to/' + encode_entry_uri(bucket, key)
  return HTTP.management_post(url)
end
image_mogrify_save_as(bucket, key, source_image_url, options) click to toggle source
# File lib/qiniu/management.rb, line 114
def image_mogrify_save_as(bucket, key, source_image_url, options)
  mogrify_params_string = Fop::Image.generate_mogrify_params_string(options)
  save_as(bucket, key, source_image_url, mogrify_params_string)
end
list(list_policy) click to toggle source
# File lib/qiniu/management.rb, line 119
def list(list_policy)
  url = Config.settings[:rsf_host] + '/list?' + list_policy.to_query_string()

  resp_code, resp_body, resp_headers = HTTP.management_post(url)
  if resp_code == 0 || resp_code > 299 then
    has_more = false
    return resp_code, resp_body, resp_headers, has_more, list_policy
  end

  has_more = (resp_body['marker'].is_a?(String) && resp_body['marker'] != '')
  if has_more then
    new_list_policy = list_policy.clone()
    new_list_policy.marker = resp_body['marker']
  else
    new_list_policy = list_policy
  end

  return resp_code, resp_body, resp_headers, has_more, new_list_policy
end
move(source_bucket, source_key, target_bucket, target_key) click to toggle source
# File lib/qiniu/management.rb, line 65
def move(source_bucket, source_key, target_bucket, target_key)
  uri = _generate_cp_or_mv_opstr('move', source_bucket, source_key, target_bucket, target_key)
  url = Config.settings[:rs_host] + uri
  return HTTP.management_post(url)
end
resumable_upload_with_token(uptoken, local_file, bucket, key = nil, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil) click to toggle source
# File lib/qiniu/resumable_upload.rb, line 45
def resumable_upload_with_token(uptoken,
                      local_file,
                      bucket,
                      key = nil,
                      mime_type = nil,
                      custom_meta = nil,
                      customer = nil,
                      callback_params = nil,
                      rotate = nil)
  begin
    ifile = File.open(local_file, 'rb')
    fh = FileData.new(ifile)
    fsize = fh.data_size
    key = Digest::SHA1.hexdigest(local_file + fh.mtime.to_s) if key.nil?
    if mime_type.nil? || mime_type.empty?
      mime = MIME::Types.type_for local_file
      mime_type = mime.empty? ? 'application/octet-stream' : mime[0].content_type
    end
    code, data = _resumable_upload(uptoken, fh, fsize, bucket, key, mime_type, custom_meta, customer, callback_params, rotate)
    [code, data]
  ensure
    ifile.close unless ifile.nil?
  end
end
save_as(bucket, key, source_url, op_params_string) click to toggle source
# File lib/qiniu/management.rb, line 107
def save_as(bucket, key, source_url, op_params_string)
  encoded_uri = encode_entry_uri(bucket, key)
  save_as_string = '/save-as/' + encoded_uri
  new_url = source_url + '?' + op_params_string + save_as_string
  return HTTP.management_post(new_url)
end
stat(bucket, key) click to toggle source
# File lib/qiniu/management.rb, line 54
def stat(bucket, key)
  url = Config.settings[:rs_host] + '/stat/' + encode_entry_uri(bucket, key)
  return HTTP.management_post(url)
end
upload_buffer_with_put_policy(put_policy, buf, key = nil, x_vars = nil, opts = {}) click to toggle source
# File lib/qiniu/upload.rb, line 157
def upload_buffer_with_put_policy(put_policy,
                           buf,
                           key = nil,
                           x_vars = nil,
                           opts = {})
  uptoken = Auth.generate_uptoken(put_policy)
  if key.nil? then
    key = put_policy.key
  end

  return upload_buffer_with_token(uptoken, buf, key, x_vars, opts)
rescue BucketIsMissing
  raise 'upload_buffer_with_put_policy requires :bucket option when multi_region is enabled'
end
upload_buffer_with_token(uptoken, buf, key = nil, x_vars = nil, opts = {}) click to toggle source
# File lib/qiniu/upload.rb, line 86
def upload_buffer_with_token(uptoken,
                        buf,
                        key = nil,
                        x_vars = nil,
                        opts = {})
  ### 构造 URL
  url = Config.up_host(opts[:bucket])
  url[/\/*$/] = ''
  url += '/'

  ### 构造 HTTP Body
  if buf.is_a?(String)
    data = StringIO.new(buf)
  elsif buf.respond_to?(:read)
    data = buf
  end

  data.define_singleton_method("path") do
    'NO-PATH'
  end
  data.define_singleton_method("original_filename") do
    'A-MASS-OF-DATA'
  end
  data.define_singleton_method("content_type") do
    (opts[:content_type].nil? || opts[:content_type].empty?) ? 'application/octet-stream' : opts[:content_type]
  end

  post_data = {
    :file      => data,
    :multipart => true,
  }
  if not uptoken.nil?
    post_data[:token] = uptoken
  end
  if not key.nil?
    post_data[:key] = key
  end
  if x_vars.is_a?(Hash)
    post_data.merge!(x_vars)
  end

  ### 发送请求
  HTTP.api_post(url, post_data)
rescue BucketIsMissing
  raise 'upload_buffer_with_token requires :bucket option when multi_region is enabled'
end
upload_with_put_policy(put_policy, local_file, key = nil, x_vars = nil, opts = {}) click to toggle source
授权举例

put_policy.bucket | put_policy.key | key | 语义 | 授权 :—————- | :————- | :—— | :— | :— trivial_bucket | <nil> | <nil> | 新增 | 允许,最终key为1)使用put_policy.save_key生成的值或2)资源内容的Hash值 trivial_bucket | <nil> | foo.txt | 新增 | 允许 trivial_bucket | <nil> | bar.jpg | 新增 | 允许 trivial_bucket | foo.txt | <nil> | 覆盖 | 允许,由SDK将put_policy.key赋值给key实现 trivial_bucket | foo.txt | foo.txt | 覆盖 | 允许 trivial_bucket | foo.txt | bar.jpg | 覆盖 | 禁止,put_policy.key与key不一致

# File lib/qiniu/upload.rb, line 142
def upload_with_put_policy(put_policy,
                           local_file,
                           key = nil,
                           x_vars = nil,
                           opts = {})
  uptoken = Auth.generate_uptoken(put_policy)
  if key.nil? then
    key = put_policy.key
  end

  return upload_with_token_2(uptoken, local_file, key, x_vars, opts)
rescue BucketIsMissing
  raise 'upload_with_put_policy requires :bucket option when multi_region is enabled'
end
upload_with_token(uptoken, local_file, bucket, key = nil, mime_type = nil, custom_meta = nil, callback_params = nil, enable_crc32_check = false, rotate = nil) click to toggle source
# File lib/qiniu/upload.rb, line 10
def upload_with_token(uptoken,
                      local_file,
                      bucket,
                      key = nil,
                      mime_type = nil,
                      custom_meta = nil,
                      callback_params = nil,
                      enable_crc32_check = false,
                      rotate = nil)
  action_params = _generate_action_params(
    local_file,
    bucket,
    key,
    mime_type,
    custom_meta,
    enable_crc32_check,
    rotate
  )

  if callback_params.nil?
    callback_params = {:bucket => bucket, :key => key, :mime_type => mime_type}
  end
  callback_query_string = HTTP.generate_query_string(callback_params)

  url = Config.up_host(bucket) + '/upload'
  post_data = {
    :params     => callback_query_string,
    :action     => action_params,
    :file       => File.new(local_file, 'rb'),
    :multipart  => true
  }
  if !uptoken.nil? then
    post_data[:auth] = uptoken unless uptoken.nil?
  end

  return HTTP.api_post(url, post_data)
end
upload_with_token_2(uptoken, local_file, key = nil, x_vars = nil, opts = {}) click to toggle source
# File lib/qiniu/upload.rb, line 48
def upload_with_token_2(uptoken,
                        local_file,
                        key = nil,
                        x_vars = nil,
                        opts = {})
  ### 构造URL
  url = Config.up_host(opts[:bucket])
  url[/\/*$/] = ''
  url += '/'

  ### 构造HTTP Body
  file = File.new(local_file, 'rb')
  if not opts[:content_type].nil?
    file.define_singleton_method("content_type") do
      opts[:content_type]
    end
  end

  post_data = {
    :file      => file,
    :multipart => true,
  }
  if not uptoken.nil?
    post_data[:token] = uptoken
  end
  if not key.nil?
    post_data[:key] = key
  end
  if x_vars.is_a?(Hash)
    post_data.merge!(x_vars)
  end

  ### 发送请求
  HTTP.api_post(url, post_data)
rescue BucketIsMissing
  raise 'upload_with_token_2 requires :bucket option when multi_region is enabled'
end

Private Class Methods

_batch_cp_or_mv(command, *op_args) click to toggle source
# File lib/qiniu/management.rb, line 152
def _batch_cp_or_mv(command, *op_args)
  execs = []
  op_args.each do |e|
    execs << 'op=' + _generate_cp_or_mv_opstr(command, e[0], e[1], e[2], e[3]) if e.size == 4
  end
  url = Config.settings[:rs_host] + "/batch"
  return HTTP.management_post(url, execs.join("&"))
end
_block_count(fsize) click to toggle source
# File lib/qiniu/resumable_upload.rb, line 213
def _block_count(fsize)
    ((fsize + Config.settings[:block_size] - 1) / Config.settings[:block_size]).to_i
end
_call_binary_with_token(uptoken, url, data, content_type = nil, retry_times = 0) click to toggle source
# File lib/qiniu/resumable_upload.rb, line 97
def _call_binary_with_token(uptoken, url, data, content_type = nil, retry_times = 0)
  options = {
      :headers => {
          :content_type   => 'application/octet-stream',
          'Authorization' => 'UpToken ' + uptoken
      }
  }
  if !content_type.nil? && !content_type.empty? then
      options[:headers][:content_type] = content_type
  end

  code, data, raw_headers = HTTP.api_post(url, data, options)
  unless HTTP.is_response_ok?(code)
      retry_times += 1
      if Config.settings[:auto_reconnect] && retry_times < Config.settings[:max_retry_times]
          return _call_binary_with_token(uptoken, url, data, options[:content_type], retry_times)
      end
  end
  return code, data, raw_headers
end
_generate_action_params(local_file, bucket, key = nil, mime_type = nil, custom_meta = nil, enable_crc32_check = false, rotate = nil) click to toggle source
# File lib/qiniu/upload.rb, line 173
def _generate_action_params(local_file,
                            bucket,
                            key = nil,
                            mime_type = nil,
                            custom_meta = nil,
                            enable_crc32_check = false,
                            rotate = nil)
  raise NoSuchFileError, local_file unless File.exist?(local_file)

  if key.nil?
    key = Digest::SHA1.hexdigest(local_file + Time.now.to_s)
  end

  entry_uri = bucket + ':' + key
  if mime_type.nil? || mime_type.empty?
    mime = MIME::Types.type_for local_file
    mime_type = mime.empty? ? 'application/octet-stream' : mime[0].content_type
  end

  action_params = '/rs-put/' + Utils.urlsafe_base64_encode(entry_uri) + '/mimeType/' + Utils.urlsafe_base64_encode(mime_type)
  action_params += '/meta/' + Utils.urlsafe_base64_encode(custom_meta) unless custom_meta.nil?
  action_params += '/crc32/' + Utils.crc32checksum(local_file).to_s if enable_crc32_check
  action_params += '/rotate/' + rotate if !rotate.nil? && rotate.to_i >= 0
  action_params
end
_generate_cp_or_mv_opstr(command, source_bucket, source_key, target_bucket, target_key) click to toggle source
# File lib/qiniu/management.rb, line 146
def _generate_cp_or_mv_opstr(command, source_bucket, source_key, target_bucket, target_key)
  source_encoded_entry_uri = encode_entry_uri(source_bucket, source_key)
  target_encoded_entry_uri = encode_entry_uri(target_bucket, target_key)
  %Q(/#{command}/#{source_encoded_entry_uri}/#{target_encoded_entry_uri})
end
_mkblock(bucket, uptoken, block_size, body) click to toggle source
# File lib/qiniu/resumable_upload.rb, line 118
def _mkblock(bucket, uptoken, block_size, body)
    url = Config.up_host(bucket) + "/mkblk/#{block_size}"
    _call_binary_with_token(uptoken, url, body)
end
_mkfile(uphost, uptoken, entry_uri, fsize, checksums, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil) click to toggle source
# File lib/qiniu/resumable_upload.rb, line 256
def _mkfile(uphost,
            uptoken,
            entry_uri,
            fsize,
            checksums,
            mime_type = nil,
            custom_meta = nil,
            customer = nil,
            callback_params = nil,
            rotate = nil)
  path = '/rs-mkfile/' + Utils.urlsafe_base64_encode(entry_uri) + "/fsize/#{fsize}"
  path += '/mimeType/' + Utils.urlsafe_base64_encode(mime_type) if !mime_type.nil? && !mime_type.empty?
  path += '/meta/' + Utils.urlsafe_base64_encode(custom_meta) if !custom_meta.nil? && !custom_meta.empty?
  path += '/customer/' + customer if !customer.nil? && !customer.empty?
  callback_query_string = HTTP.generate_query_string(callback_params) if !callback_params.nil? && !callback_params.empty?
  path += '/params/' + Utils.urlsafe_base64_encode(callback_query_string) if !callback_query_string.nil? && !callback_query_string.empty?
  path += '/rotate/' + rotate if !rotate.nil? && rotate.to_i >= 0
  url = uphost + path
  #body = ''
  #checksums.each do |checksum|
  #    body += Utils.urlsafe_base64_decode(checksum)
  #end
  body = checksums.join(',')
  _call_binary_with_token(uptoken, url, body, 'text/plain')
end
_new_block_put_progress_data() click to toggle source
# File lib/qiniu/resumable_upload.rb, line 93
def _new_block_put_progress_data
  {:ctx => nil, :offset => 0, :restsize => nil, :status_code => nil, :host => nil}
end
_putblock(uphost, uptoken, ctx, offset, body) click to toggle source
# File lib/qiniu/resumable_upload.rb, line 123
def _putblock(uphost, uptoken, ctx, offset, body)
    url = uphost + "/bput/#{ctx}/#{offset}"
    _call_binary_with_token(uptoken, url, body)
end
_resumable_put(bucket, uptoken, fh, checksums, progresses, block_notifier = nil, chunk_notifier = nil) click to toggle source
# File lib/qiniu/resumable_upload.rb, line 217
def _resumable_put(bucket,
                   uptoken,
                   fh,
                   checksums,
                   progresses,
                   block_notifier = nil,
                   chunk_notifier = nil)
    code, data = 0, {}
    fsize = fh.data_size
    block_count = _block_count(fsize)
    checksum_count = checksums.length
    progress_count = progresses.length
    if checksum_count != block_count || progress_count != block_count
        raise BlockCountNotMathchError.new(fh.path, block_count, checksum_count, progress_count)
    end
    0.upto(block_count-1).each do |block_index|
        if checksums[block_index].nil? || checksums[block_index].empty?
            block_size = Config.settings[:block_size]
            if block_index == block_count - 1
                block_size = fsize - block_index*Config.settings[:block_size]
            end
            if progresses[block_index].nil?
                progresses[block_index] = _new_block_put_progress_data
            end
            #code, data = _resumable_put_block(uptoken, fh, block_index, block_size, Config.settings[:chunk_size], progresses[block_index], Config.settings[:max_retry_times], chunk_notifier)
            # Put the whole block as a chunk
            code, data = _resumable_put_block(bucket, uptoken, fh, block_index, block_size, block_size, progresses[block_index], Config.settings[:max_retry_times], chunk_notifier)
            if HTTP.is_response_ok?(code)
                #checksums[block_index] = data["checksum"]
                checksums[block_index] = data["ctx"]
                if !block_notifier.nil? && block_notifier.respond_to?("notify")
                    block_notifier.notify(block_index, checksums[block_index])
                end
            end
        end
    end
    return [code, data]
end
_resumable_put_block(bucket, uptoken, fh, block_index, block_size, chunk_size, progress, retry_times, notifier) click to toggle source
# File lib/qiniu/resumable_upload.rb, line 128
def _resumable_put_block(bucket,
                         uptoken,
                         fh,
                         block_index,
                         block_size,
                         chunk_size,
                         progress,
                         retry_times,
                         notifier)
    code, data = 0, {}
    fpath = fh.path

    # this block has never been uploaded.
    if progress[:ctx] == nil || progress[:ctx].empty?
        progress[:offset] = 0
        progress[:restsize] = block_size
        # choose the smaller one
        body_length = [block_size, chunk_size].min
        for i in 1..retry_times
            seek_pos = block_index*Config.settings[:block_size]
            body = fh.get_data(seek_pos, body_length)
            result_length = body.length
            if result_length != body_length
                raise FileSeekReadError.new(fpath, block_index, seek_pos, body_length, result_length)
            end

            code, data, raw_headers = _mkblock(bucket, uptoken, block_size, body)
            Utils.debug "Mkblk : #{code.inspect} #{data.inspect} #{raw_headers.inspect}"

            body_crc32 = Zlib.crc32(body)
            if HTTP.is_response_ok?(code) && data["crc32"] == body_crc32
                progress[:ctx] = data["ctx"]
                progress[:offset] = body_length
                progress[:restsize] = block_size - body_length
                progress[:status_code] = code
                progress[:host] = data["host"]
                if !notifier.nil? && notifier.respond_to?("notify")
                    notifier.notify(block_index, progress)
                end
                break
            elsif i == retry_times && data["crc32"] != body_crc32
                Log.logger.error %Q(Uploading block error. Expected crc32: #{body_crc32}, but got: #{data["crc32"]})
                return code, data, raw_headers
            end
        end
    elsif progress[:offset] + progress[:restsize] != block_size
        raise BlockSizeNotMathchError.new(fpath, block_index, progress[:offset], progress[:restsize], block_size)
    end

    # loop uploading other chunks except the first one
    while progress[:restsize].to_i > 0 && progress[:restsize] < block_size
        # choose the smaller one
        body_length = [progress[:restsize], chunk_size].min
        for i in 1..retry_times
            seek_pos = block_index*Config.settings[:block_size] + progress[:offset]
            body = fh.get_data(seek_pos, body_length)
            result_length = body.length
            if result_length != body_length
                raise FileSeekReadError.new(fpath, block_index, seek_pos, body_length, result_length)
            end

            code, data, raw_headers = _putblock(progress[:host], uptoken, progress[:ctx], progress[:offset], body)
            Utils.debug "Bput : #{code.inspect} #{data.inspect} #{raw_headers.inspect}"

            body_crc32 = Zlib.crc32(body)
            if HTTP.is_response_ok?(code) && data["crc32"] == body_crc32
                progress[:ctx] = data["ctx"]
                progress[:offset] += body_length
                progress[:restsize] -= body_length
                progress[:status_code] = code
                progress[:host] = data["host"]
                if !notifier.nil? && notifier.respond_to?("notify")
                    notifier.notify(block_index, progress)
                end
                break
            elsif i == retry_times && data["crc32"] != body_crc32
                Log.logger.error %Q(Uploading block error. Expected crc32: #{body_crc32}, but got: #{data["crc32"]})
                return code, data, raw_headers
            end
        end
    end
    # return
    return code, data, raw_headers
end
_resumable_upload(uptoken, fh, fsize, bucket, key, mime_type = nil, custom_meta = nil, customer = nil, callback_params = nil, rotate = nil) click to toggle source
# File lib/qiniu/resumable_upload.rb, line 282
def _resumable_upload(uptoken,
                      fh,
                      fsize,
                      bucket,
                      key,
                      mime_type = nil,
                      custom_meta = nil,
                      customer = nil,
                      callback_params = nil,
                      rotate = nil)

  block_count = _block_count(fsize)

  chunk_notifier = ChunkProgressNotifier.new()
  block_notifier = BlockProgressNotifier.new()

  progresses = []
  block_count.times{progresses << _new_block_put_progress_data}
  checksums = []
  block_count.times{checksums << ''}

  code, data, raw_headers = _resumable_put(bucket, uptoken, fh, checksums, progresses, block_notifier, chunk_notifier)

  if HTTP.is_response_ok?(code)
    uphost = data["host"]
    entry_uri = bucket + ':' + key
    code, data, raw_headers = _mkfile(uphost, uptoken, entry_uri, fsize, checksums, mime_type, custom_meta, customer, callback_params, rotate)
    Utils.debug "Mkfile : #{code.inspect} #{data.inspect} #{raw_headers.inspect}"
  end

  if HTTP.is_response_ok?(code)
    Utils.debug "File #{fh.path} {size: #{fsize}} successfully uploaded."
  end

  return code, data, raw_headers
end