module RedisUtility

module RedisUtility namespace for redis methods

Constants

KEY_BATCH_SIZE
VERSION
WRITE_BATCH_SIZE

Public Instance Methods

cache(key, params = {}) { |self| ... } click to toggle source
# File lib/redis_utility.rb, line 112
def cache(key, params = {})
  expire = params[:expire]
  recalculate = params[:recalculate]

  if recalculate || (value = redis.get(key)).nil?
    value = MultiJson.encode(yield(self))

    redis.set(key, value)
    redis.expire(key, expire) if expire
  end

  MultiJson.decode(value)
end
cache_string(key, params = {}) { |self| ... } click to toggle source
# File lib/redis_utility.rb, line 98
def cache_string(key, params = {})
  expire = params[:expire]
  recalculate = params[:recalculate]

  if recalculate || (value = redis.get(key)).nil?
    value = yield(self).to_s

    redis.set(key, value)
    redis.expire(key, expire) if expire
  end

  value
end
export_data(key_patterns, filename) click to toggle source

Export the key pattern

# File lib/redis_utility.rb, line 56
def export_data(key_patterns, filename)
  key_patterns = [key_patterns] if key_patterns.is_a? String

  File.open(filename, 'w+b') do |f|
    key_patterns.each do |kp|
      allkeys = kp.include?('*') ? redis.keys(kp).sort : [kp]
      print "Working on #{kp}: #{quantity_with_unit(allkeys.size, 'key')}\n"
      nstart = 0
      while nstart < allkeys.size
        keys = allkeys[nstart...nstart + KEY_BATCH_SIZE]
        types = redis.pipelined { keys.each { |k| redis.type(k) } }
        # print "Got types\n"
        string_keys = []
        pkeys = []
        pvals = redis.pipelined do
          keys.each_with_index do |key, idx|
            case types[idx]
            when 'string'
              string_keys << key
            when 'hash'
              pkeys << key
              redis.hgetall(key)
            when 'list'
              pkeys << key
              redis.lrange(key, 0, -1)
            when 'zset'
              pkeys << key
              redis.zrange(key, 0, -1, with_scores: true)
            else
              print "RedisUtility: Can not deal with #{types[idx]} for key #{key}, skipped\n"
            end
          end
        end
        write_pipelined_results(pkeys, pvals, f)
        write_string_keys(string_keys, f)
        nstart += KEY_BATCH_SIZE
        print '.' if nstart < allkeys.size
      end
    end
  end
end
export_string_data(key_patterns, filename) click to toggle source
# File lib/redis_utility.rb, line 126
def export_string_data(key_patterns, filename)
  key_patterns = [key_patterns] if key_patterns.is_a? String

  File.open(filename, 'w') do |f|
    key_patterns.each do |kp|
      keys = redis.keys(kp)
      write_string_keys(keys, f)
    end
  end
end
import_data(file) click to toggle source

Imports a line-by-line json string

# File lib/redis_utility.rb, line 16
def import_data(file)
  bzip2_openfile(file, 'rb') do |f1|
    until f1.eof?
      keys = 0
      redis.pipelined do
        # rubocop:disable Lint/AssignmentInCondition
        while curr_line = f1.gets
          # rubocop:enable Lint/AssignmentInCondition
          line = JSON.parse(curr_line)
          line.each do |key, val|
            keys += 1
            # first delete the record from the server before adding new value
            case val
            when Hash
              redis.del key
              redis.mapped_hmset(key, val)
            when Array
              redis.del key
              if val[0].is_a?(Array) && val[0][1].is_a?(Float) # zset
                val = val.map { |v| [v[1], v[0]] }
                redis.zadd(key, val)
              else
                redis.rpush(key, val)
              end
            else
              redis.set(key, val)
            end
          end
          # Done with the line
          if keys > KEY_BATCH_SIZE
            print '.'
            break
          end
        end
      end
    end
  end
end
reconnect() click to toggle source
# File lib/redis_utility.rb, line 155
def reconnect
  if @redis
    @redis._client.disconnect
    @redis = nil
    redis # This reconnects to redis with right configurations
  end
  nil
end
redis() click to toggle source
# File lib/redis_utility.rb, line 145
def redis
  unless @redis
    # print "RedisUtility: Connecting\n"
    cfg = redis_config.dup
    cfg[:timeout] = 60 # Set longer timeout for efficient bulk loading/save
    @redis = Redis.new(cfg)
  end
  @redis
end
redis_config() click to toggle source
# File lib/redis_utility.rb, line 137
def redis_config
  @redis_config
end
redis_config=(redis_conf) click to toggle source
# File lib/redis_utility.rb, line 141
def redis_config=(redis_conf)
  @redis_config = redis_conf
end

Private Instance Methods

bzip2_openfile(file_nm, mode, &block) click to toggle source

Opens the bzip2 and yield with File object (file itself is opened)

# File lib/redis_utility.rb, line 178
def bzip2_openfile(file_nm, mode, &block)
  raise 'Not implemented for writing bzip2 yet' if mode.include?('w')

  file_nm = file_nm.to_s
  if file_nm.end_with?('.bz2')
    raise "#{file_nm} does not exist" unless File.exist?(file_nm)

    file = file_nm[0..-5]
    File.delete(file) if File.exist?(file)
    system_with_print("bunzip2 -kd #{file_nm}")
  else
    file = file_nm
  end

  File.open(file, mode, &block)

  File.delete(file) if file_nm.end_with?('.bz2')
end
quantity_with_unit(quantity, unit, unit_s = nil) click to toggle source
# File lib/redis_utility.rb, line 167
def quantity_with_unit(quantity, unit, unit_s = nil)
  "#{quantity} #{quantity > 1 ? (unit_s || "#{unit}s") : unit}"
end
system_with_print(cmd) click to toggle source
# File lib/redis_utility.rb, line 171
def system_with_print(cmd)
  print "EXEC:#{cmd}\n"
  ret = system(cmd)
  print "EXEC failed\n" if ret.nil?
end
write_pipelined_results(keys, vals, file_obj) click to toggle source
# File lib/redis_utility.rb, line 226
def write_pipelined_results(keys, vals, file_obj)
  keys.each_with_index do |key, idx|
    begin
      Yajl::Encoder.encode({ key => vals[idx] }, file_obj)
    rescue EncodingError
      print "Skipped #{key}. Encoding error\n"
    end
    file_obj.write("\n")
  end
end
write_string_keys(string_keys, file_obj) click to toggle source
# File lib/redis_utility.rb, line 218
def write_string_keys(string_keys, file_obj)
  while string_keys.size > WRITE_BATCH_SIZE
    first_chunk = string_keys.shift(WRITE_BATCH_SIZE)
    write_string_keys_chunk(first_chunk, file_obj)
  end
  write_string_keys_chunk(string_keys, file_obj)
end
write_string_keys_chunk(string_keys, file_obj) click to toggle source
# File lib/redis_utility.rb, line 197
def write_string_keys_chunk(string_keys, file_obj)
  return unless string_keys.size.positive?

  hash = {}
  string_vals = redis.mget(string_keys)
  string_keys.each_with_index do |key, idx|
    val = string_vals[idx]
    next unless val
    next if key.start_with?('YMProd:rails_cache') # Migration to avoid legacy cache

    if val.nil?
      print "RedisUtility: Can not get value for key #{key}, skipped\n"
    else
      hash[key] = val
    end
  end
  Yajl::Encoder.encode(hash, file_obj)
  file_obj.write("\n")
end