module RedisUtility
module RedisUtility
namespace for redis methods
Constants
- KEY_BATCH_SIZE
- VERSION
- WRITE_BATCH_SIZE
Public Instance Methods
cache(key, params = {}) { |self| ... }
click to toggle source
# File lib/redis_utility.rb, line 112 def cache(key, params = {}) expire = params[:expire] recalculate = params[:recalculate] if recalculate || (value = redis.get(key)).nil? value = MultiJson.encode(yield(self)) redis.set(key, value) redis.expire(key, expire) if expire end MultiJson.decode(value) end
cache_string(key, params = {}) { |self| ... }
click to toggle source
# File lib/redis_utility.rb, line 98 def cache_string(key, params = {}) expire = params[:expire] recalculate = params[:recalculate] if recalculate || (value = redis.get(key)).nil? value = yield(self).to_s redis.set(key, value) redis.expire(key, expire) if expire end value end
export_data(key_patterns, filename)
click to toggle source
Export the key pattern
# File lib/redis_utility.rb, line 56 def export_data(key_patterns, filename) key_patterns = [key_patterns] if key_patterns.is_a? String File.open(filename, 'w+b') do |f| key_patterns.each do |kp| allkeys = kp.include?('*') ? redis.keys(kp).sort : [kp] print "Working on #{kp}: #{quantity_with_unit(allkeys.size, 'key')}\n" nstart = 0 while nstart < allkeys.size keys = allkeys[nstart...nstart + KEY_BATCH_SIZE] types = redis.pipelined { keys.each { |k| redis.type(k) } } # print "Got types\n" string_keys = [] pkeys = [] pvals = redis.pipelined do keys.each_with_index do |key, idx| case types[idx] when 'string' string_keys << key when 'hash' pkeys << key redis.hgetall(key) when 'list' pkeys << key redis.lrange(key, 0, -1) when 'zset' pkeys << key redis.zrange(key, 0, -1, with_scores: true) else print "RedisUtility: Can not deal with #{types[idx]} for key #{key}, skipped\n" end end end write_pipelined_results(pkeys, pvals, f) write_string_keys(string_keys, f) nstart += KEY_BATCH_SIZE print '.' if nstart < allkeys.size end end end end
export_string_data(key_patterns, filename)
click to toggle source
# File lib/redis_utility.rb, line 126 def export_string_data(key_patterns, filename) key_patterns = [key_patterns] if key_patterns.is_a? String File.open(filename, 'w') do |f| key_patterns.each do |kp| keys = redis.keys(kp) write_string_keys(keys, f) end end end
import_data(file)
click to toggle source
Imports a line-by-line json string
# File lib/redis_utility.rb, line 16 def import_data(file) bzip2_openfile(file, 'rb') do |f1| until f1.eof? keys = 0 redis.pipelined do # rubocop:disable Lint/AssignmentInCondition while curr_line = f1.gets # rubocop:enable Lint/AssignmentInCondition line = JSON.parse(curr_line) line.each do |key, val| keys += 1 # first delete the record from the server before adding new value case val when Hash redis.del key redis.mapped_hmset(key, val) when Array redis.del key if val[0].is_a?(Array) && val[0][1].is_a?(Float) # zset val = val.map { |v| [v[1], v[0]] } redis.zadd(key, val) else redis.rpush(key, val) end else redis.set(key, val) end end # Done with the line if keys > KEY_BATCH_SIZE print '.' break end end end end end end
reconnect()
click to toggle source
# File lib/redis_utility.rb, line 155 def reconnect if @redis @redis._client.disconnect @redis = nil redis # This reconnects to redis with right configurations end nil end
redis()
click to toggle source
# File lib/redis_utility.rb, line 145 def redis unless @redis # print "RedisUtility: Connecting\n" cfg = redis_config.dup cfg[:timeout] = 60 # Set longer timeout for efficient bulk loading/save @redis = Redis.new(cfg) end @redis end
redis_config()
click to toggle source
# File lib/redis_utility.rb, line 137 def redis_config @redis_config end
redis_config=(redis_conf)
click to toggle source
# File lib/redis_utility.rb, line 141 def redis_config=(redis_conf) @redis_config = redis_conf end
Private Instance Methods
bzip2_openfile(file_nm, mode, &block)
click to toggle source
Opens the bzip2 and yield with File object (file itself is opened)
# File lib/redis_utility.rb, line 178 def bzip2_openfile(file_nm, mode, &block) raise 'Not implemented for writing bzip2 yet' if mode.include?('w') file_nm = file_nm.to_s if file_nm.end_with?('.bz2') raise "#{file_nm} does not exist" unless File.exist?(file_nm) file = file_nm[0..-5] File.delete(file) if File.exist?(file) system_with_print("bunzip2 -kd #{file_nm}") else file = file_nm end File.open(file, mode, &block) File.delete(file) if file_nm.end_with?('.bz2') end
quantity_with_unit(quantity, unit, unit_s = nil)
click to toggle source
# File lib/redis_utility.rb, line 167 def quantity_with_unit(quantity, unit, unit_s = nil) "#{quantity} #{quantity > 1 ? (unit_s || "#{unit}s") : unit}" end
system_with_print(cmd)
click to toggle source
# File lib/redis_utility.rb, line 171 def system_with_print(cmd) print "EXEC:#{cmd}\n" ret = system(cmd) print "EXEC failed\n" if ret.nil? end
write_pipelined_results(keys, vals, file_obj)
click to toggle source
# File lib/redis_utility.rb, line 226 def write_pipelined_results(keys, vals, file_obj) keys.each_with_index do |key, idx| begin Yajl::Encoder.encode({ key => vals[idx] }, file_obj) rescue EncodingError print "Skipped #{key}. Encoding error\n" end file_obj.write("\n") end end
write_string_keys(string_keys, file_obj)
click to toggle source
# File lib/redis_utility.rb, line 218 def write_string_keys(string_keys, file_obj) while string_keys.size > WRITE_BATCH_SIZE first_chunk = string_keys.shift(WRITE_BATCH_SIZE) write_string_keys_chunk(first_chunk, file_obj) end write_string_keys_chunk(string_keys, file_obj) end
write_string_keys_chunk(string_keys, file_obj)
click to toggle source
# File lib/redis_utility.rb, line 197 def write_string_keys_chunk(string_keys, file_obj) return unless string_keys.size.positive? hash = {} string_vals = redis.mget(string_keys) string_keys.each_with_index do |key, idx| val = string_vals[idx] next unless val next if key.start_with?('YMProd:rails_cache') # Migration to avoid legacy cache if val.nil? print "RedisUtility: Can not get value for key #{key}, skipped\n" else hash[key] = val end end Yajl::Encoder.encode(hash, file_obj) file_obj.write("\n") end