class Object
Public Instance Methods
aggressive_deep_symbolize_keys(maybe)
click to toggle source
# File lib/trials/utils/various.rb, line 30 def aggressive_deep_symbolize_keys(maybe) return maybe.deep_symbolize_keys if maybe.respond_to?(:deep_symbolize_keys) return maybe.map { |i| aggressive_deep_symbolize_keys(i) } if maybe.respond_to?(:each) maybe end
clean_xml(str)
click to toggle source
# File lib/trials/utils/xmls.rb, line 9 def clean_xml(str) Nokogiri::XML(str.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')).to_xml end
create_db(db_name = nil)
click to toggle source
# File lib/trials/utils/sqls.rb, line 8 def create_db(db_name = nil) db(db_name) end
db(db_name = nil)
click to toggle source
# File lib/trials/utils/sqls.rb, line 1 def db(db_name = nil) db_name ||= 'data.db' db = SQLite3::Database.new(cache_path(db_name)) db.results_as_hash = true db end
ddb_connection()
click to toggle source
# File lib/trials/utils/aws.rb, line 1 def ddb_connection @connection ||= Aws::DynamoDB::Client.new( access_key_id: secrets.aws.key, secret_access_key: secrets.aws.secret, region: secrets.aws.region, ) end
ddb_scan(query)
click to toggle source
# File lib/trials/utils/aws.rb, line 13 def ddb_scan(query) segmentation = query.delete(:segmentation) || 4 threads = (0..segmentation - 1).map do |segment| Thread.new do Thread.current[:output] = ddb_scan_without_segmentation( query.merge( total_segments: segmentation, segment: segment, ), ) end end threads.each(&:join) threads.map { |t| t[:output] }.flatten end
ddb_scan_with_cache(query)
click to toggle source
# File lib/trials/utils/aws.rb, line 9 def ddb_scan_with_cache(query) json_cache(query.dig(:table_name)) { ddb_scan(query) } end
ddb_scan_without_segmentation(query)
click to toggle source
# File lib/trials/utils/aws.rb, line 32 def ddb_scan_without_segmentation(query) result = nil items = [] loop do break unless result.blank? || result.last_evaluated_key.present? result = ddb_connection.scan(query.merge(exclusive_start_key: result&.last_evaluated_key)) items += result.items.compact.map(&:symbolize_keys) end items end
ddb_upload_items(table, all_items) { |items, i * 25| ... }
click to toggle source
# File lib/trials/utils/aws.rb, line 46 def ddb_upload_items(table, all_items) all_items.each_slice(25).with_index do |items, i| next unless items.any? begin ddb_connection.batch_write_item( request_items: { table => items.compact.map do |item| { put_request: { item: item } } end } ) end yield(items, i * 25) if block_given? end end
float?(string)
click to toggle source
# File lib/trials/utils/various.rb, line 20 def float?(string) true if Float(string) rescue false end
float_or_nil(thing)
click to toggle source
# File lib/trials/utils/various.rb, line 24 def float_or_nil(thing) Float(thing) rescue StandardError nil end
gd_session()
click to toggle source
# File lib/trials/utils/google_drive.rb, line 1 def gd_session @gd_session ||= begin write_tmp('config.json', secrets.google.drive_config_json) session = GoogleDrive::Session.from_config(tmp_path("config.json")) delete('config.json') session end end
get_db(db_name = nil)
click to toggle source
# File lib/trials/utils/sqls.rb, line 12 def get_db(db_name = nil) db(db_name) end
get_rollbar_items(token:, status: 'active') { |current_page, items| ... }
click to toggle source
# File lib/trials/utils/rollbar.rb, line 10 def get_rollbar_items(token:, status: 'active') current_page = 0 items = [] loop do single_page_of_items = get_single_rollbar_items_page(token: token, page: current_page, status: status) items += single_page_of_items break if single_page_of_items.empty? break if yield(current_page, items) current_page += 1 end items.compact.uniq end
get_single_rollbar_items_page(token:, page:, status: 'active')
click to toggle source
# File lib/trials/utils/rollbar.rb, line 1 def get_single_rollbar_items_page(token:, page:, status: 'active') url = URI("https://api.rollbar.com/api/1/instances/?access_token=#{token}&page=#{page}&status=#{status}") http = Net::HTTP.new(url.host, url.port) http.use_ssl = true http.verify_mode = OpenSSL::SSL::VERIFY_NONE request = Net::HTTP::Get.new(url) JSON.parse(http.request(request).body).deep_symbolize_keys.dig(:result, :instances) end
hash_from_xml(str)
click to toggle source
# File lib/trials/utils/xmls.rb, line 1 def hash_from_xml(str) Hash.from_xml(clean_xml(str)).deep_symbolize_keys end
import_csv_into_db(db_name = 'data.db', table, csv)
click to toggle source
# File lib/trials/utils/sqls.rb, line 16 def import_csv_into_db(db_name = 'data.db', table, csv) system("sqlite3 -csv #{cache_path(db_name)} '.import #{seed_path(csv)} #{table}'") end
invalidate_json_cache()
click to toggle source
# File lib/trials/utils/jsons.rb, line 20 def invalidate_json_cache delete_cache("json") end
json_cache(key) { || ... }
click to toggle source
# File lib/trials/utils/jsons.rb, line 11 def json_cache(key) name = "json/#{key}.json" return aggressive_deep_symbolize_keys(JSON.parse(read_cache(name))) if cache_exists?(name) write_cache(name, yield.to_json) json_cache(key) end
l(item, nl: true, quiet: false)
click to toggle source
# File lib/trials/utils/logging.rb, line 40 def l(item, nl: true, quiet: false) log(item, nl: nl, quiet: quiet, each: false) end
log(item, nl: true, quiet: false, each: true)
click to toggle source
# File lib/trials/utils/logging.rb, line 16 def log(item, nl: true, quiet: false, each: true) item ||= '' if each && item.is_a?(Array) item.each { |i| log(i, nl: nl, quiet: quiet, each: false) } return end File.open(result_path('log.txt'), 'a') do |f| f << begin if item.is_a?(String) || item.is_a?(Numeric) item.to_s else PP.pp(item, '').chomp end end f << "\n" if nl end print item unless quiet puts '' if nl end
marshal_fetch(key) { || ... }
click to toggle source
# File lib/trials/utils/various.rb, line 37 def marshal_fetch(key) return Marshal.load(read_cache(key)) if cache_exists?(key) File.open(cache_path(key), 'wb') do |f| f.write(Marshal.dump(yield)) end marshal_fetch(key) end
normalize_string(string)
click to toggle source
# File lib/trials/utils/strings.rb, line 1 def normalize_string(string) string.chomp.strip.squish end
or_nil() { || ... }
click to toggle source
# File lib/trials/utils/various.rb, line 13 def or_nil val = yield raise if val.blank? val rescue StandardError end
pdf_to_text(path)
click to toggle source
# File lib/trials/utils/pdfs.rb, line 1 def pdf_to_text(path) `pdftotext "#{seed_path(path)}" #{tmp_path('tmp_pdf.txt')}; \ cat #{tmp_path('tmp_pdf.txt')}` end
prettify_xml(str)
click to toggle source
# File lib/trials/utils/xmls.rb, line 5 def prettify_xml(str) Nokogiri::XML(str) { |c| c.default_xml.noblanks }.to_xml(indent: 2) end
query_db(db_name = nil, query)
click to toggle source
# File lib/trials/utils/sqls.rb, line 20 def query_db(db_name = nil, query) db(db_name).execute(query).map(&:symbolize_keys) end
read_csv(filename)
click to toggle source
# File lib/trials/utils/csvs.rb, line 1 def read_csv(filename) CSV .foreach(seed_path(filename), headers: true) .map(&:to_h) .map(&:symbolize_keys) .select { |i| i.values.any?(&:present?) } end
read_json(filename)
click to toggle source
# File lib/trials/utils/jsons.rb, line 1 def read_json(filename) result = JSON.parse(read_seed(filename)) aggressive_deep_symbolize_keys(result) end
render_table_from_hashes(hash_set, sort: true, headers: nil)
click to toggle source
# File lib/trials/utils/logging.rb, line 1 def render_table_from_hashes(hash_set, sort: true, headers: nil) return 'no data' if hash_set.blank? headers = headers || hash_set.to_harray.uniq_keys headers.sort! if sort content = hash_set .map { |h| h.select { |k, v| v.present? }.to_h } .map { |hash| headers.map { |h| hash.dig(h) } } .map { |r| r.map(&:to_s) } TTY::Table.new(header: headers, rows: content).render(:unicode).to_s end
root_path()
click to toggle source
# File lib/trials/utils/various.rb, line 9 def root_path ROOT end
run()
click to toggle source
# File lib/trials/utils/various.rb, line 5 def run RUN end
secrets()
click to toggle source
# File lib/trials/utils/various.rb, line 1 def secrets SECRETS end
string_encode_utf_8(string)
click to toggle source
# File lib/trials/utils/strings.rb, line 9 def string_encode_utf_8(string) string.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '') end
string_to_alphanum(string)
click to toggle source
# File lib/trials/utils/strings.rb, line 5 def string_to_alphanum(string) string.gsub(/[^A-Za-z0-9]/, '') end
timeit() { || ... }
click to toggle source
# File lib/trials/utils/benchmarking.rb, line 1 def timeit log(Benchmark.measure { yield }.to_s) end
write_csv_from_hashes(file, hash_set, attrs: nil)
click to toggle source
# File lib/trials/utils/csvs.rb, line 9 def write_csv_from_hashes(file, hash_set, attrs: nil) attrs ||= hash_set.to_harray.uniq_keys CSV.open(result_path(file), 'w') do |csv| csv << attrs hash_set.each do |c| csv << attrs.map { |a| c.send(:dig, a) } end end end
write_hashes_to_json(file, hashes)
click to toggle source
# File lib/trials/utils/jsons.rb, line 7 def write_hashes_to_json(file, hashes) write(file, hashes.to_json) end