class Datapimp::Sources::Base
Attributes
required_options[RW]
format[RW]
name[R]
options[R]
path[RW]
processed[RW]
raw[RW]
refreshed_at[RW]
scopes[RW]
slug_column[RW]
Public Class Methods
new(name, options={})
click to toggle source
# File lib/datapimp/sources.rb, line 19 def initialize(name, options={}) @name ||= name @options ||= options @format ||= options.fetch(:format, :json) @path ||= options.fetch(:path) { Pathname(Dir.pwd()) } @slug_column = options.fetch(:slug_column, :_id) ensure_valid_options! end
requires(*args)
click to toggle source
# File lib/datapimp/sources.rb, line 15 def self.requires *args self.required_options = args end
Public Instance Methods
age()
click to toggle source
how long since this data source has been refreshed?
# File lib/datapimp/sources.rb, line 129 def age Time.now.to_i - refreshed_at.to_i end
compute_properties()
click to toggle source
compute properties takes the raw data of each record and sets additional properties on the records which may not be persited in the data source
# File lib/datapimp/sources.rb, line 56 def compute_properties self.processed && self.processed.map! do |row| if slug_column && row.respond_to?(slug_column) row.slug = row.send(slug_column).to_s.parameterize end row end processors.each do |processor| original = self.processed.dup modified = [] original.each_with_index do |record, index| previous = original[index - 1] modified.push(processor.call(record, index, previous: previous, set: original)) end self.processed = modified end end
data()
click to toggle source
# File lib/datapimp/sources.rb, line 133 def data refresh if need_to_refresh? processed end
ensure_valid_options!()
click to toggle source
makes sure that the required options for this data source are passed for any instance of the data source
# File lib/datapimp/sources.rb, line 86 def ensure_valid_options! missing_options = (Array(self.class.required_options) - options.keys.map(&:to_sym)) missing_options.reject! do |key| respond_to?(key) && !send(key).nil? end if missing_options.length > 0 raise 'Error: failure to supply the following options: ' + missing_options.map(&:to_s).join(",") end end
fetch()
click to toggle source
# File lib/datapimp/sources.rb, line 147 def fetch @fetched = true self.raw = [] end
file()
click to toggle source
# File lib/datapimp/sources.rb, line 184 def file @file ||= name.parameterize if name.respond_to?(:parameterize) @file.gsub!("-","_") @file = "#{@file}.json" unless @file.match(/\.json/i) @file end
fresh_on_server?()
click to toggle source
# File lib/datapimp/sources.rb, line 119 def fresh_on_server? need_to_refresh? end
has_scope?(scope_name)
click to toggle source
# File lib/datapimp/sources.rb, line 49 def has_scope?(scope_name) scope_name && (self.scopes ||= {}).key?(scope_name.to_sym) end
jsonify(value)
click to toggle source
# File lib/datapimp/sources.rb, line 195 def jsonify(value) case value when String, Numeric, NilClass, TrueClass, FalseClass value when Hash Hash[value.map { |k, v| [jsonify(k), jsonify(v)] }] when Array value.map { |v| jsonify(v) } when HappyMapper value.instance_variables.each_with_object({}) do |var_name, memo| key = var_name.to_s.sub(/^@/, '').to_sym val = value.instance_variable_get(var_name) memo[key] = jsonify(val) end else if value.respond_to?(:to_attrs) value.to_attrs elsif value.respond_to?(:as_json) value.as_json else value.to_s end end end
max_age()
click to toggle source
# File lib/datapimp/sources.rb, line 123 def max_age max = ENV['MAX_DATA_SOURCE_AGE'] (max && max.to_i) || 120 end
need_to_refresh?()
click to toggle source
# File lib/datapimp/sources.rb, line 143 def need_to_refresh? !(@fetched && @_processed) end
path_to_file()
click to toggle source
# File lib/datapimp/sources.rb, line 191 def path_to_file Pathname(path).join("#{ file }") end
persisted?()
click to toggle source
# File lib/datapimp/sources.rb, line 180 def persisted? path_to_file && path_to_file.exist? end
preprocess()
click to toggle source
# File lib/datapimp/sources.rb, line 152 def preprocess self.raw.dup end
process()
click to toggle source
# File lib/datapimp/sources.rb, line 156 def process @_processed = true self.processed = preprocess # set_id compute_properties self.processed end
processors(&block)
click to toggle source
# File lib/datapimp/sources.rb, line 78 def processors &block @processors ||= [] @processors << block if block_given? @processors end
refresh()
click to toggle source
# File lib/datapimp/sources.rb, line 102 def refresh fetch process self.refreshed_at = Time.now.to_i self end
refresh!()
click to toggle source
# File lib/datapimp/sources.rb, line 138 def refresh! refresh save_to_disk end
refresh_if_stale?()
click to toggle source
# File lib/datapimp/sources.rb, line 109 def refresh_if_stale? refresh! if stale? end
save_to_disk()
click to toggle source
# File lib/datapimp/sources.rb, line 172 def save_to_disk unless path_to_file.dirname.exist? FileUtils.mkdir(path_to_file.dirname) end path_to_file.open('w+') {|fh| fh.write(to_s) } end
scope(*args, block)
click to toggle source
defines a scope for the records in this data source a scope is a named filter, implemented in the form of a block which is passed each record. if the block returns true, it returns the record:
Example:
data_source(:galleries) do
scope :active, -> {|record| record.state == "active" }
end
# File lib/datapimp/sources.rb, line 44 def scope(*args, block) name = args.first (self.scopes ||= {})[name.to_sym] = block end
select(&block)
click to toggle source
# File lib/datapimp/sources.rb, line 98 def select(&block) data.select(&block) end
stale?()
click to toggle source
A data source is stale if it has been populated and the age is greater than the max age we allow.
# File lib/datapimp/sources.rb, line 115 def stale? !need_to_refresh? && (age > max_age) end
to_s()
click to toggle source
# File lib/datapimp/sources.rb, line 30 def to_s data.to_json end