class Datapimp::Sources::Base

Attributes

required_options[RW]
format[RW]
name[R]
options[R]
path[RW]
processed[RW]
raw[RW]
refreshed_at[RW]
scopes[RW]
slug_column[RW]

Public Class Methods

new(name, options={}) click to toggle source
# File lib/datapimp/sources.rb, line 19
def initialize(name, options={})
  @name     ||= name
  @options  ||= options
  @format   ||= options.fetch(:format, :json)
  @path     ||= options.fetch(:path) { Pathname(Dir.pwd()) }

  @slug_column = options.fetch(:slug_column, :_id)

  ensure_valid_options!
end
requires(*args) click to toggle source
# File lib/datapimp/sources.rb, line 15
def self.requires *args
  self.required_options = args
end

Public Instance Methods

age() click to toggle source

how long since this data source has been refreshed?

# File lib/datapimp/sources.rb, line 129
def age
  Time.now.to_i - refreshed_at.to_i
end
compute_properties() click to toggle source

compute properties takes the raw data of each record and sets additional properties on the records which may not be persited in the data source

# File lib/datapimp/sources.rb, line 56
def compute_properties
  self.processed && self.processed.map! do |row|
    if slug_column && row.respond_to?(slug_column)
      row.slug = row.send(slug_column).to_s.parameterize
    end

    row
  end

  processors.each do |processor|
    original = self.processed.dup
    modified = []

    original.each_with_index do |record, index|
      previous = original[index - 1]
      modified.push(processor.call(record, index, previous: previous, set: original))
    end

    self.processed = modified
  end
end
data() click to toggle source
# File lib/datapimp/sources.rb, line 133
def data
  refresh if need_to_refresh?
  processed
end
ensure_valid_options!() click to toggle source

makes sure that the required options for this data source are passed for any instance of the data source

# File lib/datapimp/sources.rb, line 86
def ensure_valid_options!
  missing_options = (Array(self.class.required_options) - options.keys.map(&:to_sym))

  missing_options.reject! do |key|
    respond_to?(key) && !send(key).nil?
  end

  if missing_options.length > 0
    raise 'Error: failure to supply the following options: ' + missing_options.map(&:to_s).join(",")
  end
end
fetch() click to toggle source
# File lib/datapimp/sources.rb, line 147
def fetch
  @fetched = true
  self.raw = []
end
file() click to toggle source
# File lib/datapimp/sources.rb, line 184
def file
  @file ||= name.parameterize if name.respond_to?(:parameterize)
  @file.gsub!("-","_")
  @file = "#{@file}.json" unless @file.match(/\.json/i)
  @file
end
fresh_on_server?() click to toggle source
# File lib/datapimp/sources.rb, line 119
def fresh_on_server?
  need_to_refresh?
end
has_scope?(scope_name) click to toggle source
# File lib/datapimp/sources.rb, line 49
def has_scope?(scope_name)
  scope_name && (self.scopes ||= {}).key?(scope_name.to_sym)
end
jsonify(value) click to toggle source
# File lib/datapimp/sources.rb, line 195
def jsonify(value)
  case value
  when String, Numeric, NilClass, TrueClass, FalseClass
    value
  when Hash
    Hash[value.map { |k, v| [jsonify(k), jsonify(v)] }]
  when Array
    value.map { |v| jsonify(v) }
  when HappyMapper
    value.instance_variables.each_with_object({}) do |var_name, memo|
      key       = var_name.to_s.sub(/^@/, '').to_sym
      val       = value.instance_variable_get(var_name)
      memo[key] = jsonify(val)
    end
  else
    if value.respond_to?(:to_attrs)
      value.to_attrs
    elsif value.respond_to?(:as_json)
      value.as_json
    else
      value.to_s
    end
  end
end
max_age() click to toggle source
# File lib/datapimp/sources.rb, line 123
def max_age
  max = ENV['MAX_DATA_SOURCE_AGE']
  (max && max.to_i) || 120
end
need_to_refresh?() click to toggle source
# File lib/datapimp/sources.rb, line 143
def need_to_refresh?
  !(@fetched && @_processed)
end
path_to_file() click to toggle source
# File lib/datapimp/sources.rb, line 191
def path_to_file
  Pathname(path).join("#{ file }")
end
persisted?() click to toggle source
# File lib/datapimp/sources.rb, line 180
def persisted?
  path_to_file && path_to_file.exist?
end
preprocess() click to toggle source
# File lib/datapimp/sources.rb, line 152
def preprocess
  self.raw.dup
end
process() click to toggle source
# File lib/datapimp/sources.rb, line 156
def process
  @_processed = true
  self.processed = preprocess
  # set_id
  compute_properties
  self.processed
end
processors(&block) click to toggle source
# File lib/datapimp/sources.rb, line 78
def processors &block
  @processors ||= []
  @processors << block if block_given?
  @processors
end
refresh() click to toggle source
# File lib/datapimp/sources.rb, line 102
def refresh
  fetch
  process
  self.refreshed_at = Time.now.to_i
  self
end
refresh!() click to toggle source
# File lib/datapimp/sources.rb, line 138
def refresh!
  refresh
  save_to_disk
end
refresh_if_stale?() click to toggle source
# File lib/datapimp/sources.rb, line 109
def refresh_if_stale?
  refresh! if stale?
end
save_to_disk() click to toggle source
# File lib/datapimp/sources.rb, line 172
def save_to_disk
  unless path_to_file.dirname.exist?
    FileUtils.mkdir(path_to_file.dirname)
  end

  path_to_file.open('w+') {|fh| fh.write(to_s) }
end
scope(*args, block) click to toggle source

defines a scope for the records in this data source a scope is a named filter, implemented in the form of a block which is passed each record. if the block returns true, it returns the record:

Example:

data_source(:galleries) do

scope :active, -> {|record| record.state == "active" }

end

# File lib/datapimp/sources.rb, line 44
def scope(*args, block)
  name = args.first
  (self.scopes ||= {})[name.to_sym] = block
end
select(&block) click to toggle source
# File lib/datapimp/sources.rb, line 98
def select(&block)
  data.select(&block)
end
stale?() click to toggle source

A data source is stale if it has been populated and the age is greater than the max age we allow.

# File lib/datapimp/sources.rb, line 115
def stale?
  !need_to_refresh? && (age > max_age)
end
to_s() click to toggle source
# File lib/datapimp/sources.rb, line 30
def to_s
  data.to_json
end