class SdbS3Env::GlueEnv

Constants

NamespaceKey
VersionKey

used to identify metadata for models (should be consistent across models) PersistLayerKey not needed, node key can be used as persistent layer key see mysql_glue_env to decouple persistent layer key from node key

Attributes

_files_mgr_class[RW]

TODO: Rather than using File class directly, should a special class be used? <- still applicable?

metadata_keys[RW]

TODO: Rather than using File class directly, should a special class be used? <- still applicable?

moab_data[RW]

TODO: Rather than using File class directly, should a special class be used? <- still applicable?

model_key[RW]

TODO: Rather than using File class directly, should a special class be used? <- still applicable?

model_save_params[RW]

TODO: Rather than using File class directly, should a special class be used? <- still applicable?

namespace_key[RW]

TODO: Rather than using File class directly, should a special class be used? <- still applicable?

node_key[RW]

TODO: Rather than using File class directly, should a special class be used? <- still applicable?

persist_layer_key[RW]

TODO: Rather than using File class directly, should a special class be used? <- still applicable?

required_instance_keys[RW]

TODO: Rather than using File class directly, should a special class be used? <- still applicable?

required_save_keys[RW]

TODO: Rather than using File class directly, should a special class be used? <- still applicable?

user_datastore_location[RW]

TODO: Rather than using File class directly, should a special class be used? <- still applicable?

user_id[RW]

TODO: Rather than using File class directly, should a special class be used? <- still applicable?

version_key[RW]

TODO: Rather than using File class directly, should a special class be used? <- still applicable?

views[RW]

TODO: Rather than using File class directly, should a special class be used? <- still applicable?

Public Class Methods

new(persist_env, data_model_bindings) click to toggle source

accessors specific to this persitence model

# File lib/glue_envs/sdb_s3_glue_env.rb, line 45
def initialize(persist_env, data_model_bindings)
  #TODO: determine if class_name is needed to segment cluster data within user data
  #host = "https://sdb.amazonaws.com/"  (not provided by user)
 
  #user_id = env[:user_id]
  sdb_s3_env = persist_env[:env]
  #TODO: validations on format
  domain_base_name = sdb_s3_env[:path]
  @user_id = sdb_s3_env[:user_id]
  @cluster_name = persist_env[:name]
  
  #data_model_bindings from NodeElementOperations
  key_fields = data_model_bindings[:key_fields] 
  initial_views_data = data_model_bindings[:views]
  
  @required_instance_keys = key_fields[:required_keys] #DataStructureModels::Tinkit::RequiredInstanceKeys
  @required_save_keys = key_fields[:required_keys] #DataStructureModels::Tinkit::RequiredSaveKeys
  @node_key = key_fields[:primary_key] #DataStructureModels::Tinkit::NodeKey
  @persist_layer_key = @node_key
  #@moab_datastore_name = MoabDatastoreName
  @version_key = VersionKey  
  @model_key = @node_key #ModelKey
  @namespace_key = NamespaceKey
  @metadata_keys = [@version_key, @namespace_key] 
  aak = ENV["AMAZON_ACCESS_KEY_ID"]
  asak = ENV["AMAZON_SECRET_ACCESS_KEY"]
  #rightaws_log = TinkitLog.set("RightAWS::SDBInterface", :warn)
  #sdb = RightAws::SdbInterface.new(aak, asak, :logger => rightaws_log, :multi_thread => true)
  sdb = AwsSdb::Service.new  #aws-sdb
  @user_datastore_location = use_domain!(sdb, "#{domain_base_name}__#{@user_id}") 
  @model_save_params = {:sdb => sdb, :domain => user_datastore_location, :node_key => @node_key}
  @_files_mgr_class = SdbS3Interface::FilesMgr
  @views = "temp"
  @moab_data = {}
  #@views_mgr = ViewsMgr.new({:data_file => @data_file_name})
  #@record_locker = {}  #tracks records that are in the process of being saved
end

Public Instance Methods

destroy_bulk(list_of_native_records) click to toggle source
# File lib/glue_envs/sdb_s3_glue_env.rb, line 189
def destroy_bulk(list_of_native_records)
  sdb = @model_save_params[:sdb]
  domain = @model_save_params[:domain]
  node_key = @model_save_params[:node_key]
  list_of_native_records.each do |rcd|
    item_name = rcd[node_key]
    #TODO: use the batch delete request
    sdb.delete_attributes(domain, item_name)
  end
end
destroy_node(model_metadata) click to toggle source
# File lib/glue_envs/sdb_s3_glue_env.rb, line 168
def destroy_node(model_metadata)
  sdb = @model_save_params[:sdb]
  domain = @model_save_params[:domain]
  #node_key = @model_save_params[:node_key]
  item_name = model_metadata[@model_key]
  @@log.info {"Deleting node: #{model_metadata.inspect} with key #{item_name} from domain: #{domain}"} if @@log.info?
  sdb.delete_attributes(domain, item_name)
end
find_contains(key, this_value) click to toggle source
# File lib/glue_envs/sdb_s3_glue_env.rb, line 119
def find_contains(key, this_value) 
  #there is probably an optimized way to do this natively
  #in sdb's pseodo sql, but I can't figure it out
  results =[]
  query_all.each do |record|
    test_val = record[key]
    results << record  if find_contains_type_helper(test_val, this_value)
  end
  results  
end
find_contains_type_helper(stored_data, this_value) click to toggle source
# File lib/glue_envs/sdb_s3_glue_env.rb, line 130
def find_contains_type_helper(stored_data, this_value)
  #p stored_dataj
  resp = nil
  #stored_data = jparse(stored_dataj)
  if stored_data.respond_to?(:"include?")
    resp = (stored_data.include?(this_value))
  else
    resp = (stored_data == this_value)
  end
  return resp
end
find_equals(key, this_value) click to toggle source
# File lib/glue_envs/sdb_s3_glue_env.rb, line 110
def find_equals(key, this_value) 
  results =[]
  query_all.each do |record|
    test_val = record[key]
    results << record  if test_val == this_value
  end
  results  
end
find_nodes_where(key, relation, this_value) click to toggle source

current relations supported:

- :equals (data in the key field matches this_value)
- :contains (this_value is contained in the key field data (same as equals for non-enumerable types )
# File lib/glue_envs/sdb_s3_glue_env.rb, line 100
def find_nodes_where(key, relation, this_value)
  res = case relation
    when :equals
      find_equals(key, this_value)
    when :contains
      find_contains(key, this_value)
  end #case
  return res    
end
generate_model_key(namespace, node_key) click to toggle source

namespace is used to distinguish between unique data sets (i.e., users) within the model, for sdb, each user has their own domain, so the key only has to be unique within the domain

# File lib/glue_envs/sdb_s3_glue_env.rb, line 180
def generate_model_key(namespace, node_key)
  "#{node_key}"
  #"#{namespace}::#{node_key}"
end
get(id) click to toggle source
# File lib/glue_envs/sdb_s3_glue_env.rb, line 142
def get(id)
  sdb = @model_save_params[:sdb]
  domain = @model_save_params[:domain]
  raw_data = sdb.get_attributes(domain, id)
  #puts "Raw Data: #{raw_data.inspect}"
  data = from_sdb(raw_data)
  data = nil if data.empty?
  return data
end
query_all() click to toggle source
# File lib/glue_envs/sdb_s3_glue_env.rb, line 83
def query_all  #TODO move to ViewsMgr
  sdb = @model_save_params[:sdb]
  domain = @model_save_params[:domain]
  query = "select * from `#{domain}`"
  raw_data = sdb.select(query).first
  data = {}
  #puts "QA Raw: #{raw_data.inspect}"
  raw_data.each do |k,v|
    data[k] = from_sdb(v)
  end
  @@log.info{"Query All data: #{data.values}"} if @@log.info?
  data.values
end
raw_all() click to toggle source
# File lib/glue_envs/sdb_s3_glue_env.rb, line 185
def raw_all
  query_all
end
save(new_data) click to toggle source
# File lib/glue_envs/sdb_s3_glue_env.rb, line 152
def save(new_data)
  sdb = @model_save_params[:sdb]
  domain = @model_save_params[:domain]
  #although we could pull @node_key directly, I do it this way to make it clear
  #that it's a parameter used in saving to the persistence model
  #I should try to be consistent on this
  node_key = @model_save_params[:node_key]
  rev_data = new_data.dup
  rev_data[@version_key] = Time.now.hash
  raw_model_data = HashKeys.sym_to_str(rev_data)
  model_data = to_sdb(raw_model_data)
  sdb.put_attributes(domain, new_data[node_key], model_data)
  raw_model_data['rev'] = raw_model_data[@version_key]
  return raw_model_data
end

Private Instance Methods

from_sdb(sdb_data) click to toggle source
# File lib/glue_envs/sdb_s3_glue_env.rb, line 222
def from_sdb(sdb_data)
  rtn_data = {}
  sdb_data.each do |k_s, v_json|
    k = k_s.to_sym
    rtn_data[k] = jparse(v_json.first)
  end
  rtn_data
end
jparse(str) click to toggle source
# File lib/glue_envs/sdb_s3_glue_env.rb, line 241
def jparse(str)
  return JSON.parse(str) if str =~ /\A\s*[{\[]/
  JSON.parse("[#{str}]")[0]
  #JSON.parse(str)
end
parse_sdb_domains(raw_list_results) click to toggle source
# File lib/glue_envs/sdb_s3_glue_env.rb, line 212
def parse_sdb_domains(raw_list_results)
  if raw_list_results.last == ""
  #if raw_list_results[:next_token].nil? #right-aws
    #return raw_list_results[:domains] #right-aws
    return raw_list_results.first  #aws-sdb
  else
    raise "Have not implemented large list handling yet"
  end
end
to_sdb(data) click to toggle source
# File lib/glue_envs/sdb_s3_glue_env.rb, line 231
def to_sdb(data)
  formatted_data = {}
  data.each do |k,v|
    k_f = k.to_s
    v_f = v.to_json
    formatted_data[k_f] = v_f
  end
  formatted_data
end
use_domain!(sdb, domain_name) click to toggle source
# File lib/glue_envs/sdb_s3_glue_env.rb, line 202
def use_domain!(sdb, domain_name)
  all_domains = parse_sdb_domains(sdb.list_domains)
  if all_domains.include?(domain_name)
    return domain_name
  else #no domain by that name exists yet
    sdb.create_domain(domain_name)
    return domain_name
  end
end