class GoodData::Helpers::DataSource

Attributes

realized[R]

Public Class Methods

interpolate_sql_params(query, params) click to toggle source
# File lib/gooddata/helpers/data_helper.rb, line 17
def interpolate_sql_params(query, params)
  keys = query.scan(/\$\{([^\{]+)\}/).flatten
  keys.reduce(query) do |a, e|
    key = e
    raise "Param #{key} is not present in schedule params yet it is expected to be interpolated in the query" unless params.key?(key)
    a.gsub("${#{key}}", params[key])
  end
end
new(opts = {}) click to toggle source
# File lib/gooddata/helpers/data_helper.rb, line 27
def initialize(opts = {})
  opts = opts.is_a?(String) ? { type: :staging, path: opts } : opts
  opts = GoodData::Helpers.symbolize_keys(opts)
  @source = opts[:type]
  @options = opts
  @realized = false
end

Public Instance Methods

realize(params = {}) click to toggle source
# File lib/gooddata/helpers/data_helper.rb, line 35
def realize(params = {})
  @realized = true
  source = @source && @source.to_s
  case source
  when 'ads'
    realize_query(params)
  when 'staging'
    realize_staging(params)
  when 'web'
    realize_link
  when 's3'
    realize_s3(params)
  else
    raise "DataSource does not support type \"#{source}\""
  end
end
realized?() click to toggle source
# File lib/gooddata/helpers/data_helper.rb, line 52
def realized?
  @realized == true
end

Private Instance Methods

realize_query(params) click to toggle source
# File lib/gooddata/helpers/data_helper.rb, line 58
def realize_query(params)
  query = DataSource.interpolate_sql_params(@options[:query], params)
  dwh = params['ads_client'] || params[:ads_client] || raise("Data Source needs a client to ads to be able to query the storage but 'ads_client' is empty.")
  filename = Digest::SHA256.new.hexdigest(query)
  measure = Benchmark.measure do
    CSV.open(filename, 'w') do |csv|
      header_written = false
      header = nil
      dwh.execute_select(query) do |row|
        unless header_written
          header_written = true
          header = row.keys
          csv << header
        end
        csv << row.values_at(*header)
      end
    end
  end
  GoodData.logger.info "Realizing SQL query \"#{query}\" took #{measure.real}"
  filename
end
realize_s3(params) click to toggle source
# File lib/gooddata/helpers/data_helper.rb, line 104
def realize_s3(params)
  s3_client = params['aws_client'] && params['aws_client']['s3_client']
  raise 'AWS client not present. Perhaps S3Middleware is missing in the brick definition?' if !s3_client || !s3_client.respond_to?(:buckets)
  bucket_name = @options[:bucket]
  key = @options[:key]
  raise 'Key "bucket" is missing in S3 datasource' if bucket_name.blank?
  raise 'Key "key" is missing in S3 datasource' if key.blank?
  puts "Realizing download from S3. Bucket #{bucket_name}, object with key #{key}."
  filename = Digest::SHA256.new.hexdigest(@options.to_json)
  bucket = s3_client.buckets[bucket_name]
  obj = bucket.objects[key]
  File.open(filename, 'wb') do |file|
    obj.read do |chunk|
      file.write(chunk)
    end
  end
  puts 'Done downloading file.'
  filename
end
realize_staging(params) click to toggle source
# File lib/gooddata/helpers/data_helper.rb, line 80
def realize_staging(params)
  path = @options[:path]
  url = URI.parse(path)
  filename = Digest::SHA256.new.hexdigest(path)
  if url.relative?
    params['gdc_project'].download_file(path, filename)
  else
    params['GDC_GD_CLIENT'].download_file(path, filename)
  end
  filename
end