class GoodDataMarketo::RESTAdapter

Attributes

token[RW]
token_timer[RW]

Public Class Methods

new(config = {}) click to toggle source
# File lib/gooddata_marketo/adapters/rest.rb, line 12
def initialize config = {}

  @api_limit = MARKETO_API_LIMIT
  @lead_list_dump = config[:file] || LEAD_LIST_DUMP_CSV || 'marketo_leads_dump.csv'
  @marketo_subdomain = config[:subdomain] || MARKETO_SUBDOMAIN
  @marketo_rest_secret = config[:secret] || config[:user] || MARKETO_REST_SECRET
  @marketo_rest_id = config[:id] || config[:user] || MARKETO_REST_ID
  @marketo_domain = "https://#{@marketo_subdomain}.mktorest.com"
  @webdav = config[:webdav]
  @lead_list_ids = []
  @token = self.get_token unless config[:token]
  @token_uri = "?access_token=#{@token}"

  # Example endpoint = "/rest/v1/lists.json"

end

Public Instance Methods

clean() click to toggle source
# File lib/gooddata_marketo/adapters/rest.rb, line 232
def clean

  lli = 'lead_list_ids.json'
  llnpt = 'lead_list_next_page_token.json'

  # Remove list ids from webdav and local
  if @webdav.exists?(lli)
    @webdav.delete(lli)
  end

  if @webdav.exists?(llnpt)
    @webdav.delete(llnpt)
  end

  File.delete(lli) if File.exists? lli
  File.delete(llnpt) if File.exists? llnpt

  # Upload marketo_dump to webdav for backup.
  @webdav.upload(@lead_list_dump)

end
get(url) click to toggle source
# File lib/gooddata_marketo/adapters/rest.rb, line 39
def get url
  @api_limit -= 1
  begin
    puts "#{Time.now} => REST:GET (#{url})" if GoodDataMarketo.logging
    response = RestClient.get url
  rescue Exception => exc
    puts exc if GoodDataMarketo.logging
    puts "#{Time.now} => Possible API Limit reached, last request was  ##{@api_limit} of #{MARKETO_API_LIMIT}"
  end
end
get_all_lead_emails(config = {}) click to toggle source
# File lib/gooddata_marketo/adapters/rest.rb, line 186
def get_all_lead_emails config = {}
  config[:fields] = 'email'
  self.get_all_leads config
end
get_all_lead_ids(config = {}) click to toggle source
# File lib/gooddata_marketo/adapters/rest.rb, line 181
def get_all_lead_ids config = {}
  config[:fields] = 'id'
  self.get_all_leads config
end
get_all_leads(config = {}) click to toggle source
# File lib/gooddata_marketo/adapters/rest.rb, line 90
def get_all_leads config = {}

  self.refresh_token?

  # Check if we already have a leads list, if so, download it.
  self.load_lists

  # Check if there was a stream broken while downloading a specific list.
  self.load_last_page

  # Check if there was a partial ID dump available on WebDAV, if so download it..
  self.load_id_dump

  @fields = config[:fields] || 'id'

  # For each of the lead list ids, download their respective leads by id.
  @csv = CSV.open(@lead_list_dump,'a+')

  lists = @lead_list_ids

  lists.each do |list|

    id = @lead_list_ids.shift

    domain = @marketo_domain
    parameters = "&fields=#{@fields}"
    endpoint= "/rest/v1/list/#{id}/leads.json"
    url = domain + endpoint + @token_uri + parameters

    # Conditional recursive function ends if nextPageToken is not available in response. Writes results to CSV.
    def rest_lead_stream url, list_id

      response = self.get url
      json = JSON.parse(response.body, :symbolize_names => true)
      results = json[:result]

      if results
        results.each { |result| @csv << [result[:id]] }
        @csv.flush
        puts "#{Time.now} => REST:leadList:#{list_id}:Results:#{results.length}" if GoodDataMarketo.logging
      end

      next_page_token = json[:nextPageToken]

      # If there is another page, remember it and then attempt the next load.
      if next_page_token

        self.remember_next_page :token => token, :list => list_id
        domain = @marketo_domain
        parameters = "&fields=#{@fields}"
        endpoint= "/rest/v1/list/#{list_id}/leads.json"
        url = domain + endpoint + @token_uri + parameters + "&nextPageToken=" + next_page_token
        rest_lead_stream url, list_id

      else

        # Update the local and remote lead lists
        File.open('lead_list_ids.json','w'){ |f| JSON.dump(@lead_list_ids, f) }
        @webdav.upload('lead_list_ids.json')

      end

    end

    # While on this list, check to see if it failed working on it before and resume where it left off.
    if @next_page_token_list == list

      puts "#{Time.now} => REST(Resumed):leadList:#{list}:NextPageToken:#{@next_page_token}" if GoodDataMarketo.logging
      domain = @marketo_domain
      parameters = "" #"&fields=#{fields}"
      endpoint= "/rest/v1/list/#{list}/leads.json"
      url = domain + endpoint + @token_uri + parameters + "&nextPageToken=" + @next_page_token

      rest_lead_stream url, list

    else

      rest_lead_stream url, list

    end

  end

  # Remove lists from WebDav & Local, upload Marketo Ids Dump to webdav.
  self.clean

  # Update the etl controller that an update as occured.
  self.resolve_with_etl_controller

end
get_lead_lists(next_page_token = nil) click to toggle source
# File lib/gooddata_marketo/adapters/rest.rb, line 50
def get_lead_lists next_page_token = nil

  endpoint = "/rest/v1/lists.json"
  url = @marketo_domain + endpoint + @token_uri

  if next_page_token
    endpoint = "/rest/v1/lists.json"
    url = @marketo_domain + endpoint + @token_uri + "&nextPageToken="+next_page_token
  end

  response = self.get url

  json = JSON.parse(response.body, :symbolize_names => true)

  json[:result].each { |m|  @lead_list_ids << m[:id] }

  next_page_token = json[:nextPageToken]

  if next_page_token
    self.get_lead_lists next_page_token
  else
    @lead_list_ids
  end

end
get_multiple_leads(config = {}) click to toggle source
# File lib/gooddata_marketo/adapters/rest.rb, line 76
def get_multiple_leads config = {}

  self.refresh_token?

  ids = config[:ids] || config[:values] || [28567,30885,32240,37161,40832]
  domain = @marketo_domain
  parameters = "&filterType=id"+"&filterValues="+ids.join(',')
  endpoint= "/rest/v1/leads.json"
  url = domain + endpoint + @token_uri + parameters

  self.get url

end
get_token() click to toggle source
# File lib/gooddata_marketo/adapters/rest.rb, line 254
def get_token


  domain = @marketo_domain
  endpoint = "/identity/oauth/token?grant_type=client_credentials"
  url = domain + endpoint + "&client_id=" + @marketo_rest_id + "&client_secret=" + @marketo_rest_secret
  response = self.get url
  results = JSON.parse(response.body)
  access_token = results["access_token"]
  @token_timer = Time.now
  @token = access_token
  access_token

end
load_id_dump() click to toggle source
# File lib/gooddata_marketo/adapters/rest.rb, line 207
def load_id_dump
  # Check if the lists leads have already been dumped. Append to it.
  if @webdav.exists? @lead_list_dump
    file = @webdav.download @lead_list_dump
    puts "#{Time.now} => WEBDAV:ID_DUMP:Load" if GoodDataMarketo.logging
    f = File.open(@lead_list_dump,'w')
    f.write(file)
  end
end
load_last_page() click to toggle source
# File lib/gooddata_marketo/adapters/rest.rb, line 191
def load_last_page

  if @webdav.include? 'lead_list_next_page_token.json'
    next_page_token_json = @webdav.download 'lead_list_next_page_token.json'
    json = JSON.parse(next_page_token_json)
  elsif File.exists? 'lead_list_next_page_token.json'
    json = JSON.parse( IO.read('lead_list_next_page_token.json'))
  else
    json = {}
  end

  @next_page_token = json[:token]
  @next_page_token_list = json[:list]

end
load_lists() click to toggle source
# File lib/gooddata_marketo/adapters/rest.rb, line 217
def load_lists
  if @webdav.include? 'lead_list_ids.json'
    lists = @webdav.download 'lead_list_ids.json'
    @lead_list_ids = JSON.parse(lists)
    puts "#{Time.now} => WEBDAV:ListsLoaded:#{@lead_list_ids.length}:lead_list_ids.json" if GoodDataMarketo.logging
  else
    self.get_lead_lists
  end
end
refresh_token?() click to toggle source

If token has existed for more than 6000 seconds, refresh it.

# File lib/gooddata_marketo/adapters/rest.rb, line 30
def refresh_token?
  if (@token_timer + 6000) < Time.now
    self.get_token
    true
  else
    false
  end
end
remember_next_page(config = {}) click to toggle source
# File lib/gooddata_marketo/adapters/rest.rb, line 227
def remember_next_page config = {}
  File.open('lead_list_next_page_token.json','w'){ |f| JSON.dump(config, f) }
  @webdav.upload('lead_list_next_page_token.json')
end
resolve_with_etl_controller() click to toggle source
# File lib/gooddata_marketo/adapters/rest.rb, line 281
def resolve_with_etl_controller

  puts 'Complete'
  # Here it would update the log that a full upload as occured.
end
rest_lead_stream(url, list_id) click to toggle source

Conditional recursive function ends if nextPageToken is not available in response. Writes results to CSV.

# File lib/gooddata_marketo/adapters/rest.rb, line 120
def rest_lead_stream url, list_id

  response = self.get url
  json = JSON.parse(response.body, :symbolize_names => true)
  results = json[:result]

  if results
    results.each { |result| @csv << [result[:id]] }
    @csv.flush
    puts "#{Time.now} => REST:leadList:#{list_id}:Results:#{results.length}" if GoodDataMarketo.logging
  end

  next_page_token = json[:nextPageToken]

  # If there is another page, remember it and then attempt the next load.
  if next_page_token

    self.remember_next_page :token => token, :list => list_id
    domain = @marketo_domain
    parameters = "&fields=#{@fields}"
    endpoint= "/rest/v1/list/#{list_id}/leads.json"
    url = domain + endpoint + @token_uri + parameters + "&nextPageToken=" + next_page_token
    rest_lead_stream url, list_id

  else

    # Update the local and remote lead lists
    File.open('lead_list_ids.json','w'){ |f| JSON.dump(@lead_list_ids, f) }
    @webdav.upload('lead_list_ids.json')

  end

end
usage() click to toggle source
# File lib/gooddata_marketo/adapters/rest.rb, line 269
def usage
  domain = @marketo_domain
  endpoint = "/rest/v1/stats/usage.json"
  url = domain + endpoint + @token_uri

  # Note this is the only rest call which does not use the log so as not to print the keys to log.
  response = RestClient.get url

  #Parse reponse and return only access token
  results = JSON.parse(response.body)['result']
end