class Mvp::Forge
Public Class Methods
new(options = {})
click to toggle source
# File lib/mvp/forge.rb, line 8 def initialize(options = {}) @useragent = 'Puppet Community Stats Monitor' @forgeapi = options[:forgeapi] ||'https://forgeapi.puppet.com' end
Public Instance Methods
flatten_modules(data)
click to toggle source
# File lib/mvp/forge.rb, line 94 def flatten_modules(data) data.each do |row| row['owner'] = row['owner']['username'] row['superseded_by'] = row['superseded_by']['slug'] rescue nil row['pdk'] = row['current_release']['pdk'] row['supported'] = row['current_release']['supported'] row['version'] = row['current_release']['version'] row['validation_score'] = row['current_release']['validation_score'] row['license'] = row['current_release']['metadata']['license'] row['source'] = row['current_release']['metadata']['source'] row['project_page'] = row['current_release']['metadata']['project_page'] row['issues_url'] = row['current_release']['metadata']['issues_url'] row['tasks'] = row['current_release']['tasks'].map{|task| task['name']} rescue [] row['plans'] = row['current_release']['plans'].map{|task| task['name']} rescue [] row['release_count'] = row['releases'].count rescue 0 row['releases'] = row['releases'].map{|r| r['version']} rescue [] simplify_metadata(row, row['current_release']['metadata']) row.delete('current_release') end data end
flatten_releases(data)
click to toggle source
# File lib/mvp/forge.rb, line 118 def flatten_releases(data) data.each do |row| row['name'] = row['module']['name'] row['owner'] = row['module']['owner']['username'] row['license'] = row['metadata']['license'] row['source'] = row['metadata']['source'] row['project_page'] = row['metadata']['project_page'] row['issues_url'] = row['metadata']['issues_url'] row['tasks'] = row['tasks'].map{|task| task['name']} rescue [] row['plans'] = row['plans'].map{|task| task['name']} rescue [] simplify_metadata(row, row['metadata']) # These items are just too big to store in the table, and the malware scan isn't done yet ['module', 'changelog', 'readme', 'reference', 'malware_scan'].each do |column| row.delete(column) end end data end
flatten_validations(name, scores)
click to toggle source
# File lib/mvp/forge.rb, line 139 def flatten_validations(name, scores) row = { 'name' => name } scores.each do |entry| row[entry['name']] = entry['score'] end row end
munge_dates(object)
click to toggle source
transform dates into a format that bigquery knows
# File lib/mvp/forge.rb, line 82 def munge_dates(object) ["created_at", "updated_at", "deprecated_at", "deleted_at"].each do |field| next unless object.first.keys.include? field object.each do |record| next unless record[field] record[field] = DateTime.parse(record[field]).strftime("%Y-%m-%d %H:%M:%S") end end object end
retrieve(entity) { |results, offset| ... }
click to toggle source
# File lib/mvp/forge.rb, line 13 def retrieve(entity) raise 'Please process downloaded data by passing a block' unless block_given? # using authors for git repo terminology consistency entity = :users if entity == :authors begin offset = 0 endpoint = "/v3/#{entity}?sort_by=downloads&limit=50" while endpoint do response = HTTParty.get("#{@forgeapi}#{endpoint}", headers: {"User-Agent" => @useragent}) raise "Forge Error: #{@response.body}" unless response.code == 200 data = JSON.parse(response.body) results = munge_dates(data['results']) case entity when :modules results = flatten_modules(results) when :releases results = flatten_releases(results) end yield results, offset offset += 50 endpoint = data['pagination']['next'] if (endpoint and (offset % 250 == 0)) GC.start end end rescue => e $logger.error e.message $logger.debug e.backtrace.join("\n") end nil end
retrieve_validations(modules, period = 25) { |results, offset| ... }
click to toggle source
# File lib/mvp/forge.rb, line 52 def retrieve_validations(modules, period = 25) raise 'Please process validations by passing a block' unless block_given? offset = 0 begin modules.each_slice(period) do |group| offset += period results = group.map { |mod| validations(mod[:slug]) } yield results, offset GC.start end rescue => e $logger.error e.message $logger.debug e.backtrace.join("\n") end nil end
simplify_metadata(data, metadata)
click to toggle source
# File lib/mvp/forge.rb, line 147 def simplify_metadata(data, metadata) data['operatingsystem'] = metadata['operatingsystem_support'].map{|i| i['operatingsystem']} rescue [] data['dependencies'] = metadata['dependencies'].map{|i| i['name'].sub('/', '-')} rescue [] data['puppet_range'] = metadata['requirements'].select{|r| r['name'] == 'puppet'}.first['version_requirement'] rescue nil data['metadata'] = metadata.to_json if data['puppet_range'] range = SemanticPuppet::VersionRange.parse(data['puppet_range']) data['puppet_2x'] = range.include? SemanticPuppet::Version.parse('2.99.99') data['puppet_3x'] = range.include? SemanticPuppet::Version.parse('3.99.99') data['puppet_4x'] = range.include? SemanticPuppet::Version.parse('4.99.99') data['puppet_5x'] = range.include? SemanticPuppet::Version.parse('5.99.99') data['puppet_6x'] = range.include? SemanticPuppet::Version.parse('6.99.99') data['puppet_99x'] = range.include? SemanticPuppet::Version.parse('99.99.99') # identify unbounded ranges end end
test()
click to toggle source
# File lib/mvp/forge.rb, line 164 def test() require 'pry' binding.pry end
validations(name)
click to toggle source
# File lib/mvp/forge.rb, line 72 def validations(name) endpoint = "/private/validations/" response = HTTParty.get("#{@forgeapi}#{endpoint}#{name}", headers: {'User-Agent' => @useragent}) raise "Forge Error: #{@response.body}" unless response.code == 200 flatten_validations(name, JSON.parse(response.body)) end