class Middleman::Cli::CrawlerIndex
This class provides a “crawler” command for the middleman CLI.
Public Class Methods
exit_on_failure?()
click to toggle source
Tell Thor to exit with a nonzero exit code on failure
# File lib/middleman-crawler/commands.rb, line 85 def self.exit_on_failure? true end
Public Instance Methods
crawler()
click to toggle source
# File lib/middleman-crawler/commands.rb, line 89 def crawler configs = get_configs env_config = File.join(Middleman::Application.root, 'environments', "#{configs[:environment]}.rb") return unless File.exist? env_config configurations = Configure.new configurations.instance_eval File.read(env_config), env_config, 1 response = {} raw = Rawler::Base.new(configurations.config[:base_url], $stdout, rawler_options) raw.validate response['errors'] = raw.responses.reject{ |link, response| (100..399).include?(response[:status].to_i) } say("There are some errors: #{errors_message_for(response['errors'])}", :red) if response['errors'] puts "All crawling actions ok" unless response['errors'] end
Private Instance Methods
errors_message_for(responses)
click to toggle source
# File lib/middleman-crawler/commands.rb, line 146 def errors_message_for(responses) collection = responses.each_with_object([]) do |(url, response), a| a << "url: #{url}, error_code: #{response[:status]}" end collection.join("; ") end
get_configs()
click to toggle source
# File lib/middleman-crawler/commands.rb, line 111 def get_configs configs ||= { environment: (options['environment'].presence || "development").to_sym, username: options['username'], password: options['password'], skip: options['skip'], wait: options['wait'], css: options['css'], local: options['local'], log: options['log'], logfile: options['logfile'], iskip: options['iskip'], include: options['include'], iinclude: options['iinclude'], ignore_fragments: options['ignore_fragments'] } end
rawler_options()
click to toggle source
# File lib/middleman-crawler/commands.rb, line 129 def rawler_options opts = {} opts[:username] = get_configs[:username].to_s if get_configs[:username].present? opts[:password] = get_configs[:password].to_s if get_configs[:password].present? opts[:skip] = get_configs[:skip].present? ? get_configs[:skip].to_s : "\#$" opts[:wait] = get_configs[:wait].present? ? get_configs[:wait].to_f : 0.1 opts[:css] = get_configs[:css] != nil ? get_configs[:css] : true opts[:local] = get_configs[:local] != nil ? get_configs[:local] : true opts[:log] = get_configs[:log] if get_configs[:log] != nil opts[:logfile] = get_configs[:logfile].to_s if get_configs[:logfile].present? opts[:iskip] = get_configs[:iskip].to_s if get_configs[:iskip].present? opts[:include] = get_configs[:include].to_s if get_configs[:include].present? opts[:iinclude] = get_configs[:iinclude].to_s if get_configs[:iskip].present? opts[:ignore_fragments] = get_configs[:ignore_fragments] if get_configs[:ignore_fragments] != nil opts end