class Middleman::Cli::CrawlerIndex

This class provides a “crawler” command for the middleman CLI.

Public Class Methods

exit_on_failure?() click to toggle source

Tell Thor to exit with a nonzero exit code on failure

# File lib/middleman-crawler/commands.rb, line 85
def self.exit_on_failure?
  true
end

Public Instance Methods

crawler() click to toggle source
# File lib/middleman-crawler/commands.rb, line 89
def crawler
  configs = get_configs

  env_config = File.join(Middleman::Application.root, 'environments', "#{configs[:environment]}.rb")
  return unless File.exist? env_config

  configurations = Configure.new

  configurations.instance_eval File.read(env_config), env_config, 1

  response = {}
  raw = Rawler::Base.new(configurations.config[:base_url], $stdout, rawler_options)
  raw.validate
  response['errors'] = raw.responses.reject{ |link, response|
    (100..399).include?(response[:status].to_i)
  }
  say("There are some errors: #{errors_message_for(response['errors'])}", :red) if response['errors']
  puts "All crawling actions ok" unless response['errors']
end

Private Instance Methods

errors_message_for(responses) click to toggle source
# File lib/middleman-crawler/commands.rb, line 146
def errors_message_for(responses)
  collection = responses.each_with_object([])  do |(url, response), a|
    a << "url: #{url}, error_code: #{response[:status]}"
  end

  collection.join("; ")

end
get_configs() click to toggle source
# File lib/middleman-crawler/commands.rb, line 111
def get_configs
  configs ||= {
    environment: (options['environment'].presence || "development").to_sym,
    username: options['username'],
    password: options['password'],
    skip: options['skip'],
    wait: options['wait'],
    css: options['css'],
    local: options['local'],
    log: options['log'],
    logfile: options['logfile'],
    iskip: options['iskip'],
    include: options['include'],
    iinclude: options['iinclude'],
    ignore_fragments: options['ignore_fragments']
  }
end
rawler_options() click to toggle source
# File lib/middleman-crawler/commands.rb, line 129
def rawler_options
  opts = {}
  opts[:username] = get_configs[:username].to_s if get_configs[:username].present?
  opts[:password] = get_configs[:password].to_s if get_configs[:password].present?
  opts[:skip] = get_configs[:skip].present? ? get_configs[:skip].to_s : "\#$"
  opts[:wait] = get_configs[:wait].present? ? get_configs[:wait].to_f : 0.1
  opts[:css] = get_configs[:css] != nil ? get_configs[:css] : true
  opts[:local] = get_configs[:local] != nil ? get_configs[:local] : true
  opts[:log] = get_configs[:log] if get_configs[:log] != nil
  opts[:logfile] = get_configs[:logfile].to_s if get_configs[:logfile].present?
  opts[:iskip] = get_configs[:iskip].to_s if get_configs[:iskip].present?
  opts[:include] = get_configs[:include].to_s if get_configs[:include].present?
  opts[:iinclude] = get_configs[:iinclude].to_s if get_configs[:iskip].present?
  opts[:ignore_fragments] = get_configs[:ignore_fragments] if get_configs[:ignore_fragments] != nil
  opts
end