class Classroom

Constants

HELP
VERSION

Public Class Methods

new(config) click to toggle source
# File lib/classroom.rb, line 13
def initialize(config)
  @config = config
end

Public Instance Methods

bailout?(message = 'Continue?') click to toggle source
# File lib/classroom.rb, line 32
def bailout?(message = 'Continue?')
  raise "User cancelled" unless confirm?(message)
end
check_success(status=nil) click to toggle source
# File lib/classroom.rb, line 36
def check_success(status=nil)
  status = status.nil? ? ($? == 0) : status

  if status
    printf("\[\033[32m  OK  \033[0m]\n")
  else
    printf("[\033[31m FAIL \033[0m]\n")
  end
end
confirm?(message = 'Continue?', default = true) click to toggle source
# File lib/classroom.rb, line 22
def confirm?(message = 'Continue?', default = true)
  if default
    print "#{message} [Y/n]: "
    return [ 'y', 'yes', '' ].include? STDIN.gets.strip.downcase
  else
    print "#{message} [y/N]: "
    return [ 'y', 'yes' ].include? STDIN.gets.strip.downcase
  end
end
debug() click to toggle source
# File lib/classroom.rb, line 80
def debug
  require 'pry'
  binding.pry
end
help() click to toggle source
# File lib/classroom.rb, line 75
def help
  require 'classroom/help'
  puts Classroom::HELP
end
page() click to toggle source
# File lib/classroom/page.rb, line 2
def page
  require 'json'
  require 'rest-client'

  begin
    config = showoff_config
    pd_key = File.read('/opt/pltraining/etc/pagerduty.key').strip
    raise 'Missing PagerDuty key' if pd_key.empty?
  rescue => e
    puts "Cannot load configuration"
    puts e.message
    exit 1
  end

  puts "---------  You're about to page and possibly wake someone up.  ---------"
  puts "Please check the Troubleshooting Guide for solutions to common problems."
  puts
  puts "https://github.com/puppetlabs/courseware/blob/master/TroubleshootingGuide.md"
  puts
  if confirm?("Have you done everything in the troubleshooting guide?") then
    print 'Describe the problem in a short sentence: '
    description  = STDIN.gets.strip

    print 'Enter the email or phone number where you can be reached: '
    contact      = STDIN.gets.strip

    page_message = "#{description}\n" +
                   "Contact: #{contact}\n" +
                   "Course: #{config['course']} #{config['version']}\n" +
                   "ID: #{config['event_id']}"
    page_data = {
      "service_key" => pd_key,
      "event_type"  => "trigger",
      "description" => page_message
    }

    puts "Sending page. Make sure you've posted about the issue in HipChat."
    response = JSON.parse(RestClient.post(
      "https://events.pagerduty.com/generic/2010-04-15/create_event.json",
      page_data.to_json,
      :content_type => :json,
      :accept => :json
    ))
    puts response unless response['status'] == 'success'
  end
end
performance(subject) click to toggle source
# File lib/classroom/performance.rb, line 2
def performance(subject)
  if subject.empty?
    puts "This will take performance related snapshots, which will be uploaded for"
    puts "engineering analysis. You may also record performance notes into the log."
    puts
    puts "Usage: classroom performance [ log <message>  | snapshot ]"
    puts
    puts "  * log: Record a message into classroom log and take a snapshot."
    puts "  * snapshot: Save snapshot of classroom statistics."
    puts
    exit 1
  end

  case subject.shift
  when 'log'
    message = subject.empty? ? "Misc performance issue noted." : subject.join(' ')
    $logger.warn message
    record_snapshot

  when 'snapshot'
    $logger.debug "Scheduled performance snapshot"
    record_snapshot

  else
    raise "No such action"
  end

  def record_snapshot
    $logger.debug "-------------------------------- top -bn1 ----------------------------------\n#{`top -bn1`}"
    $logger.debug "-------------------------------- vmstat ------------------------------------\n#{`vmstat`}"
    $logger.debug "-------------------------------- netstat -a --------------------------------\n#{`netstat -a`}"
    $logger.debug "-------------------------------- iostat ------------------------------------\n#{`iostat`}"
    $logger.debug "-------------------------------- mpstat -P ALL -----------------------------\n#{`mpstat -P ALL`}"

    FileUtils.mkdir_p '/var/log/puppetlabs/classroom-traffic'
    `tcpdump -G 15 -W 1 -w /var/log/puppetlabs/classroom-traffic/#{Time.now.to_i}.pcap -i any > /dev/null 2>&1 &`
  end

end
record_snapshot() click to toggle source
# File lib/classroom/performance.rb, line 29
def record_snapshot
  $logger.debug "-------------------------------- top -bn1 ----------------------------------\n#{`top -bn1`}"
  $logger.debug "-------------------------------- vmstat ------------------------------------\n#{`vmstat`}"
  $logger.debug "-------------------------------- netstat -a --------------------------------\n#{`netstat -a`}"
  $logger.debug "-------------------------------- iostat ------------------------------------\n#{`iostat`}"
  $logger.debug "-------------------------------- mpstat -P ALL -----------------------------\n#{`mpstat -P ALL`}"

  FileUtils.mkdir_p '/var/log/puppetlabs/classroom-traffic'
  `tcpdump -G 15 -W 1 -w /var/log/puppetlabs/classroom-traffic/#{Time.now.to_i}.pcap -i any > /dev/null 2>&1 &`
end
reload_service(service, pattern) click to toggle source
# File lib/classroom/restart.rb, line 87
def reload_service(service, pattern)
  puts "> Reloading #{service}..."
  system("kill -HUP `pgrep -f #{pattern}`")
end
reset(subject) click to toggle source
# File lib/classroom/reset.rb, line 2
  def reset(subject)
    if subject.size != 1
      puts <<-EOF
Usage: classroom reset <password | certificates | filesync>

This tool will reset or regenerate:
  * root's login password and update the /etc/issue screen
  * delete and redeploy all FileSync caches
  * *all* ssl certificates in the PE stack. (warning: destructive!)

EOF
      exit 1
    end

    case subject.first
    when :password
      reset_password
    when :certificates
      reset_certificates
    when :filesync
      reset_filesync
    else
      raise "Unknown action."
    end
  end
reset_certificates() click to toggle source
# File lib/classroom/reset.rb, line 41
def reset_certificates
  require "fileutils"

  # Automate the process of regenerating certificates on a monolithic master
  # https://docs.puppet.com/pe/latest/trouble_regenerate_certs_monolithic.html
  #
  timestamp     = Time.now.to_i
  certname      = `puppet master --configprint certname`.strip
  ssldir        = '/etc/puppetlabs/puppet/ssl'
  puppetdbcerts = '/etc/puppetlabs/puppetdb/ssl'
  consolecerts  = '/opt/puppetlabs/server/data/console-services/certs'
  pgsqlcerts    = '/opt/puppetlabs/server/data/postgresql/9.6/data/certs'
  orchcerts     = '/etc/puppetlabs/orchestration-services/ssl'

  ["puppet", "puppetdb", "console-services", "postgresql", "orchestration"].each do |path|
    FileUtils.mkdir_p("/root/certificates.bak/#{path}")
  end
  FileUtils.cp_r("#{ssldir}",        "/root/certificates.bak/puppet/#{timestamp}")
  FileUtils.cp_r("#{puppetdbcerts}", "/root/certificates.bak/puppetdb/#{timestamp}")
  FileUtils.cp_r("#{consolecerts}",  "/root/certificates.bak/console-services/#{timestamp}")
  FileUtils.cp_r("#{pgsqlcerts}",    "/root/certificates.bak/postgresql/#{timestamp}")
  FileUtils.cp_r("#{orchcerts}",     "/root/certificates.bak/orchestration/#{timestamp}")

  puts "Certificates backed up to ~/certificates.bak"

  puts
  puts
  puts "#####################################################################"
  puts
  puts "      If you regenerate the Puppet CA to start fresh, then"
  puts "ALL client certificates will be invalidated and must be regenerated!"
  puts
  puts "        -- This should only be done as a last resort --"
  puts
  puts "#####################################################################"
  puts
  if confirm?('Would you like to regenerate the CA?', false)
    FileUtils.rm_rf("#{ssldir}/*")
    system("puppet cert list -a")
  end

  FileUtils.rm_f("/opt/puppetlabs/puppet/cache/client_data/catalog/#{certname}.json")
  system("puppet cert clean #{certname}")
  system("puppet infrastructure configure --no-recover")
  system("puppet agent -t")

  puts "All done. If you regenerated the CA, then regenerate all client certificates now."
end
reset_filesync() click to toggle source
# File lib/classroom/reset.rb, line 90
def reset_filesync
  puts
  puts
  puts "################################################################################"
  puts
  puts "This script will completely delete and redeploy all environments without backup!"
  puts "            The operation may take up to five minutes to complete."
  puts
  puts "################################################################################"
  puts
  bailout?

  system("systemctl stop pe-puppetserver")

  # filesync cache
  FileUtils.rm_rf("/opt/puppetlabs/server/data/puppetserver/filesync")

  # r10k cache
  FileUtils.rm_rf("/opt/puppetlabs/server/data/code-manager/git")

  # code manager worker thread caches
  FileUtils.rm_rf("/opt/puppetlabs/server/data/code-manager/worker-caches")
  FileUtils.rm_rf("/opt/puppetlabs/server/data/code-manager/cache")

  # possibly stale environment codebases
  FileUtils.rm_rf("/etc/puppetlabs/code/*")
  FileUtils.rm_rf("/etc/puppetlabs/code-staging/environments")

  system("systemctl start pe-puppetserver")
  system("puppet code deploy --all --wait")
end
reset_password() click to toggle source
# File lib/classroom/reset.rb, line 28
def reset_password
  print "Enter new root password: "
  password = STDIN.gets.chomp

  %x(echo "root:#{password}"|chpasswd)

  File.open('/var/local/password','w') do |f|
    f.puts password
  end

  %x(/etc/rc.local 2>/dev/null)
end
restart(subject) click to toggle source
# File lib/classroom/restart.rb, line 2
  def restart(subject)
    if subject.empty?
      puts <<-EOF
This tool simply helps restart the PE services in the right order.
It will send a HUP signal to Puppetserver by default which is much
faster than a full restart.

You can also restart Docker containers in classes that use them.

If you do need the full restart, please pass the -f option.

Service names:
    * puppetserver
    * console
    * puppetdb
    * orchestration
    * mcollective
    * containers
    * all (restart all PE services in the proper order)

Examples:
    * classroom restart puppetdb puppetserver
    * classroom restart puppetserver console -f
    * classroom restart all -f

EOF
      exit 1
    end

    # normalize to lowercase strings so we can pattern match
    subject.map! { |x| x.to_s.downcase }

    if subject.include? 'all'
      puts "Restarting all PE stack services. This may take a few minutes..."
      subject.concat ['puppetdb', 'puppetserver', 'orchestrator', 'console', 'mcollective', 'puppet']
      subject.uniq!
    else
      puts "Restarting selected PE components."
    end

    if subject.grep(/puppetdb|pdb/).any?
      restart_service('pe-postgresql.service')
      restart_service('pe-puppetdb.service')
    end

    if subject.grep(/puppetserver|server/).any?
      if @config[:force]
        restart_service('pe-puppetserver.service')
      else
        reload_service('pe-puppetserver.service', 'puppet-server')
      end
    end

    if subject.grep(/orch|pxp/).any?
      restart_service('pe-orchestration-services.service')
      restart_service('pxp-agent.service')
    end

    if subject.grep(/console/).any?
      restart_service('pe-console-services.service')
      restart_service('pe-nginx.service')
    end

    if subject.grep(/mco/).any?
      restart_service('pe-activemq.service')
      restart_service('mcollective.service')
    end

    if subject.include? 'puppet'
      restart_service('puppet.service')
    end

    if subject.include? 'containers'
      `systemctl list-units`.each_line do |line|
          restart_service($1) if line =~ /^(docker-\S+)/
      end
    end

  end
restart_service(service) click to toggle source
# File lib/classroom/restart.rb, line 82
def restart_service(service)
  puts "- Restarting #{service}..."
  system("systemctl restart #{service}")
end
same_file(filename, list) click to toggle source
# File lib/classroom/troubleshoot.rb, line 166
def same_file(filename, list)
  require 'digest'
  list  = Array(list) # coerce if needed
  left  = Digest::MD5.hexdigest(File.read(filename))

  list.each do |path|
    return false unless (left == Digest::MD5.hexdigest(File.read(path)))
  end

  return true
end
sanitize() click to toggle source
# File lib/classroom/sanitize.rb, line 2
def sanitize
  require 'yaml'
  require 'fileutils'
  require 'puppetclassify'

  puts 'Sanitizing your VM for your next delivery...'

  certname     = `puppet master --configprint certname`.strip
  master       = `puppet agent --configprint server`.strip
  classifier   = "http://#{master}:4433/classifier-api"
  known_groups = [ 'All Nodes', 'Agent-specified environment', 'Production environment', /^PE / ]
  known_users  = [ 'admin', 'api_user', 'deployer' ]
  auth_info    = {
    'ca_certificate_path' => `puppet master --configprint localcacert`.strip,
    'certificate_path'    => `puppet master --configprint hostcert`.strip,
    'private_key_path'    => `puppet master --configprint hostprivkey`.strip,
  }

  group_pattern  = Regexp.union(known_groups)
  puppetclassify = PuppetClassify.new(classifier, auth_info)
  puppetclassify.groups.get_groups.each do |group|
    next if group['name'].match(group_pattern)
    puppetclassify.groups.delete_group(group['id'])
    print '.'
  end
  puts

  # depends on pltraining/rbac module
  users = YAML.load(`puppet resource rbac_user --to_yaml`)
  users['rbac_user'].each do |user, data|
    next if known_users.include? user
    puts "puppet resource rbac_user #{user} ensure=absent"
    system("puppet resource rbac_user #{user} ensure=absent")
    print '.'
  end
  puts

  `puppet cert list --all --machine-readable`.each_line do |line|
    next unless line.start_with? '+'
    name = line.gsub('"', '').split[1]
    next if name.start_with? 'pe-internal'
    next if name == certname

    system("puppet node deactivate #{name}")
    system("puppet cert clean #{name}")
    print '.'
  end
  puts

  Dir.glob('/home/*').each do |path|
    next if ['/home/training', '/home/showoff'].include? path
    system("userdel #{File.basename(path)}")
    FileUtils.rm_rf path
    print '.'
  end
  puts

  Dir.glob('/etc/puppetlabs/code-staging/environments/*').each do |path|
    next if File.basename(path) == 'production'
    FileUtils.rm_rf path
    print '.'
  end
  puts

  Dir.glob('/etc/puppetlabs/code/environments/*').each do |path|
    next if File.basename(path) == 'production'
    FileUtils.rm_rf path
    print '.'
  end
  puts

  FileUtils.rm_rf('/var/repositories/*')
end
showoff_config() click to toggle source
# File lib/classroom.rb, line 46
def showoff_config
  presentation = showoff_working_directory()
  data         = JSON.parse(File.read("#{presentation}/stats/metadata.json")) rescue {}

  data['event_id'] ||= Time.now.to_i
  data['course']   ||= File.basename(presentation.strip)
  data['path']     ||= presentation
  data
end
showoff_working_directory() click to toggle source
# File lib/classroom.rb, line 56
def showoff_working_directory
  begin
    # get the path of the currently configured showoff presentation
    data = {}
    path = '/usr/lib/systemd/system/showoff-courseware.service'
    File.read(path).each_line do |line|
      setting = line.split('=')
      next unless setting.size == 2

      data[setting.first] = setting.last
    end
    data['WorkingDirectory'].strip
  rescue Errno::ENOENT => e
    $logger.warn 'Cannot find classroom Showoff presentation'
    $logger.debug e.message
    'unconfigured'
  end
end
submit() click to toggle source
# File lib/classroom/submit.rb, line 2
def submit
  require 'fileutils'
  require 'aws-sdk'

  config   = showoff_config
  event_id = config['event_id']
  course   = config['course']
  location = config['path']

  print 'Enter your Puppet email address: '
  email = STDIN.gets.strip

  if email =~ /@puppet(labs)?.com$/
    puts "Please go to your learn dashboard and ensure that attendance is accurate"
    puts "and then close this class delivery to mark it as complete."
    puts " -- #{@config[:learndot]}"
    puts
  end

  begin
    # depends on root's credentials as managed by bootstrap
    s3 = Aws::S3::Resource.new(region:'us-west-2')

    # record the module versions in use
    system("puppet module list > /var/log/puppetlabs/classroom-modules")

    filename = "classroom-perflogs-#{course}-#{email}-#{event_id}.tar.gz"
    fullpath = "/var/cache/#{filename}"
    system("tar -cf #{fullpath} /var/log/puppetlabs/")
    obj = s3.bucket('classroom-performance').object(filename)
    obj.upload_file(fullpath)
    FileUtils.rm(fullpath)

    filename = "classroom-stats-#{course}-#{email}-#{event_id}.tar.gz"
    fullpath = "/var/cache/#{filename}"
    system("tar -cf #{fullpath} #{location}/stats/")
    obj = s3.bucket('classroom-statistics').object(filename)
    obj.upload_file(fullpath)
    FileUtils.rm(fullpath)

  rescue LoadError, StandardError => e
    $logger.warn "S3 upload failed. No network?"
    $logger.warn e.message
    $logger.debug e.backtrace
  end

  # clean up for next delivery
  system("puppet resource service showoff-courseware ensure=stopped > /dev/null")
  FileUtils.rm_rf("#{location}/stats")
  FileUtils.rm_f("#{location}/courseware.yaml")
  FileUtils.rm_f("#{location}/_files/share/nearby_events.html")
  system("puppet resource service showoff-courseware ensure=running > /dev/null")

end
troubleshoot() click to toggle source
# File lib/classroom/troubleshoot.rb, line 2
def troubleshoot
  master   = `puppet agent --configprint server`.strip
  codedir  = `puppet master --configprint codedir`.strip
  filesync = '/etc/puppetlabs/puppetserver/conf.d/file-sync.conf'
  release  = File.read('/etc/puppetlabs-release').to_f rescue 0
  legacy   = release < 7.0

  if File.exist? filesync
    # why isn't there a configprint setting for this?
    staging = `hocon -f #{filesync} get file-sync.repos.puppet-code.staging-dir`.strip
    puts "Running checks for Code Manager configurations:"
  else
    staging = codedir
    puts "Running checks for configurations without Code Manager:"
  end

  print "Cleaning any stray .git directories in: #{codedir}..."
  sleep 1
  system("find #{codedir} -name .git -type d -print -exec rm -rf {} +")
  check_success

  print "Validating permissions on: #{codedir}..."
  sleep 1
  system("find #{codedir} '!' -user pe-puppet -print -exec chown pe-puppet:pe-puppet {} +")
  check_success

  if codedir != staging
    puts "Validating permissions on: #{staging}..."
    sleep 1
    system("find #{staging} '!' -user pe-puppet -print -exec chown pe-puppet:pe-puppet {} +")
    check_success
  end

  # only check legacy systems that rely on manual installs
  if legacy
    if File.exist? '/home/training/courseware'
      print "Sanitizing uploaded courseware..."
      sleep 1
      FileUtils.rm_f '/home/training/courseware/stats/viewstats.json'
      FileUtils.rm_f '/home/training/courseware/stats/forms.json'
      check_success
    else
      check_success(false)
      puts "\tYou don't seem to have uploaded the courseware from your host system"
    end
  end

  print "Checking Forge connection..."
  if system("curl -s https://forge.puppet.com >/dev/null 2>&1")
    if legacy
      puts "Ensuring the latest version of pltraining/classroom in #{staging}..."
      system("puppet module upgrade pltraining/classroom --modulepath #{staging}")
      check_success
    else
      check_success(true)
    end
  else
    if `awk '$1 == "server" {print $2}' /etc/ntp.conf` != master
      check_success(false)
      puts "\tCould not reach the Forge. You should classify your master as $offline => true"
    else
      puts "\tYou appear to be in offline mode."
    end
  end

  if codedir != staging
    print "Ensuring you have a valid deploy token..."
    if File.exist? '/root/.puppetlabs/token'
      token  = `puppet access show`
      api    = 'https://#{master}:4433/rbac-api/v1/users/current'
      status = `curl -k --write-out "%{http_code}" --silent --output /dev/null #{api} -H "X-Authentication:#{token}"`.strip
      if status != "200"
        print "\nRegenerating invalid token..."
        FileUtils.rm_f('/root/.puppetlabs/token')
        check_success
      end
    end

    unless File.exist? '/root/.puppetlabs/token'
      print "\nGenerating new token."
      system('puppet plugin download > /dev/null')
      system('puppet resource rbac_user deployer ensure=present display_name=deployer email=deployer@puppetlabs.vm password=puppetlabs roles=4 > /dev/null')
      system('echo "puppetlabs" | HOME=/root /opt/puppetlabs/bin/puppet-access login deployer --lifetime 14d > /dev/null')
      check_success
    else
      check_success(true)
    end

    puts
    puts "If you're having trouble with Code Manager or FileSync, deleting all deployed"
    puts "code and destroying all caches can sometimes help you get going again."
    puts
    if confirm?('Would you like to nuke it all and start over?', false)
      reset([:filesync])
    end
  end

  print "Validating SSL certificates..."
  if valid_certificates
    check_success(true)
  else
    puts
    puts "It looks like there is an inconsistency with your master's SSL certificates."
    puts "Regenerating certificates may take up to five minutes."
    puts
    if confirm?('Would you like to try regenerating certificates?', false)
      reset([:certificates])
    end
  end

  puts
  puts 'Done checking. Fix any errors noted above and try again.'
  puts 'If still having troubles, try some of the following steps.'
  puts 'Note that both tail and journalctl have a "-f" follow mode.'
  puts
  puts 'Log files:'
  puts '  * tail /var/log/puppetlabs/puppetserver/puppetserver.log'
  puts '  * tail /var/log/puppetlabs/console-services/console-services.log'
  puts '  * tail any other interesting log files in /var/log/puppetlabs'
  puts 'System logs:'
  puts '  * journalctl -eu pe-puppetserver'
  puts '  * journalctl -eu pe-console-services'
  puts '  * systemctl list-units | egrep "pe-|puppet"'
  puts 'Edu tools:'
  puts '  * tail /var/log/puppetfactory'
  puts '  * journalctl -eu abalone'
  puts '  * journalctl -eu puppetfactory'
  puts '  * journalctl -eu showoff-courseware'
  puts '  * classroom reset certificates'
  puts '  * classroom restart ${1}'
  puts
  puts 'Have you searched the Troubleshooting Guide for your issue?'
  puts "If you're still stuck, page the on-call support with 'classroom page'"
end
update() click to toggle source
# File lib/classroom.rb, line 17
def update
  puts "Updating system and courseware..."
  system("#{@config[:bindir]}/puppet agent -t --confdir #{@config[:confdir]}")
end
valid_certificates() click to toggle source
# File lib/classroom/troubleshoot.rb, line 137
def valid_certificates
  certname      = `puppet master --configprint certname`.strip
  ssldir        = '/etc/puppetlabs/puppet/ssl'
  puppetdbcerts = '/etc/puppetlabs/puppetdb/ssl'
  consolecerts  = '/opt/puppetlabs/server/data/console-services/certs'
  pgsqlcerts    = '/opt/puppetlabs/server/data/postgresql/9.6/data/certs'
  orchcerts     = '/etc/puppetlabs/orchestration-services/ssl'

  cert = same_file("#{ssldir}/certs/#{certname}.pem", [
                                                        "#{puppetdbcerts}/#{certname}.cert.pem",
                                                        "#{pgsqlcerts}/_local.cert.pem",
                                                        "#{consolecerts}/#{certname}.cert.pem",
                                                        "#{orchcerts}/#{certname}.cert.pem",
                                                      ])
  public_key = same_file("#{ssldir}/public_keys/#{certname}.pem", [
                                                        "#{puppetdbcerts}/#{certname}.public_key.pem",
                                                        "#{consolecerts}/#{certname}.public_key.pem",
                                                        "#{orchcerts}/#{certname}.public_key.pem",
                                                      ])
  private_key = same_file("#{ssldir}/private_keys/#{certname}.pem", [
                                                        "#{puppetdbcerts}/#{certname}.private_key.pem",
                                                        "#{pgsqlcerts}/_local.private_key.pem",
                                                        "#{consolecerts}/#{certname}.private_key.pem",
                                                        "#{orchcerts}/#{certname}.private_key.pem",
                                                      ])

  return (cert and public_key and private_key)
end
validate() click to toggle source
# File lib/classroom/validate.rb, line 2
def validate
  require 'rake'
  require 'rspec/core/rake_task'

  puts "Validating configuration..."
  Dir.chdir(@config[:specdir]) do
    RSpec::Core::RakeTask.new(:spec) do |t|
      t.rspec_opts = "-I #{@config[:specdir]} --format progress"
      t.pattern    = 'localhost/*_spec.rb'
      t.verbose    = false
    end

    Rake::Task[:spec].invoke
  end

end