module Evoker

Evoker is a tool to manage external dependencies of a project using Rake to run downloads.

Constants

ENTITIES

{Rake::FileList} of defined entities @example can be used as dependency for the default target

task :default => Evoker::ENTITIES

Public Class Methods

entity(name, *args, &block) click to toggle source

Base entity definition (wrapper over {EntityTask})

@param [#to_s] name name of task and directory @param *args arguments for {EntityTask#initialize} @yield [Rake::Task] block executed to populate target directory @return [EntityTask] defined task

# File lib/evoker.rb, line 71
def entity(name, *args, &block)
  Evoker::EntityTask.define_task(name, *args, &block)
end

git(name, opts={}) click to toggle source

Check out Git repository

# File lib/evoker.rb, line 128
def git(name, opts={})
  opts[:git] ||= "git"
  entity name do |t|
    cmd = "#{opts[:git]} clone"
    cmd << " #{opts[:clone_args]}" if opts[:clone_args]
    cmd << " #{t.config[:clone_args]}" if t.config && t.config[:clone_args]
    cmd << " #{opts[:url]}" if opts[:url]
    cmd << " #{t.config[:url]}" if t.config && t.config[:url]
    cmd << " #{t.name}"

    if rev = opts[:revision] || ( t.config && t.config[:revision] )
      cmd << " && cd #{t.name}" \
        " && #{opts[:git]} checkout -b evoker-checkout #{rev}"
    end
    sh cmd
  end
end

mercurial(name, opts={}) click to toggle source

Check out Mercurial repository

# File lib/evoker.rb, line 148
def mercurial(name, opts={})
  opts[:hg] ||= "hg"
  entity name do |t|
    cmd = "#{opts[:hg]} clone"
    cmd << " #{args}" if args = opts[:clone_args] || ( t.config && t.config[:clone_args] )
    cmd << " -r #{opts[:revision]}" if opts[:revision]
    cmd << " -r #{t.config[:revision]}" if t.config && t.config[:revision]
    cmd << " #{opts[:url]}" if opts[:url]
    cmd << " #{t.config[:url]}" if t.config && t.config[:url]
    cmd << " #{t.name}"
    sh cmd
  end
end

patch(entity_name, patches, patch_args=nil) click to toggle source

Apply patch to an entity

# File lib/evoker.rb, line 176
def patch(entity_name, patches, patch_args=nil)
  task entity_name => patches do |t|
    patches = [ patches ] unless patches.respond_to?(:each)
    cmd = "set -e -x\ncd #{t.name}\n"
    patches.each do |patch|
      cmd << "patch #{patch_args} < ../#{patch}\n"
    end
    sh cmd
  end
end

pip_requirements(file, args={}) click to toggle source

Download Python requirements using pip

# File lib/evoker/python.rb, line 58
def pip_requirements(file, args={})
  stampfile = "#{file}.stamp"
  if args[:virtualenv]
    args[:pip] = "#{args[:virtualenv]}/bin/pip"
  else
    args[:pip] ||= smart_const_get(:pip)
  end
  pip_cmd = "#{args[:pip]}"
  pip_cmd << " #{args[:args]}" if args[:args]
  pip_cmd << " install"
  pip_cmd << " #{args[:install_args]}" if args[:install_args]
  pip_cmd << " -r #{file}"

  t = file stampfile => file do
    sh pip_cmd
    File.open(stampfile, 'w') { |f| f.write(DateTime::now.to_s) }
  end
  task t => args[:virtualenv] if args[:virtualenv]
  CLOBBER.add t.name
  ENTITIES.add t.name
  t
end

smart_const_get(name) click to toggle source

Get smart constant’s effective value

Effective value is:

name.to_s.upcase environment variable, if present
Otherwise, user-defined top-level constant named ‘name.to_s.upcase`
Otherwise, default set with {smart_const}
Otherwise, nil

@param name [#to_s] constant’s name

# File lib/evoker.rb, line 219
def smart_const_get(name)
  name = name.to_s.upcase
  if ENV.has_key?(name)
    ENV[name]
  elsif Object.const_defined?(name)
    Object.const_get(name)      
  else
    @@SMART_CONST_DEFAULTS ||= {}
    @@SMART_CONST_DEFAULTS[name]
  end
end

subversion(name, opts={}) click to toggle source

Check out Subversion repository

# File lib/evoker.rb, line 108
def subversion(name, opts={})
  opts[:svn] ||= "svn"
  entity name do |t|
    cmd = "#{opts[:svn]}"
    cmd << " #{opts[:svn_args]}" if opts[:svn_args]
    cmd << " #{t.config[:svn_args]}" if t.config && t.config[:svn_args]
    cmd << " checkout -q"
    cmd << " #{opts[:checkout_args]}" if opts[:checkout_args]
    cmd << " #{t.config[:checkout_args]}" if t.config && t.config[:checkout_args]
    cmd << " -r #{opts[:revision]}" if opts[:revision]
    cmd << " -r #{t.config[:revision]}" if t.config && t.config[:revision]
    cmd << " #{opts[:url]}" if opts[:url]
    cmd << " #{t.config[:url]}" if t.config && t.config[:url]
    cmd << " #{t.name}"
    sh cmd
  end
end

symlink_(target, original, args={}) click to toggle source

Entity that is a symlink to another path (FIXME:rename)

# File lib/evoker.rb, line 190
def symlink_(target, original, args={})
  entity target => original do
    require 'pathname'
    original = Pathname.new(original.to_s).relative_path_from(
      Pathname.new(File.dirname(original.to_s)))
    ln_sf original.to_s, target.to_s
  end
end

tarball(basename, options={}) click to toggle source

Download & unpack a tarball

# File lib/evoker.rb, line 164
def tarball(basename, options={})
  tarball = wget options[:url], options[:wget_options]||{}
  entity basename => tarball do |t|
    dirname = options[:dirname] || File.basename(tarball.name, options[:ext] || '.tar.gz')
    rm_rf dirname
    sh "#{options[:decompress] || 'tar -xzf'} #{tarball}"
    ln_s dirname, basename unless options[:no_symlink]
  end
end

virtualenv(*args) click to toggle source

Create Python virtual environment

# File lib/evoker/python.rb, line 11
def virtualenv(*args)
  if args.last.is_a? Hash
    opts = args.pop
  else
    opts = {}
  end
  
  if opts[:download_virtualenv]
    opts[:python] ||= smart_const_get(:python)
    opts[:virtualenv] = "#{opts[:python]} ./virtualenv.py"
    opts[:virtualenv_version] ||= smart_const_get(:virtualenv_version)
    opts[:virtualenv_url] ||= "http://github.com/pypa/virtualenv/raw/#{opts[:virtualenv_version]}/virtualenv.py"
    wget_virtualenv = wget opts[:virtualenv_url],
      :args => '--no-check-certificate',
      :no_entity => true
    CLOBBER.add(['virtualenv.pyc', 'setuptools-*.egg'])
  else
    opts[:virtualenv] ||= 'virtualenv'
    wget_virtualenv = nil
  end
  opts[:args] ||= nil

  virtualenv_command = "#{opts[:virtualenv]}"
  virtualenv_command << " #{opts[:args]}" if opts[:args]

  desc "Python virtual environment"
  venv = entity(*args) do |t|
    sh "#{virtualenv_command} #{t.name}"
  end

  task venv => wget_virtualenv if wget_virtualenv
  venv
end

virtualenv_site_package(path, opts={}) click to toggle source

Create a symbolic link to virtualenv’s site-packages dir

# File lib/evoker/python.rb, line 47
def virtualenv_site_package(path, opts={})
  opts[:target] ||= File.basename(path)
  opts[:virtualenv] ||= :python
  venv = Rake::Task[opts[:virtualenv]].name
  ln_sf File.join('..', '..', '..', '..', path),
        File.join(Dir["#{venv}/lib/python*/site-packages"].first,
                  opts[:target])
end

wget(url, opts={}) click to toggle source

Download a file using wget.

@param [#to_s] url address to download from @param [Hash] opts options @option opts [#to_s] :output_file (basename of ‘url`) name of target file @option opts [#to_s] :wget (’wget’) wget command to use @option opts [#to_s] :args (nil) custom command line arguments for wget @option opts [True, False] :no_entity (false)

do not add task to {Evoker::ENTITIES}

# File lib/evoker.rb, line 85
def wget(url, opts={})
  opts[:output_file] ||= begin
                           require 'uri'
                           URI.parse(url).path.split('/').last
                         end
  opts[:wget] ||= 'wget'

  wget_command = "#{opts[:wget]} -O #{opts[:output_file]}"
  wget_command << " #{opts[:args]}" if opts[:args]
  wget_command << " #{url} && touch #{opts[:output_file]}"

  CLOBBER.add(opts[:output_file])
  ENTITIES.add(opts[:output_file]) unless opts[:no_entity]

  desc "Download #{url} as #{opts[:output_file]}"
  file opts[:output_file] do
    sh wget_command
    touch opts[:output_file]
  end
end

Private Class Methods

smart_const(name, default) click to toggle source

Define smart constant’s default @param name [#to_s] constant’s name (will be upcased) @param default constant’s default value

# File lib/evoker.rb, line 205
def self.smart_const(name, default)
  @@SMART_CONST_DEFAULTS ||= {}
  @@SMART_CONST_DEFAULTS[name.to_s.upcase] = default
end

Public Instance Methods

_get_bucket() click to toggle source

# File lib/evoker/s3cache.rb, line 8
def _get_bucket
  $s3 ||= Fog::Storage.new(
    :provider => "AWS",
    :aws_access_key_id => CACHE_S3_ACCESS_KEY_ID,
    :aws_secret_access_key => CACHE_S3_SECRET_ACCESS_KEY,
    :persistent => false)
  $bucket ||= $s3.directories.get(CACHE_S3_BUCKET)
end

cached(output_file, checksum=nil) { |output_file| ... } click to toggle source

Cache result of a file task in local directory.

If cached ‘output_file` exists and matches the checksum (if one is given), task to copy file from cache directory to target is returned, and block is not executed.

If cached ‘output_file` exists but does not match the checksum, it is removed.

If ‘output file` does not exist or did not match the checksum, block is executed. Block should return a file task. This task will have extra code appended:

a checksum test: if checksum is given - error is raised if created file does not match the checksum
copying created file to cache directory

Cache directory is taken from a smart constant (see {#smart_const_get}) ‘:cache_path`, default is ’cache’.

@param [String] output_file File to uncache or create @param [String] checksum SHA-256 checksum of file (optional, but recommended) @yield Task to create file if not found in cache @return Task to uncache or create file

# File lib/evoker/local_cache.rb, line 38
def cached(output_file, checksum=nil)
  raise 'Block for Evoker::cached not provided' unless block_given?

  cached_path_elts = []
  cached_path_elts << smart_const_get(:cache_path)
  cached_path_elts << checksum[0..1] if checksum
  cached_path_elts << checksum[2..3] if checksum
  cached_path_elts << File.basename(output_file)
  cached_path = File.join(*cached_path_elts)

  if File.exists?(cached_path) &&
      checksum &&
      Digest::SHA256.file(cached_path).hexdigest != checksum
    puts "WARN: checksum mismatch for cached #{File.basename(output_file)}, removing."
    FileUtils::rm cached_path
  end

  if File.exists?(cached_path)
    # Cached file exists and matches the given checksum
    rv = file output_file do
      FileUtils::cp cached_path, output_file
    end
  else
    # Cached file does not exist
    rv = yield output_file

    # Cache file after downloading
    task rv do
      if checksum &&
          Digest::SHA256.file(output_file).hexdigest != checksum
        raise "Checksum mismatch for downloaded #{File.basename(output_file)}."
      end
      FileUtils::mkdir_p(File.dirname(cached_path))
      FileUtils::cp output_file, cached_path
    end
  end

  CLEAN << output_file
  CLOBBER << cached_path

  rv
end

cached_wget(url, opts={}) click to toggle source

Download a file using wget, or copy it from local cache

@param [#to_s] url address to download from @param [Hash] opts options (same as wget, + :checksum) @option opts [#to_s] :checksum sha256 sum of file to download

# File lib/evoker/local_cache.rb, line 87
def cached_wget(url, opts={})
  opts[:output_file] ||= begin
                           require 'uri'
                           URI.parse(url).path.split('/').last
                         end

  cached(opts[:output_file], opts[:checksum]) do
    wget url, opts
  end
end

dl(filename) click to toggle source

# File lib/evoker/fullstack.rb, line 103
def dl(filename)
  File.join(smart_const_get(:download_path), filename)
end

download(url, args={}) click to toggle source

# File lib/evoker/fullstack.rb, line 96
def download(url, args={})
  args[:output_file] ||= File.expand_path(File.join(
      smart_const_get(:download_path),
      args[:filename] || File.basename(url)))
  cached_wget(url, args)
end

from_tarball(task_name, tarball_url, args={}, &block) click to toggle source

Download tarball from given URL and unpack it for build.

A file from address ‘tarball_url` is downloaded (via {#cached_wget}) to directory specified in the `:download_path` smart constant, and then unpacked in directory specified in the `:build_path` smart constant.

The block is called in context that defines following methods:

‘tarball_filename`
‘tarball_path`
‘tarball_extension` (e.g. `“.tar.gz”`)
‘source_dir_basename`
‘source_dir`
‘download` (a Rake task that downloads the tarball)
‘unpack` (a Rake task that unpacks the tarball).

Block should define a task or chain of tasks, that compile (and possibly install, depending on needs) module contained in the tarball. First of the tasks should depend on ‘unpack`, and last should be returned from the block.

Task named ‘task_name` that depends on the task returned from the block will be created.

@example

from_tarball :carbon, CARBON_URL do
  file installed('bin/carbon-aggregator.py') => [ PIP, :py2cairo ] do
    rm_f source_file('setup.cfg')
    pip_install source_dir
  end
end

# File lib/evoker/fullstack.rb, line 52
def from_tarball(task_name, tarball_url, args={}, &block)
  task task_name

  build_path = File.expand_path(smart_const_get(:build_path))
  download_path = File.expand_path(smart_const_get(:download_path))

  mkdir_p build_path unless File.directory?(build_path)
  mkdir_p download_path unless File.directory?(download_path)

  tarball_filename = args[:filename] || File.basename(tarball_url)
  tarball_path = File.join(download_path, tarball_filename)
  tarball_extension = args[:extension] ||
    ( tarball_filename =~ /\.(tar(\.(Z|gz|bz2))?|zip)$/ ? $& : nil )
  source_dir_basename = args[:directory] ||
    File.basename(tarball_filename, tarball_extension)
  source_dir = File.join(build_path, source_dir_basename)
  unpack_command = args[:unpack] || {
    '.tar.gz' => 'tar -xzf',
    '.tar.bz2' => 'tar -xjf',
    '.tar.Z' => 'tar -xzf',
    '.tar' => 'tar -xf',
    '.zip' => 'unzip' }[tarball_extension.downcase]

  download = cached_wget(
    tarball_url, args.merge(:output_file => tarball_path))
   
  unpack = file source_dir => download do
    chdir smart_const_get(:build_path) do
      rm_rf source_dir_basename
      sh "#{unpack_command} #{tarball_path}"
    end
  end
  
  ctx = FullStack::Context.new(
    tarball_filename, tarball_path, tarball_extension,
    source_dir_basename, source_dir,
    download, unpack)

  final_file = ctx.instance_eval(&block)

  task final_file => unpack
  task task_name => final_file
end

Private Instance Methods