module Evoker
Evoker
is a tool to manage external dependencies of a project using Rake to run downloads.
Constants
- ENTITIES
{Rake::FileList} of defined entities @example can be used as dependency for the default target
task :default => Evoker::ENTITIES
Public Class Methods
Base entity definition (wrapper over {EntityTask})
@param [#to_s] name name of task and directory @param *args arguments for {EntityTask#initialize} @yield [Rake::Task] block executed to populate target directory @return [EntityTask] defined task
# File lib/evoker.rb, line 71 def entity(name, *args, &block) Evoker::EntityTask.define_task(name, *args, &block) end
Check out Git repository
# File lib/evoker.rb, line 128 def git(name, opts={}) opts[:git] ||= "git" entity name do |t| cmd = "#{opts[:git]} clone" cmd << " #{opts[:clone_args]}" if opts[:clone_args] cmd << " #{t.config[:clone_args]}" if t.config && t.config[:clone_args] cmd << " #{opts[:url]}" if opts[:url] cmd << " #{t.config[:url]}" if t.config && t.config[:url] cmd << " #{t.name}" if rev = opts[:revision] || ( t.config && t.config[:revision] ) cmd << " && cd #{t.name}" \ " && #{opts[:git]} checkout -b evoker-checkout #{rev}" end sh cmd end end
Check out Mercurial repository
# File lib/evoker.rb, line 148 def mercurial(name, opts={}) opts[:hg] ||= "hg" entity name do |t| cmd = "#{opts[:hg]} clone" cmd << " #{args}" if args = opts[:clone_args] || ( t.config && t.config[:clone_args] ) cmd << " -r #{opts[:revision]}" if opts[:revision] cmd << " -r #{t.config[:revision]}" if t.config && t.config[:revision] cmd << " #{opts[:url]}" if opts[:url] cmd << " #{t.config[:url]}" if t.config && t.config[:url] cmd << " #{t.name}" sh cmd end end
Apply patch to an entity
# File lib/evoker.rb, line 176 def patch(entity_name, patches, patch_args=nil) task entity_name => patches do |t| patches = [ patches ] unless patches.respond_to?(:each) cmd = "set -e -x\ncd #{t.name}\n" patches.each do |patch| cmd << "patch #{patch_args} < ../#{patch}\n" end sh cmd end end
Download Python requirements using pip
# File lib/evoker/python.rb, line 58 def pip_requirements(file, args={}) stampfile = "#{file}.stamp" if args[:virtualenv] args[:pip] = "#{args[:virtualenv]}/bin/pip" else args[:pip] ||= smart_const_get(:pip) end pip_cmd = "#{args[:pip]}" pip_cmd << " #{args[:args]}" if args[:args] pip_cmd << " install" pip_cmd << " #{args[:install_args]}" if args[:install_args] pip_cmd << " -r #{file}" t = file stampfile => file do sh pip_cmd File.open(stampfile, 'w') { |f| f.write(DateTime::now.to_s) } end task t => args[:virtualenv] if args[:virtualenv] CLOBBER.add t.name ENTITIES.add t.name t end
Get smart constant’s effective value
Effective value is:
-
name.to_s.upcase
environment variable, if present -
Otherwise, user-defined top-level constant named ‘name.to_s.upcase`
-
Otherwise, default set with {smart_const}
-
Otherwise,
nil
@param name [#to_s] constant’s name
# File lib/evoker.rb, line 219 def smart_const_get(name) name = name.to_s.upcase if ENV.has_key?(name) ENV[name] elsif Object.const_defined?(name) Object.const_get(name) else @@SMART_CONST_DEFAULTS ||= {} @@SMART_CONST_DEFAULTS[name] end end
Check out Subversion repository
# File lib/evoker.rb, line 108 def subversion(name, opts={}) opts[:svn] ||= "svn" entity name do |t| cmd = "#{opts[:svn]}" cmd << " #{opts[:svn_args]}" if opts[:svn_args] cmd << " #{t.config[:svn_args]}" if t.config && t.config[:svn_args] cmd << " checkout -q" cmd << " #{opts[:checkout_args]}" if opts[:checkout_args] cmd << " #{t.config[:checkout_args]}" if t.config && t.config[:checkout_args] cmd << " -r #{opts[:revision]}" if opts[:revision] cmd << " -r #{t.config[:revision]}" if t.config && t.config[:revision] cmd << " #{opts[:url]}" if opts[:url] cmd << " #{t.config[:url]}" if t.config && t.config[:url] cmd << " #{t.name}" sh cmd end end
Entity that is a symlink to another path (FIXME:rename)
# File lib/evoker.rb, line 190 def symlink_(target, original, args={}) entity target => original do require 'pathname' original = Pathname.new(original.to_s).relative_path_from( Pathname.new(File.dirname(original.to_s))) ln_sf original.to_s, target.to_s end end
Download & unpack a tarball
# File lib/evoker.rb, line 164 def tarball(basename, options={}) tarball = wget options[:url], options[:wget_options]||{} entity basename => tarball do |t| dirname = options[:dirname] || File.basename(tarball.name, options[:ext] || '.tar.gz') rm_rf dirname sh "#{options[:decompress] || 'tar -xzf'} #{tarball}" ln_s dirname, basename unless options[:no_symlink] end end
Create Python virtual environment
# File lib/evoker/python.rb, line 11 def virtualenv(*args) if args.last.is_a? Hash opts = args.pop else opts = {} end if opts[:download_virtualenv] opts[:python] ||= smart_const_get(:python) opts[:virtualenv] = "#{opts[:python]} ./virtualenv.py" opts[:virtualenv_version] ||= smart_const_get(:virtualenv_version) opts[:virtualenv_url] ||= "http://github.com/pypa/virtualenv/raw/#{opts[:virtualenv_version]}/virtualenv.py" wget_virtualenv = wget opts[:virtualenv_url], :args => '--no-check-certificate', :no_entity => true CLOBBER.add(['virtualenv.pyc', 'setuptools-*.egg']) else opts[:virtualenv] ||= 'virtualenv' wget_virtualenv = nil end opts[:args] ||= nil virtualenv_command = "#{opts[:virtualenv]}" virtualenv_command << " #{opts[:args]}" if opts[:args] desc "Python virtual environment" venv = entity(*args) do |t| sh "#{virtualenv_command} #{t.name}" end task venv => wget_virtualenv if wget_virtualenv venv end
Create a symbolic link to virtualenv’s site-packages dir
# File lib/evoker/python.rb, line 47 def virtualenv_site_package(path, opts={}) opts[:target] ||= File.basename(path) opts[:virtualenv] ||= :python venv = Rake::Task[opts[:virtualenv]].name ln_sf File.join('..', '..', '..', '..', path), File.join(Dir["#{venv}/lib/python*/site-packages"].first, opts[:target]) end
Download a file using wget.
@param [#to_s] url address to download from @param [Hash] opts options @option opts [#to_s] :output_file (basename of ‘url`) name of target file @option opts [#to_s] :wget (’wget’) wget command to use @option opts [#to_s] :args (nil) custom command line arguments for wget @option opts [True, False] :no_entity (false)
do not add task to {Evoker::ENTITIES}
# File lib/evoker.rb, line 85 def wget(url, opts={}) opts[:output_file] ||= begin require 'uri' URI.parse(url).path.split('/').last end opts[:wget] ||= 'wget' wget_command = "#{opts[:wget]} -O #{opts[:output_file]}" wget_command << " #{opts[:args]}" if opts[:args] wget_command << " #{url} && touch #{opts[:output_file]}" CLOBBER.add(opts[:output_file]) ENTITIES.add(opts[:output_file]) unless opts[:no_entity] desc "Download #{url} as #{opts[:output_file]}" file opts[:output_file] do sh wget_command touch opts[:output_file] end end
Private Class Methods
Define smart constant’s default @param name [#to_s] constant’s name (will be upcased) @param default constant’s default value
# File lib/evoker.rb, line 205 def self.smart_const(name, default) @@SMART_CONST_DEFAULTS ||= {} @@SMART_CONST_DEFAULTS[name.to_s.upcase] = default end
Public Instance Methods
# File lib/evoker/s3cache.rb, line 8 def _get_bucket $s3 ||= Fog::Storage.new( :provider => "AWS", :aws_access_key_id => CACHE_S3_ACCESS_KEY_ID, :aws_secret_access_key => CACHE_S3_SECRET_ACCESS_KEY, :persistent => false) $bucket ||= $s3.directories.get(CACHE_S3_BUCKET) end
Cache result of a file task in local directory.
If cached ‘output_file` exists and matches the checksum (if one is given), task to copy file from cache directory to target is returned, and block is not executed.
If cached ‘output_file` exists but does not match the checksum, it is removed.
If ‘output file` does not exist or did not match the checksum, block is executed. Block should return a file task. This task will have extra code appended:
-
a checksum test: if checksum is given - error is raised if created file does not match the checksum
-
copying created file to cache directory
Cache directory is taken from a smart constant (see {#smart_const_get}) ‘:cache_path`, default is ’cache’.
@param [String] output_file File to uncache or create @param [String] checksum SHA-256 checksum of file (optional, but recommended) @yield Task to create file if not found in cache @return Task to uncache or create file
# File lib/evoker/local_cache.rb, line 38 def cached(output_file, checksum=nil) raise 'Block for Evoker::cached not provided' unless block_given? cached_path_elts = [] cached_path_elts << smart_const_get(:cache_path) cached_path_elts << checksum[0..1] if checksum cached_path_elts << checksum[2..3] if checksum cached_path_elts << File.basename(output_file) cached_path = File.join(*cached_path_elts) if File.exists?(cached_path) && checksum && Digest::SHA256.file(cached_path).hexdigest != checksum puts "WARN: checksum mismatch for cached #{File.basename(output_file)}, removing." FileUtils::rm cached_path end if File.exists?(cached_path) # Cached file exists and matches the given checksum rv = file output_file do FileUtils::cp cached_path, output_file end else # Cached file does not exist rv = yield output_file # Cache file after downloading task rv do if checksum && Digest::SHA256.file(output_file).hexdigest != checksum raise "Checksum mismatch for downloaded #{File.basename(output_file)}." end FileUtils::mkdir_p(File.dirname(cached_path)) FileUtils::cp output_file, cached_path end end CLEAN << output_file CLOBBER << cached_path rv end
Download a file using wget, or copy it from local cache
@param [#to_s] url address to download from @param [Hash] opts options (same as wget, + :checksum) @option opts [#to_s] :checksum sha256 sum of file to download
# File lib/evoker/local_cache.rb, line 87 def cached_wget(url, opts={}) opts[:output_file] ||= begin require 'uri' URI.parse(url).path.split('/').last end cached(opts[:output_file], opts[:checksum]) do wget url, opts end end
# File lib/evoker/fullstack.rb, line 103 def dl(filename) File.join(smart_const_get(:download_path), filename) end
# File lib/evoker/fullstack.rb, line 96 def download(url, args={}) args[:output_file] ||= File.expand_path(File.join( smart_const_get(:download_path), args[:filename] || File.basename(url))) cached_wget(url, args) end
Download tarball from given URL and unpack it for build.
A file from address ‘tarball_url` is downloaded (via {#cached_wget}) to directory specified in the `:download_path` smart constant, and then unpacked in directory specified in the `:build_path` smart constant.
The block is called in context that defines following methods:
-
‘tarball_filename`
-
‘tarball_path`
-
‘tarball_extension` (e.g. `“.tar.gz”`)
-
‘source_dir_basename`
-
‘source_dir`
-
‘download` (a Rake task that downloads the tarball)
-
‘unpack` (a Rake task that unpacks the tarball).
Block should define a task or chain of tasks, that compile (and possibly install, depending on needs) module contained in the tarball. First of the tasks should depend on ‘unpack`, and last should be returned from the block.
Task named ‘task_name` that depends on the task returned from the block will be created.
@example
from_tarball :carbon, CARBON_URL do file installed('bin/carbon-aggregator.py') => [ PIP, :py2cairo ] do rm_f source_file('setup.cfg') pip_install source_dir end end
# File lib/evoker/fullstack.rb, line 52 def from_tarball(task_name, tarball_url, args={}, &block) task task_name build_path = File.expand_path(smart_const_get(:build_path)) download_path = File.expand_path(smart_const_get(:download_path)) mkdir_p build_path unless File.directory?(build_path) mkdir_p download_path unless File.directory?(download_path) tarball_filename = args[:filename] || File.basename(tarball_url) tarball_path = File.join(download_path, tarball_filename) tarball_extension = args[:extension] || ( tarball_filename =~ /\.(tar(\.(Z|gz|bz2))?|zip)$/ ? $& : nil ) source_dir_basename = args[:directory] || File.basename(tarball_filename, tarball_extension) source_dir = File.join(build_path, source_dir_basename) unpack_command = args[:unpack] || { '.tar.gz' => 'tar -xzf', '.tar.bz2' => 'tar -xjf', '.tar.Z' => 'tar -xzf', '.tar' => 'tar -xf', '.zip' => 'unzip' }[tarball_extension.downcase] download = cached_wget( tarball_url, args.merge(:output_file => tarball_path)) unpack = file source_dir => download do chdir smart_const_get(:build_path) do rm_rf source_dir_basename sh "#{unpack_command} #{tarball_path}" end end ctx = FullStack::Context.new( tarball_filename, tarball_path, tarball_extension, source_dir_basename, source_dir, download, unpack) final_file = ctx.instance_eval(&block) task final_file => unpack task task_name => final_file end
Private Instance Methods
Base entity definition (wrapper over {EntityTask})
@param [#to_s] name name of task and directory @param *args arguments for {EntityTask#initialize} @yield [Rake::Task] block executed to populate target directory @return [EntityTask] defined task
# File lib/evoker.rb, line 71 def entity(name, *args, &block) Evoker::EntityTask.define_task(name, *args, &block) end
Check out Git repository
# File lib/evoker.rb, line 128 def git(name, opts={}) opts[:git] ||= "git" entity name do |t| cmd = "#{opts[:git]} clone" cmd << " #{opts[:clone_args]}" if opts[:clone_args] cmd << " #{t.config[:clone_args]}" if t.config && t.config[:clone_args] cmd << " #{opts[:url]}" if opts[:url] cmd << " #{t.config[:url]}" if t.config && t.config[:url] cmd << " #{t.name}" if rev = opts[:revision] || ( t.config && t.config[:revision] ) cmd << " && cd #{t.name}" \ " && #{opts[:git]} checkout -b evoker-checkout #{rev}" end sh cmd end end
Check out Mercurial repository
# File lib/evoker.rb, line 148 def mercurial(name, opts={}) opts[:hg] ||= "hg" entity name do |t| cmd = "#{opts[:hg]} clone" cmd << " #{args}" if args = opts[:clone_args] || ( t.config && t.config[:clone_args] ) cmd << " -r #{opts[:revision]}" if opts[:revision] cmd << " -r #{t.config[:revision]}" if t.config && t.config[:revision] cmd << " #{opts[:url]}" if opts[:url] cmd << " #{t.config[:url]}" if t.config && t.config[:url] cmd << " #{t.name}" sh cmd end end
Apply patch to an entity
# File lib/evoker.rb, line 176 def patch(entity_name, patches, patch_args=nil) task entity_name => patches do |t| patches = [ patches ] unless patches.respond_to?(:each) cmd = "set -e -x\ncd #{t.name}\n" patches.each do |patch| cmd << "patch #{patch_args} < ../#{patch}\n" end sh cmd end end
Download Python requirements using pip
# File lib/evoker/python.rb, line 58 def pip_requirements(file, args={}) stampfile = "#{file}.stamp" if args[:virtualenv] args[:pip] = "#{args[:virtualenv]}/bin/pip" else args[:pip] ||= smart_const_get(:pip) end pip_cmd = "#{args[:pip]}" pip_cmd << " #{args[:args]}" if args[:args] pip_cmd << " install" pip_cmd << " #{args[:install_args]}" if args[:install_args] pip_cmd << " -r #{file}" t = file stampfile => file do sh pip_cmd File.open(stampfile, 'w') { |f| f.write(DateTime::now.to_s) } end task t => args[:virtualenv] if args[:virtualenv] CLOBBER.add t.name ENTITIES.add t.name t end
Get smart constant’s effective value
Effective value is:
-
name.to_s.upcase
environment variable, if present -
Otherwise, user-defined top-level constant named ‘name.to_s.upcase`
-
Otherwise, default set with {smart_const}
-
Otherwise,
nil
@param name [#to_s] constant’s name
# File lib/evoker.rb, line 219 def smart_const_get(name) name = name.to_s.upcase if ENV.has_key?(name) ENV[name] elsif Object.const_defined?(name) Object.const_get(name) else @@SMART_CONST_DEFAULTS ||= {} @@SMART_CONST_DEFAULTS[name] end end
Check out Subversion repository
# File lib/evoker.rb, line 108 def subversion(name, opts={}) opts[:svn] ||= "svn" entity name do |t| cmd = "#{opts[:svn]}" cmd << " #{opts[:svn_args]}" if opts[:svn_args] cmd << " #{t.config[:svn_args]}" if t.config && t.config[:svn_args] cmd << " checkout -q" cmd << " #{opts[:checkout_args]}" if opts[:checkout_args] cmd << " #{t.config[:checkout_args]}" if t.config && t.config[:checkout_args] cmd << " -r #{opts[:revision]}" if opts[:revision] cmd << " -r #{t.config[:revision]}" if t.config && t.config[:revision] cmd << " #{opts[:url]}" if opts[:url] cmd << " #{t.config[:url]}" if t.config && t.config[:url] cmd << " #{t.name}" sh cmd end end
Entity that is a symlink to another path (FIXME:rename)
# File lib/evoker.rb, line 190 def symlink_(target, original, args={}) entity target => original do require 'pathname' original = Pathname.new(original.to_s).relative_path_from( Pathname.new(File.dirname(original.to_s))) ln_sf original.to_s, target.to_s end end
Download & unpack a tarball
# File lib/evoker.rb, line 164 def tarball(basename, options={}) tarball = wget options[:url], options[:wget_options]||{} entity basename => tarball do |t| dirname = options[:dirname] || File.basename(tarball.name, options[:ext] || '.tar.gz') rm_rf dirname sh "#{options[:decompress] || 'tar -xzf'} #{tarball}" ln_s dirname, basename unless options[:no_symlink] end end
Create Python virtual environment
# File lib/evoker/python.rb, line 11 def virtualenv(*args) if args.last.is_a? Hash opts = args.pop else opts = {} end if opts[:download_virtualenv] opts[:python] ||= smart_const_get(:python) opts[:virtualenv] = "#{opts[:python]} ./virtualenv.py" opts[:virtualenv_version] ||= smart_const_get(:virtualenv_version) opts[:virtualenv_url] ||= "http://github.com/pypa/virtualenv/raw/#{opts[:virtualenv_version]}/virtualenv.py" wget_virtualenv = wget opts[:virtualenv_url], :args => '--no-check-certificate', :no_entity => true CLOBBER.add(['virtualenv.pyc', 'setuptools-*.egg']) else opts[:virtualenv] ||= 'virtualenv' wget_virtualenv = nil end opts[:args] ||= nil virtualenv_command = "#{opts[:virtualenv]}" virtualenv_command << " #{opts[:args]}" if opts[:args] desc "Python virtual environment" venv = entity(*args) do |t| sh "#{virtualenv_command} #{t.name}" end task venv => wget_virtualenv if wget_virtualenv venv end
Create a symbolic link to virtualenv’s site-packages dir
# File lib/evoker/python.rb, line 47 def virtualenv_site_package(path, opts={}) opts[:target] ||= File.basename(path) opts[:virtualenv] ||= :python venv = Rake::Task[opts[:virtualenv]].name ln_sf File.join('..', '..', '..', '..', path), File.join(Dir["#{venv}/lib/python*/site-packages"].first, opts[:target]) end
Download a file using wget.
@param [#to_s] url address to download from @param [Hash] opts options @option opts [#to_s] :output_file (basename of ‘url`) name of target file @option opts [#to_s] :wget (’wget’) wget command to use @option opts [#to_s] :args (nil) custom command line arguments for wget @option opts [True, False] :no_entity (false)
do not add task to {Evoker::ENTITIES}
# File lib/evoker.rb, line 85 def wget(url, opts={}) opts[:output_file] ||= begin require 'uri' URI.parse(url).path.split('/').last end opts[:wget] ||= 'wget' wget_command = "#{opts[:wget]} -O #{opts[:output_file]}" wget_command << " #{opts[:args]}" if opts[:args] wget_command << " #{url} && touch #{opts[:output_file]}" CLOBBER.add(opts[:output_file]) ENTITIES.add(opts[:output_file]) unless opts[:no_entity] desc "Download #{url} as #{opts[:output_file]}" file opts[:output_file] do sh wget_command touch opts[:output_file] end end