class Elasticrawl::Config
Represents the current configuration which is persisted to ~/.elasticrawl/ and contains 3 configuration files.
aws.yml - AWS access credentials unless stored in the environment
variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.
cluster.yml - Elastic MapReduce cluster config including instance groups. jobs.yml - Elastic MapReduce jobs config and the S3 bucket used for
storing data and logs.
This directory also contains the Elasticrawl
SQLite database.
Constants
- CONFIG_DIR
- DATABASE_FILE
- TEMPLATES_DIR
- TEMPLATE_FILES
Attributes
Public Class Methods
Sets the AWS access credentials needed for the S3 and EMR API calls.
# File lib/elasticrawl/config.rb, line 22 def initialize(access_key_id = nil, secret_access_key = nil) # Credentials have been provided to the init command. @access_key_id = access_key_id @secret_access_key = secret_access_key # If credentials are not set then check if they are available in aws.yml. if dir_exists? config = load_config('aws') key = config['access_key_id'] secret = config['secret_access_key'] @access_key_id ||= key unless key == 'ACCESS_KEY_ID' @secret_access_key ||= secret unless secret == 'SECRET_ACCESS_KEY' end # If credentials are still not set then check AWS environment variables. @access_key_id ||= ENV['AWS_ACCESS_KEY_ID'] @secret_access_key ||= ENV['AWS_SECRET_ACCESS_KEY'] # Set AWS credentials for use when accessing the S3 API. AWS.config(:access_key_id => @access_key_id, :secret_access_key => @secret_access_key) end
Public Instance Methods
Displayed by init command.
# File lib/elasticrawl/config.rb, line 139 def access_key_prompt prompt = "Enter AWS Access Key ID:" prompt += " [#{@access_key_id}]" if @access_key_id.present? prompt end
Checks if a S3 bucket name is in use.
# File lib/elasticrawl/config.rb, line 96 def bucket_exists?(bucket_name) begin s3 = AWS::S3.new s3.buckets[bucket_name].exists? rescue AWS::S3::Errors::SignatureDoesNotMatch => e raise AWSCredentialsInvalidError, 'AWS access credentials are invalid' rescue AWS::Errors::Base => s3e raise S3AccessError.new(s3e.http_response), s3e.message end end
Returns the location of the config directory.
# File lib/elasticrawl/config.rb, line 47 def config_dir File.join(Dir.home, CONFIG_DIR) end
Creates the S3 bucket and config directory. Deploys the config templates and creates the sqlite database.
# File lib/elasticrawl/config.rb, line 110 def create(bucket_name) create_bucket(bucket_name) deploy_templates(bucket_name) load_database status_message(bucket_name, 'created') end
Deletes the S3 bucket and config directory.
# File lib/elasticrawl/config.rb, line 119 def delete bucket_name = load_config('jobs')['s3_bucket_name'] delete_bucket(bucket_name) delete_config_dir status_message(bucket_name, 'deleted') end
Displayed by destroy command to confirm deletion.
# File lib/elasticrawl/config.rb, line 128 def delete_warning bucket_name = load_config('jobs')['s3_bucket_name'] message = ['WARNING:'] message << "Bucket s3://#{bucket_name} and its data will be deleted" message << "Config dir #{config_dir} will be deleted" message.join("\n") end
Checks if the configuration directory exists.
# File lib/elasticrawl/config.rb, line 52 def dir_exists? Dir.exists?(config_dir) end
Loads a YAML configuration file.
# File lib/elasticrawl/config.rb, line 57 def load_config(config_file) if dir_exists? begin config_file = File.join(config_dir, "#{config_file}.yml") config = YAML::load(File.open(config_file)) rescue StandardError => e raise FileAccessError, e.message end else raise ConfigDirMissingError, 'Config dir missing. Run init command' end end
Loads the sqlite database. If no database exists it will be created and the database migrations will be run.
# File lib/elasticrawl/config.rb, line 73 def load_database if dir_exists? config = { 'adapter' => 'sqlite3', 'database' => File.join(config_dir, DATABASE_FILE), 'pool' => 5, 'timeout' => 5000 } begin ActiveRecord::Base.establish_connection(config) ActiveRecord::Migrator.migrate(File.join(File.dirname(__FILE__), \ '../../db/migrate'), ENV['VERSION'] ? ENV['VERSION'].to_i : nil ) rescue StandardError => e raise DatabaseAccessError, e.message end else raise ConfigDirMissingError, 'Config dir missing. Run init command' end end
Displayed by init command.
# File lib/elasticrawl/config.rb, line 147 def secret_key_prompt prompt = "Enter AWS Secret Access Key:" prompt += " [#{@secret_access_key}]" if @secret_access_key.present? prompt end
Private Instance Methods
Creates a bucket using the S3 API.
# File lib/elasticrawl/config.rb, line 156 def create_bucket(bucket_name) begin s3 = AWS::S3.new s3.buckets.create(bucket_name) rescue AWS::Errors::Base => s3e raise S3AccessError.new(s3e.http_response), s3e.message end end
Deletes a bucket and its contents using the S3 API.
# File lib/elasticrawl/config.rb, line 167 def delete_bucket(bucket_name) begin s3 = AWS::S3.new bucket = s3.buckets[bucket_name] bucket.delete! rescue AWS::Errors::Base => s3e raise S3AccessError.new(s3e.http_response), s3e.message end end
Deletes the config directory including its contents.
# File lib/elasticrawl/config.rb, line 222 def delete_config_dir begin FileUtils.rm_r(config_dir) if dir_exists? rescue StandardError => e raise FileAccessError, e.message end end
Creates config directory and copies config templates into it. Saves S3 bucket name to jobs.yml and AWS credentials to aws.yml.
# File lib/elasticrawl/config.rb, line 180 def deploy_templates(bucket_name) begin Dir.mkdir(config_dir, 0755) if dir_exists? == false TEMPLATE_FILES.each do |template_file| FileUtils.cp(File.join(File.dirname(__FILE__), TEMPLATES_DIR, template_file), File.join(config_dir, template_file)) end save_config('jobs', { 'BUCKET_NAME' => bucket_name }) save_aws_config rescue StandardError => e raise FileAccessError, e.message end end
Saves AWS access credentials to aws.yml unless they are configured as environment variables.
# File lib/elasticrawl/config.rb, line 199 def save_aws_config env_key = ENV['AWS_ACCESS_KEY_ID'] env_secret = ENV['AWS_SECRET_ACCESS_KEY'] creds = {} creds['ACCESS_KEY_ID'] = @access_key_id unless @access_key_id == env_key creds['SECRET_ACCESS_KEY'] = @secret_access_key \ unless @secret_access_key == env_secret save_config('aws', creds) end
Saves config values by overwriting placeholder values in template.
# File lib/elasticrawl/config.rb, line 212 def save_config(template, params) config_file = File.join(config_dir, "#{template}.yml") config = File.read(config_file) params.map { |key, value| config = config.gsub(key, value) } File.open(config_file, 'w') { |file| file.write(config) } end
Notifies user of results of init or destroy commands.
# File lib/elasticrawl/config.rb, line 232 def status_message(bucket_name, state) message = ['', "Bucket s3://#{bucket_name} #{state}"] message << "Config dir #{config_dir} #{state}" state = 'complete' if state == 'created' message << "Config #{state}" message.join("\n") end