module S3MetaSync
Constants
- CORRUPT_FILES_LOG
- META_FILE
- RemoteCorrupt
- RemoteWithoutMeta
- VERSION
Public Class Methods
parse_options(argv)
click to toggle source
# File lib/s3_meta_sync.rb, line 20 def parse_options(argv) options = { key: ENV["AWS_ACCESS_KEY_ID"], secret: ENV["AWS_SECRET_ACCESS_KEY"], zip: false, } OptionParser.new do |opts| opts.banner = <<-BANNER.gsub(/^ {10}/, "") Sync folders with s3 using a metadata file with md5 sums. # upload local files and remove everything that is not local s3-meta-sync <local> <bucket:folder> --key <aws-access-key> --secret <aws-secret-key> # download files and remove everything that is not remote s3-meta-sync <bucket:folder> <local> # no credentials required Key and secret can also be supplied using AWS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY Options: BANNER opts.on("-k", "--key KEY", "AWS access key") { |c| options[:key] = c } opts.on("-s", "--secret SECRET", "AWS secret key") { |c| options[:secret] = c } opts.on("-r", "--region REGION", "AWS region if not us-standard") { |c| options[:region] = c } opts.on("-p", "--parallel COUNT", Integer, "Use COUNT threads for download/upload default: 10") { |c| options[:parallel] = c } opts.on("-o", "--open-timeout TIMEOUT", Integer, "Net::HTTP open timeout in seconds default: 5") { |c| options[:open_timeout] = c } opts.on("-t", "--read-timeout TIMEOUT", Integer, "Net::HTTP read timeout in seconds default: 10") { |c| options[:read_timeout] = c } opts.on("--ssl-none", "Do not verify ssl certs") { options[:ssl_none] = true } opts.on("-z", "--zip", "Zip when uploading to save bandwidth") { options[:zip] = true } opts.on("--no-local-changes", "Do not md5 all the local files, they did not change") { options[:no_local_changes] = true } opts.on("--retries MAX", Integer, "MAX number of times retrying failed http requests default: 2") { |c| options[:max_retries] = c } opts.on("-V", "--verbose", "Verbose mode"){ options[:verbose] = true } opts.on("-h", "--help", "Show this.") { puts opts; exit } opts.on("-v", "--version", "Show Version") { puts VERSION; exit} end.parse!(argv) raise "need source and destination" unless argv.size == 2 raise "need 1 local and 1 remote" unless argv.select { |a| a.include?(":") }.size == 1 raise "need credentials --key + --secret" if argv.last.include?(":") and (not options[:key] or not options[:secret]) [*argv, options] end
run(argv)
click to toggle source
# File lib/s3_meta_sync.rb, line 14 def run(argv) source, dest, options = parse_options(argv) Syncer.new(options).sync(source, dest) 0 end