class BranchableCDNAssets::FileManager

Attributes

branch[R]
config[R]
manifest[R]
root[R]

Public Class Methods

new(config, branch=nil) click to toggle source

handles moving files to/from our various cdn locations

# File lib/branchable_cdn_assets/file_manager.rb, line 17
def initialize config, branch=nil
  @config   = config
  @root     = config.cdn_dir
  @branch   = branch || config.branch
  @manifest = Manifest.new( File.join( @root, "#{@branch.gsub('/','_')}.manifest" ) )
end

Public Instance Methods

doctor() click to toggle source

x files were not on every remote remote-x (x files):

  • file/path

  • file/path

remote-y (x files):

  • path/to/file

  • path

# File lib/branchable_cdn_assets/file_manager.rb, line 133
def doctor
  db    = uniq_files_per_remote
  count = config.remotes.flat_map { |r| db[r.host].length }.inject(0, :+)

  if count > 0
    puts "#{count} files were not on every remote"
    config.remotes.each do |remote|
      puts "#{remote.host} (#{db[remote.host].length} files)"
      if db[remote.host].length > 0
        db[remote.host].each { |f| puts "  + #{f}" }
      end
    end
  else
    puts "all good, content is the same on all remotes"
  end

  return nil
end
find(file, location=:all) click to toggle source
# File lib/branchable_cdn_assets/file_manager.rb, line 109
def find file, location=:all
  if [:all, :local].include?(location) && list(:local).include?(file)
    return :local
  elsif [:all, :remote].include?(location) && list(:remote).include?(file)
    return File.join(config.url, file)
  end

  if config.env != :production && location != :local
    production_config   = Config.new( config.raw_data, config.production_branch )
    production_manifest = Manifest.new( File.join( root, "#{config.production_branch}.manifest" ) )

    return File.join( production_config.url, file ) if production_manifest.files.include?(file)
  end
end
heal() click to toggle source
# File lib/branchable_cdn_assets/file_manager.rb, line 152
def heal
  db       = uniq_files_per_remote
  all_uniq = []
  config.remotes.each do |remote|
    all_uniq.push *db[remote.host]
    list_file = create_list_file db[remote.host].join("\n")
    pull_list = map_rsync_output_to_list rsync_files_from_remote(list_file, remote)
    list_file.unlink

    puts "#{pull_list.length} files pulled from #{remote.host}"
  end

  config.remotes.each do |remote|
    list_file = create_list_file ( all_uniq - db[remote.host] ).join("\n")
    push_list = map_rsync_output_to_list rsync_files_to_remote( list_file, remote )
    list_file.unlink

    puts "#{push_list.length} files pushed to #{remote.host}"
  end

  nil
end
list(where=:all) click to toggle source

@param where [Symbol] :local, :remote, :all (default) @return [Array]

# File lib/branchable_cdn_assets/file_manager.rb, line 26
def list where=:all
  case where
  when :local  then return list_local_files
  when :remote then return list_remote_files
  when :both   then return list_local_files & list_remote_files
  else return list_local_files + list_remote_files
  end
end
move_to_production(branch) click to toggle source

move the current cdn files to the production cdn guarded to only run from prod

# File lib/branchable_cdn_assets/file_manager.rb, line 91
def move_to_production branch
  branch_cdn = FileManager.new( Config.new( config.raw_data, branch ) )
  if branch_cdn.manifest.files.empty?
    puts "no files to move from #{branch}"
    abort
  end
  branch_cdn.pull!
  push!
end
prune!() click to toggle source

remove local files that exist on the remote @return [Array]

# File lib/branchable_cdn_assets/file_manager.rb, line 76
def prune!
  puts "files to be removed:\n#{list(:both).join("\n")}"
  abort unless Shell.get_input("do you want to continue? (y|n) ") == "y"

  list(:both).each do |f|
    Shell.run_local "rm #{File.join( root, f )}"
  end

  remove_empty_directories
  return nil
end
pull!() click to toggle source

destructive pull, removes references to remote files once they're pulled @return [Array] the pulled files

# File lib/branchable_cdn_assets/file_manager.rb, line 47
def pull!
  pull_list = pull_all
  @manifest.remove_files( pull_list )
  @manifest.update_source_file!

  return pull_list
end
push!() click to toggle source

destructive push, removes the local versions of the pushed files and adds them to the manifest @return [Array] the pushed files

# File lib/branchable_cdn_assets/file_manager.rb, line 59
def push!
  setup_remotes

  push_list = push
  if config.env.to_sym == :production
    invalidate( push_list )
  end

  @manifest.merge_files( push_list )
  @manifest.update_source_file!

  return push_list
end
remove_empty_directories() click to toggle source
# File lib/branchable_cdn_assets/file_manager.rb, line 102
def remove_empty_directories
  empty_directories.each do |dir|
    Dir.rmdir( dir )
  end
end
setup() click to toggle source

make cdn dir on the remotes with given permissions

# File lib/branchable_cdn_assets/file_manager.rb, line 37
def setup
  setup_remotes

  return nil
end

Private Instance Methods

create_list_file(contents) click to toggle source

creates a temp file with contens and returns it's path @param contents [String] content for the file @return [String] file path

# File lib/branchable_cdn_assets/file_manager.rb, line 339
def create_list_file contents
  file = Tempfile.new( "list_file" )
  file.write contents
  file.close

  return file
end
empty_directories() click to toggle source
# File lib/branchable_cdn_assets/file_manager.rb, line 251
def empty_directories
  directories = Dir[ File.join( Dir.pwd, root, '**/*' ) ].select do |d|
    File.directory?(d)
  end

  empty_directories = directories.select do |d|
    # a directory is empty if it only includes other empty directories
    (( Dir.entries(d) - ['.','..'] ).map { |entry| File.join(d, entry) } - directories ).empty?
  end

  # reversing to ensure that nested directories are listed first
  # so on removal we don't try and remove upper directories *then*
  # the nested directories that aren't there anymore
  empty_directories.sort_by { |i| i.length }.reverse
end
ensure_local_file_permissions() click to toggle source
# File lib/branchable_cdn_assets/file_manager.rb, line 297
def ensure_local_file_permissions
  list_local_files.each do |file|
    Shell.run_local "chmod 644 #{File.join( root, file )}"
  end
end
files_per_remote() click to toggle source
# File lib/branchable_cdn_assets/file_manager.rb, line 198
def files_per_remote
  config.remotes.each_with_object({}) do |remote, collection|
    resp = Shell.run_remote "find #{remote.root} -type f", hostname: remote.host
    if resp.success?
      collection[remote.host] = resp.stdout.split("\n")
                                    .map { |f| f.sub /^#{Regexp.escape(remote.root)}\/?/, '' }
    else
      warn "problem pulling file list for #{remote.host}\n#{resp.stderr}"
    end
  end
end
invalidate(files) click to toggle source
# File lib/branchable_cdn_assets/file_manager.rb, line 225
def invalidate files
  if config.invalidator
    to_invalidate = ( files & list(:local) ).map do |f|
      if config.invalidator.config.respond_to? :path_prefix
        File.join( config.invalidator.config.path_prefix, f )
      else
        f
      end
    end

    config.invalidator.invalidate_files( to_invalidate )
  end
end
list_local_files() click to toggle source
# File lib/branchable_cdn_assets/file_manager.rb, line 357
def list_local_files
  Dir[ File.join( Dir.pwd, root, '**/*' ) ].keep_if do |f|
    !f.match(/\.manifest$/) && !File.directory?(f)
  end.map do |f|
    f.sub( "#{File.join( Dir.pwd, root )}/", "" )
  end.select do |f|
    if config.file_filter
      config.file_filter.call(f)
    else
      true
    end
  end
end
list_remote_files() click to toggle source
# File lib/branchable_cdn_assets/file_manager.rb, line 371
def list_remote_files
  manifest.files
end
map_rsync_output_to_list(output) click to toggle source

@param output [String] @return [Array]

# File lib/branchable_cdn_assets/file_manager.rb, line 349
def map_rsync_output_to_list output
  return output.split("\n").keep_if do |f|
    f.match( /^(?:<|>)f/ ) && !f.match(/\/$/ )
  end.map do |f|
    f.split()[1]
  end
end
pull() click to toggle source

Take files from the cdn and move them locally @return [Array] list of files pulled

# File lib/branchable_cdn_assets/file_manager.rb, line 305
def pull
  # create list for rsync
  list_file = create_list_file( list(:remote).join("\n") )
  pull_list = map_rsync_output_to_list rsync_files_from_remote(list_file)
  list_file.unlink

  puts "#{pull_list.length} files pulled from #{branch}"
  return pull_list
end
pull_all() click to toggle source
# File lib/branchable_cdn_assets/file_manager.rb, line 177
def pull_all
  db      = uniq_files_per_remote
  pulled  = []
  config.remotes.each_with_index do |remote, idx|
    if idx == 0
      file_list = db[remote.host] + db["all"]
    else
      file_list = db[remote.host]
    end

    list_file = create_list_file file_list.join("\n")
    pull_list = map_rsync_output_to_list rsync_files_from_remote( list_file, remote )
    list_file.unlink

    pulled.push *pull_list
    puts "#{pull_list.length} files pulled from #{remote.host}"
  end

  return pulled
end
push() click to toggle source

@return [Array] the pushed files (only those updated on remote)

# File lib/branchable_cdn_assets/file_manager.rb, line 268
def push
  list_file = create_list_file( list(:local).join("\n") )
  ensure_local_file_permissions

  push_list = config.remotes.flat_map do |remote|
    map_rsync_output_to_list( rsync_files_to_remote(list_file, remote) )
  end.uniq

  list_file.unlink
  push_list = resolve_conflicts(push_list)
  puts "#{push_list.length} files pushed to #{branch}"
  return push_list
end
resolve_conflicts(push_list) click to toggle source

@return [Array] resolved list

# File lib/branchable_cdn_assets/file_manager.rb, line 324
def resolve_conflicts push_list
  conflicts = list(:local) - push_list
  return push_list if conflicts.empty?

  puts "these local files were not pushed by rsync:"
  puts conflicts.join("\n")
  if Shell.get_input("add to the #{branch} manifest? (y|n) ") == "y"
    push_list.push(*conflicts)
  end
  return push_list
end
rsync_files_from_remote(list_file, remote) click to toggle source
# File lib/branchable_cdn_assets/file_manager.rb, line 315
def rsync_files_from_remote list_file, remote
  resp = Shell.run_local "rsync #{config.rsync_flags} --files-from=#{list_file.path}" +
                         " -e ssh #{remote.host}:#{remote.root} #{root}/"

  raise "rsync failed\n#{resp.stderr}" unless resp.success?
  return resp.stdout
end
rsync_files_to_remote(list_file, remote) click to toggle source
# File lib/branchable_cdn_assets/file_manager.rb, line 282
def rsync_files_to_remote list_file, remote
  dir_perms  = Permissions.new( remote.dir_permissions ).to_chmod
  file_perms = Permissions.new( remote.file_permissions ).to_chmod
  file_permissions = "--chmod=" +
                     "Du=#{dir_perms.user},Dg=#{dir_perms.group},Do=#{dir_perms.others}," +
                     "Fu=#{file_perms.user},Fg=#{file_perms.group},Fo=#{file_perms.others}"


  resp = Shell.run_local "rsync #{remote.rsync_flags} #{file_permissions} --files-from=#{list_file.path} " +
                         "-e ssh #{root}/ #{remote.host}:#{remote.root}"

  raise "rsync failed for #{remote.host}\n#{resp.stderr}" unless resp.success?
  resp.stdout
end
setup_remotes() click to toggle source

create the root dir on remote

# File lib/branchable_cdn_assets/file_manager.rb, line 240
def setup_remotes
  config.remotes.each do |remote|
    resp = Shell.run_remote "mkdir -p -m #{remote.dir_permissions} #{remote.root}", hostname: remote.host
    unless resp.success?
      raise "problem setting up #{remote.host}\n#{resp.stderr}"
    else
      puts "set up #{remote.root} on #{remote.host}"
    end
  end
end
uniq_files_per_remote() click to toggle source
# File lib/branchable_cdn_assets/file_manager.rb, line 210
def uniq_files_per_remote
  files_by_remote = files_per_remote
  files_on_all    = []
  files_by_remote.each_with_index do |(remote, files), idx|
    if idx == 0
      files_on_all = files
    else
      files_on_all = files_on_all & files
    end
  end

  Hash[ files_by_remote.map { |r, files| [r, (files - files_on_all)] }
                       .push ["all", files_on_all] ]
end