class BranchableCDNAssets::FileManager
Attributes
Public Class Methods
handles moving files to/from our various cdn locations
# File lib/branchable_cdn_assets/file_manager.rb, line 17 def initialize config, branch=nil @config = config @root = config.cdn_dir @branch = branch || config.branch @manifest = Manifest.new( File.join( @root, "#{@branch.gsub('/','_')}.manifest" ) ) end
Public Instance Methods
x files were not on every remote remote-x (x files):
-
file/path
-
file/path
remote-y (x files):
-
path/to/file
-
path
# File lib/branchable_cdn_assets/file_manager.rb, line 133 def doctor db = uniq_files_per_remote count = config.remotes.flat_map { |r| db[r.host].length }.inject(0, :+) if count > 0 puts "#{count} files were not on every remote" config.remotes.each do |remote| puts "#{remote.host} (#{db[remote.host].length} files)" if db[remote.host].length > 0 db[remote.host].each { |f| puts " + #{f}" } end end else puts "all good, content is the same on all remotes" end return nil end
# File lib/branchable_cdn_assets/file_manager.rb, line 109 def find file, location=:all if [:all, :local].include?(location) && list(:local).include?(file) return :local elsif [:all, :remote].include?(location) && list(:remote).include?(file) return File.join(config.url, file) end if config.env != :production && location != :local production_config = Config.new( config.raw_data, config.production_branch ) production_manifest = Manifest.new( File.join( root, "#{config.production_branch}.manifest" ) ) return File.join( production_config.url, file ) if production_manifest.files.include?(file) end end
# File lib/branchable_cdn_assets/file_manager.rb, line 152 def heal db = uniq_files_per_remote all_uniq = [] config.remotes.each do |remote| all_uniq.push *db[remote.host] list_file = create_list_file db[remote.host].join("\n") pull_list = map_rsync_output_to_list rsync_files_from_remote(list_file, remote) list_file.unlink puts "#{pull_list.length} files pulled from #{remote.host}" end config.remotes.each do |remote| list_file = create_list_file ( all_uniq - db[remote.host] ).join("\n") push_list = map_rsync_output_to_list rsync_files_to_remote( list_file, remote ) list_file.unlink puts "#{push_list.length} files pushed to #{remote.host}" end nil end
@param where [Symbol] :local, :remote, :all (default) @return [Array]
# File lib/branchable_cdn_assets/file_manager.rb, line 26 def list where=:all case where when :local then return list_local_files when :remote then return list_remote_files when :both then return list_local_files & list_remote_files else return list_local_files + list_remote_files end end
move the current cdn files to the production cdn guarded to only run from prod
# File lib/branchable_cdn_assets/file_manager.rb, line 91 def move_to_production branch branch_cdn = FileManager.new( Config.new( config.raw_data, branch ) ) if branch_cdn.manifest.files.empty? puts "no files to move from #{branch}" abort end branch_cdn.pull! push! end
remove local files that exist on the remote @return [Array]
# File lib/branchable_cdn_assets/file_manager.rb, line 76 def prune! puts "files to be removed:\n#{list(:both).join("\n")}" abort unless Shell.get_input("do you want to continue? (y|n) ") == "y" list(:both).each do |f| Shell.run_local "rm #{File.join( root, f )}" end remove_empty_directories return nil end
destructive pull, removes references to remote files once they're pulled @return [Array] the pulled files
# File lib/branchable_cdn_assets/file_manager.rb, line 47 def pull! pull_list = pull_all @manifest.remove_files( pull_list ) @manifest.update_source_file! return pull_list end
destructive push, removes the local versions of the pushed files and adds them to the manifest @return [Array] the pushed files
# File lib/branchable_cdn_assets/file_manager.rb, line 59 def push! setup_remotes push_list = push if config.env.to_sym == :production invalidate( push_list ) end @manifest.merge_files( push_list ) @manifest.update_source_file! return push_list end
# File lib/branchable_cdn_assets/file_manager.rb, line 102 def remove_empty_directories empty_directories.each do |dir| Dir.rmdir( dir ) end end
make cdn dir on the remotes with given permissions
# File lib/branchable_cdn_assets/file_manager.rb, line 37 def setup setup_remotes return nil end
Private Instance Methods
creates a temp file with contens and returns it's path @param contents [String] content for the file @return [String] file path
# File lib/branchable_cdn_assets/file_manager.rb, line 339 def create_list_file contents file = Tempfile.new( "list_file" ) file.write contents file.close return file end
# File lib/branchable_cdn_assets/file_manager.rb, line 251 def empty_directories directories = Dir[ File.join( Dir.pwd, root, '**/*' ) ].select do |d| File.directory?(d) end empty_directories = directories.select do |d| # a directory is empty if it only includes other empty directories (( Dir.entries(d) - ['.','..'] ).map { |entry| File.join(d, entry) } - directories ).empty? end # reversing to ensure that nested directories are listed first # so on removal we don't try and remove upper directories *then* # the nested directories that aren't there anymore empty_directories.sort_by { |i| i.length }.reverse end
# File lib/branchable_cdn_assets/file_manager.rb, line 297 def ensure_local_file_permissions list_local_files.each do |file| Shell.run_local "chmod 644 #{File.join( root, file )}" end end
# File lib/branchable_cdn_assets/file_manager.rb, line 198 def files_per_remote config.remotes.each_with_object({}) do |remote, collection| resp = Shell.run_remote "find #{remote.root} -type f", hostname: remote.host if resp.success? collection[remote.host] = resp.stdout.split("\n") .map { |f| f.sub /^#{Regexp.escape(remote.root)}\/?/, '' } else warn "problem pulling file list for #{remote.host}\n#{resp.stderr}" end end end
# File lib/branchable_cdn_assets/file_manager.rb, line 225 def invalidate files if config.invalidator to_invalidate = ( files & list(:local) ).map do |f| if config.invalidator.config.respond_to? :path_prefix File.join( config.invalidator.config.path_prefix, f ) else f end end config.invalidator.invalidate_files( to_invalidate ) end end
# File lib/branchable_cdn_assets/file_manager.rb, line 357 def list_local_files Dir[ File.join( Dir.pwd, root, '**/*' ) ].keep_if do |f| !f.match(/\.manifest$/) && !File.directory?(f) end.map do |f| f.sub( "#{File.join( Dir.pwd, root )}/", "" ) end.select do |f| if config.file_filter config.file_filter.call(f) else true end end end
# File lib/branchable_cdn_assets/file_manager.rb, line 371 def list_remote_files manifest.files end
@param output [String] @return [Array]
# File lib/branchable_cdn_assets/file_manager.rb, line 349 def map_rsync_output_to_list output return output.split("\n").keep_if do |f| f.match( /^(?:<|>)f/ ) && !f.match(/\/$/ ) end.map do |f| f.split()[1] end end
Take files from the cdn and move them locally @return [Array] list of files pulled
# File lib/branchable_cdn_assets/file_manager.rb, line 305 def pull # create list for rsync list_file = create_list_file( list(:remote).join("\n") ) pull_list = map_rsync_output_to_list rsync_files_from_remote(list_file) list_file.unlink puts "#{pull_list.length} files pulled from #{branch}" return pull_list end
# File lib/branchable_cdn_assets/file_manager.rb, line 177 def pull_all db = uniq_files_per_remote pulled = [] config.remotes.each_with_index do |remote, idx| if idx == 0 file_list = db[remote.host] + db["all"] else file_list = db[remote.host] end list_file = create_list_file file_list.join("\n") pull_list = map_rsync_output_to_list rsync_files_from_remote( list_file, remote ) list_file.unlink pulled.push *pull_list puts "#{pull_list.length} files pulled from #{remote.host}" end return pulled end
@return [Array] the pushed files (only those updated on remote)
# File lib/branchable_cdn_assets/file_manager.rb, line 268 def push list_file = create_list_file( list(:local).join("\n") ) ensure_local_file_permissions push_list = config.remotes.flat_map do |remote| map_rsync_output_to_list( rsync_files_to_remote(list_file, remote) ) end.uniq list_file.unlink push_list = resolve_conflicts(push_list) puts "#{push_list.length} files pushed to #{branch}" return push_list end
@return [Array] resolved list
# File lib/branchable_cdn_assets/file_manager.rb, line 324 def resolve_conflicts push_list conflicts = list(:local) - push_list return push_list if conflicts.empty? puts "these local files were not pushed by rsync:" puts conflicts.join("\n") if Shell.get_input("add to the #{branch} manifest? (y|n) ") == "y" push_list.push(*conflicts) end return push_list end
# File lib/branchable_cdn_assets/file_manager.rb, line 315 def rsync_files_from_remote list_file, remote resp = Shell.run_local "rsync #{config.rsync_flags} --files-from=#{list_file.path}" + " -e ssh #{remote.host}:#{remote.root} #{root}/" raise "rsync failed\n#{resp.stderr}" unless resp.success? return resp.stdout end
# File lib/branchable_cdn_assets/file_manager.rb, line 282 def rsync_files_to_remote list_file, remote dir_perms = Permissions.new( remote.dir_permissions ).to_chmod file_perms = Permissions.new( remote.file_permissions ).to_chmod file_permissions = "--chmod=" + "Du=#{dir_perms.user},Dg=#{dir_perms.group},Do=#{dir_perms.others}," + "Fu=#{file_perms.user},Fg=#{file_perms.group},Fo=#{file_perms.others}" resp = Shell.run_local "rsync #{remote.rsync_flags} #{file_permissions} --files-from=#{list_file.path} " + "-e ssh #{root}/ #{remote.host}:#{remote.root}" raise "rsync failed for #{remote.host}\n#{resp.stderr}" unless resp.success? resp.stdout end
create the root dir on remote
# File lib/branchable_cdn_assets/file_manager.rb, line 240 def setup_remotes config.remotes.each do |remote| resp = Shell.run_remote "mkdir -p -m #{remote.dir_permissions} #{remote.root}", hostname: remote.host unless resp.success? raise "problem setting up #{remote.host}\n#{resp.stderr}" else puts "set up #{remote.root} on #{remote.host}" end end end
# File lib/branchable_cdn_assets/file_manager.rb, line 210 def uniq_files_per_remote files_by_remote = files_per_remote files_on_all = [] files_by_remote.each_with_index do |(remote, files), idx| if idx == 0 files_on_all = files else files_on_all = files_on_all & files end end Hash[ files_by_remote.map { |r, files| [r, (files - files_on_all)] } .push ["all", files_on_all] ] end