class Puppet::FileBucketFile::File
Public Instance Methods
# File lib/puppet/indirector/file_bucket_file/file.rb 14 def find(request) 15 request.options[:bucket_path] ||= Puppet[:bucketdir] 16 # If filebucket mode is 'list' 17 if request.options[:list_all] 18 return nil unless ::File.exist?(request.options[:bucket_path]) 19 return list(request) 20 end 21 checksum, files_original_path = request_to_checksum_and_path(request) 22 contents_file = path_for(request.options[:bucket_path], checksum, 'contents') 23 paths_file = path_for(request.options[:bucket_path], checksum, 'paths') 24 25 if Puppet::FileSystem.exist?(contents_file) && matches(paths_file, files_original_path) 26 if request.options[:diff_with] 27 other_contents_file = path_for(request.options[:bucket_path], request.options[:diff_with], 'contents') 28 raise _("could not find diff_with %{diff}") % { diff: request.options[:diff_with] } unless Puppet::FileSystem.exist?(other_contents_file) 29 raise _("Unable to diff on this platform") unless Puppet[:diff] != "" 30 return diff(Puppet::FileSystem.path_string(contents_file), Puppet::FileSystem.path_string(other_contents_file)) 31 else 32 #TRANSLATORS "FileBucket" should not be translated 33 Puppet.info _("FileBucket read %{checksum}") % { checksum: checksum } 34 model.new(Puppet::FileSystem.binread(contents_file)) 35 end 36 else 37 nil 38 end 39 end
# File lib/puppet/indirector/file_bucket_file/file.rb 90 def head(request) 91 checksum, files_original_path = request_to_checksum_and_path(request) 92 contents_file = path_for(request.options[:bucket_path], checksum, 'contents') 93 paths_file = path_for(request.options[:bucket_path], checksum, 'paths') 94 95 Puppet::FileSystem.exist?(contents_file) && matches(paths_file, files_original_path) 96 end
# File lib/puppet/indirector/file_bucket_file/file.rb 41 def list(request) 42 if request.remote? 43 raise Puppet::Error, _("Listing remote file buckets is not allowed") 44 end 45 46 fromdate = request.options[:fromdate] || "0:0:0 1-1-1970" 47 todate = request.options[:todate] || Time.now.strftime("%F %T") 48 begin 49 to = Time.parse(todate) 50 rescue ArgumentError 51 raise Puppet::Error, _("Error while parsing 'todate'") 52 end 53 begin 54 from = Time.parse(fromdate) 55 rescue ArgumentError 56 raise Puppet::Error, _("Error while parsing 'fromdate'") 57 end 58 # Setting hash's default value to [], needed by the following loop 59 bucket = Hash.new {[]} 60 msg = "" 61 # Get all files with mtime between 'from' and 'to' 62 Pathname.new(request.options[:bucket_path]).find { |item| 63 if item.file? and item.basename.to_s == "paths" 64 filenames = item.read.strip.split("\n") 65 filestat = Time.parse(item.stat.mtime.to_s) 66 if from <= filestat and filestat <= to 67 filenames.each do |filename| 68 bucket[filename] += [[ item.stat.mtime , item.parent.basename ]] 69 end 70 end 71 end 72 } 73 # Sort the results 74 bucket.each { |filename, contents| 75 contents.sort_by! do |item| 76 # NOTE: Ruby 2.4 may reshuffle item order even if the keys in sequence are sorted already 77 item[0] 78 end 79 } 80 # Build the output message. Sorted by names then by dates 81 bucket.sort.each { |filename,contents| 82 contents.each { |mtime, chksum| 83 date = mtime.strftime("%F %T") 84 msg += "#{chksum} #{date} #{filename}\n" 85 } 86 } 87 return model.new(msg) 88 end
# File lib/puppet/indirector/file_bucket_file/file.rb 98 def save(request) 99 instance = request.instance 100 _, files_original_path = request_to_checksum_and_path(request) 101 contents_file = path_for(instance.bucket_path, instance.checksum_data, 'contents') 102 paths_file = path_for(instance.bucket_path, instance.checksum_data, 'paths') 103 104 save_to_disk(instance, files_original_path, contents_file, paths_file) 105 106 # don't echo the request content back to the agent 107 model.new('') 108 end
# File lib/puppet/indirector/file_bucket_file/file.rb 110 def validate_key(request) 111 # There are no ACLs on filebucket files so validating key is not important 112 end
Private Instance Methods
@param contents_file [Pathname] Opaque file path to intended backup
location
@param expected_checksum_data [String] expected value of checksum of type
checksum_type
@param checksum_type [String] type of check sum of checksum_data, ie “md5” @return [Boolean] whether the checksum of the contents_file matches the
supplied checksum
@api private
# File lib/puppet/indirector/file_bucket_file/file.rb 241 def contents_file_matches_checksum?(contents_file, expected_checksum_data, checksum_type) 242 contents_file_checksum_data = Puppet::Util::Checksums.method(:"#{checksum_type}_file").call(contents_file.to_path) 243 contents_file_checksum_data == expected_checksum_data 244 end
@param contents_file [Pathname] Opaque file path to intended backup
location
@param bucket_file [Puppet::FileBucket::File] IO object representing
content to back up
@return [void] @api private
# File lib/puppet/indirector/file_bucket_file/file.rb 252 def copy_bucket_file_to_contents_file(contents_file, bucket_file) 253 Puppet::FileSystem.replace_file(contents_file, 0440) do |of| 254 # PUP-1044 writes all of the contents 255 bucket_file.stream() do |src| 256 FileUtils.copy_stream(src, of) 257 end 258 end 259 end
@param paths_file [Object] Opaque file path @param files_original_path [String]
# File lib/puppet/indirector/file_bucket_file/file.rb 119 def matches(paths_file, files_original_path) 120 # Puppet will have already written the paths_file in the systems encoding 121 # given its possible that request.options[:bucket_path] or Puppet[:bucketdir] 122 # contained characters in an encoding that are not represented the 123 # same way when the bytes are decoded as UTF-8, continue using system encoding 124 Puppet::FileSystem.open(paths_file, 0640, 'a+:external') do |f| 125 path_match(f, files_original_path) 126 end 127 end
@return [Object] Opaque path as constructed by the Puppet::FileSystem
# File lib/puppet/indirector/file_bucket_file/file.rb 212 def path_for(bucket_path, digest, subfile = nil) 213 bucket_path ||= Puppet[:bucketdir] 214 215 dir = ::File.join(digest[0..7].split("")) 216 basedir = ::File.join(bucket_path, dir, digest) 217 218 Puppet::FileSystem.pathname(subfile ? ::File.join(basedir, subfile) : basedir) 219 end
# File lib/puppet/indirector/file_bucket_file/file.rb 129 def path_match(file_handle, files_original_path) 130 return true unless files_original_path # if no path was provided, it's a match 131 file_handle.rewind 132 file_handle.each_line do |line| 133 return true if line.chomp == files_original_path 134 end 135 return false 136 end
# File lib/puppet/indirector/file_bucket_file/file.rb 199 def request_to_checksum_and_path(request) 200 checksum_type, checksum, path = request.key.split(/\//, 3) 201 if path == '' # Treat "md5/<checksum>/" like "md5/<checksum>" 202 path = nil 203 end 204 raise ArgumentError, _("Unsupported checksum type %{checksum_type}") % { checksum_type: checksum_type.inspect } if checksum_type != Puppet[:digest_algorithm] 205 expected = method(checksum_type + "_hex_length").call 206 raise _("Invalid checksum %{checksum}") % { checksum: checksum.inspect } if checksum !~ /^[0-9a-f]{#{expected}}$/ 207 [checksum, path] 208 end
@param bucket_file [Puppet::FileBucket::File] IO object representing
content to back up
@param files_original_path [String] Path to original source file on disk @param contents_file [Pathname] Opaque file path to intended backup
location
@param paths_file [Pathname] Opaque file path to file containing source
file paths on disk
@return [void] @raise [Puppet::FileBucket::BucketError] on possible sum collision between
existing and new backup
@api private
# File lib/puppet/indirector/file_bucket_file/file.rb 149 def save_to_disk(bucket_file, files_original_path, contents_file, paths_file) 150 Puppet::Util.withumask(0007) do 151 unless Puppet::FileSystem.dir_exist?(paths_file) 152 Puppet::FileSystem.dir_mkpath(paths_file) 153 end 154 155 # Puppet will have already written the paths_file in the systems encoding 156 # given its possible that request.options[:bucket_path] or Puppet[:bucketdir] 157 # contained characters in an encoding that are not represented the 158 # same way when the bytes are decoded as UTF-8, continue using system encoding 159 Puppet::FileSystem.exclusive_open(paths_file, 0640, 'a+:external') do |f| 160 if Puppet::FileSystem.exist?(contents_file) 161 if verify_identical_file(contents_file, bucket_file) 162 #TRANSLATORS "FileBucket" should not be translated 163 Puppet.info _("FileBucket got a duplicate file %{file_checksum}") % { file_checksum: bucket_file.checksum } 164 # Don't touch the contents file on Windows, since we can't update the 165 # mtime of read-only files there. 166 if !Puppet::Util::Platform.windows? 167 Puppet::FileSystem.touch(contents_file) 168 end 169 elsif contents_file_matches_checksum?(contents_file, bucket_file.checksum_data, bucket_file.checksum_type) 170 # If the contents or sizes don't match, but the checksum does, 171 # then we've found a conflict (potential hash collision). 172 # Unlikely, but quite bad. Don't remove the file in case it's 173 # needed, but ask the user to validate. 174 # Note: Don't print the full path to the bucket file in the 175 # exception to avoid disclosing file system layout on server. 176 #TRANSLATORS "FileBucket" should not be translated 177 Puppet.err(_("Unable to verify existing FileBucket backup at '%{path}'.") % { path: contents_file.to_path }) 178 raise Puppet::FileBucket::BucketError, _("Existing backup and new file have different content but same checksum, %{value}. Verify existing backup and remove if incorrect.") % 179 { value: bucket_file.checksum } 180 else 181 # PUP-1334 If the contents_file exists but does not match its 182 # checksum, our backup has been corrupted. Warn about overwriting 183 # it, and proceed with new backup. 184 Puppet.warning(_("Existing backup does not match its expected sum, %{sum}. Overwriting corrupted backup.") % { sum: bucket_file.checksum }) 185 copy_bucket_file_to_contents_file(contents_file, bucket_file) 186 end 187 else 188 copy_bucket_file_to_contents_file(contents_file, bucket_file) 189 end 190 191 unless path_match(f, files_original_path) 192 f.seek(0, IO::SEEK_END) 193 f.puts(files_original_path) 194 end 195 end 196 end 197 end
@param contents_file [Pathname] Opaque file path to intended backup
location
@param bucket_file [Puppet::FileBucket::File] IO object representing
content to back up
@return [Boolean] whether the data in contents_file is of the same size
and content as that in the bucket_file
@api private
# File lib/puppet/indirector/file_bucket_file/file.rb 228 def verify_identical_file(contents_file, bucket_file) 229 (bucket_file.to_binary.bytesize == Puppet::FileSystem.size(contents_file)) && 230 (bucket_file.stream() {|s| Puppet::FileSystem.compare_stream(contents_file, s) }) 231 end