class Catalog::Compiler
Attributes
Public Class Methods
# File lib/puppet/indirector/catalog/compiler.rb 84 def initialize 85 Puppet::Util::Profiler.profile(_("Setup server facts for compiling"), [:compiler, :init_server_facts]) do 86 set_server_facts 87 end 88 end
Public Instance Methods
@param request [Puppet::Indirector::Request] an indirection request
(possibly) containing facts
@return [Puppet::Node::Facts] facts object corresponding to facts in request
# File lib/puppet/indirector/catalog/compiler.rb 21 def extract_facts_from_request(request) 22 text_facts = request.options[:facts] 23 return unless text_facts 24 format = request.options[:facts_format] 25 unless format 26 raise ArgumentError, _("Facts but no fact format provided for %{request}") % { request: request.key } 27 end 28 29 Puppet::Util::Profiler.profile(_("Found facts"), [:compiler, :find_facts]) do 30 facts = text_facts.is_a?(Puppet::Node::Facts) ? text_facts : 31 convert_wire_facts(text_facts, format) 32 33 unless facts.name == request.key 34 raise Puppet::Error, _("Catalog for %{request} was requested with fact definition for the wrong node (%{fact_name}).") % { request: request.key.inspect, fact_name: facts.name.inspect } 35 end 36 return facts 37 end 38 end
filter-out a catalog to remove exported resources
# File lib/puppet/indirector/catalog/compiler.rb 79 def filter(catalog) 80 return catalog.filter { |r| r.virtual? } if catalog.respond_to?(:filter) 81 catalog 82 end
Compile a node's catalog.
# File lib/puppet/indirector/catalog/compiler.rb 47 def find(request) 48 facts = extract_facts_from_request(request) 49 50 save_facts_from_request(facts, request) if !facts.nil? 51 52 node = node_from_request(facts, request) 53 node.trusted_data = Puppet.lookup(:trusted_information) { Puppet::Context::TrustedInformation.local(node) }.to_h 54 55 if node.environment 56 # If the requested environment doesn't match the server specified environment, 57 # as determined by the node terminus, and the request wants us to check for an 58 # environment mismatch, then return an empty catalog with the server-specified 59 # enviroment. 60 if request.remote? && request.options[:check_environment] && node.environment != request.environment 61 return Puppet::Resource::Catalog.new(node.name, node.environment) 62 end 63 64 node.environment.with_text_domain do 65 envs = Puppet.lookup(:environments) 66 envs.guard(node.environment.name) 67 begin 68 compile(node, request.options) 69 ensure 70 envs.unguard(node.environment.name) 71 end 72 end 73 else 74 compile(node, request.options) 75 end 76 end
Is our compiler part of a network, or are we just local?
# File lib/puppet/indirector/catalog/compiler.rb 91 def networked? 92 Puppet.run_mode.server? 93 end
# File lib/puppet/indirector/catalog/compiler.rb 95 def require_environment? 96 false 97 end
# File lib/puppet/indirector/catalog/compiler.rb 40 def save_facts_from_request(facts, request) 41 Puppet::Node::Facts.indirection.save(facts, nil, 42 :environment => request.environment, 43 :transaction_uuid => request.options[:transaction_uuid]) 44 end
Private Instance Methods
Add any extra data necessary to the node.
# File lib/puppet/indirector/catalog/compiler.rb 118 def add_node_data(node) 119 # Merge in our server-side facts, so they can be used during compilation. 120 node.add_server_facts(@server_facts) 121 end
Determine which checksum to use; if agent_checksum_type is not nil, use the first entry in it that is also in known_checksum_types. If no match is found, return nil.
# File lib/puppet/indirector/catalog/compiler.rb 126 def common_checksum_type(agent_checksum_type) 127 if agent_checksum_type 128 agent_checksum_types = agent_checksum_type.split('.').map {|type| type.to_sym} 129 checksum_type = agent_checksum_types.drop_while do |type| 130 not known_checksum_types.include? type 131 end.first 132 end 133 checksum_type 134 end
Compile the actual catalog.
# File lib/puppet/indirector/catalog/compiler.rb 295 def compile(node, options) 296 if node.environment && node.environment.static_catalogs? && options[:static_catalog] && options[:code_id] 297 # Check for errors before compiling the catalog 298 checksum_type = common_checksum_type(options[:checksum_type]) 299 raise Puppet::Error, _("Unable to find a common checksum type between agent '%{agent_type}' and master '%{master_type}'.") % { agent_type: options[:checksum_type], master_type: known_checksum_types } unless checksum_type 300 end 301 302 escaped_node_name = node.name.gsub(/%/, '%%') 303 if checksum_type 304 if node.environment 305 escaped_node_environment = node.environment.to_s.gsub(/%/, '%%') 306 benchmark_str = _("Compiled static catalog for %{node} in environment %{environment} in %%{seconds} seconds") % { node: escaped_node_name, environment: escaped_node_environment } 307 profile_str = _("Compiled static catalog for %{node} in environment %{environment}") % { node: node.name, environment: node.environment } 308 else 309 benchmark_str = _("Compiled static catalog for %{node} in %%{seconds} seconds") % { node: escaped_node_name } 310 profile_str = _("Compiled static catalog for %{node}") % { node: node.name } 311 end 312 else 313 if node.environment 314 escaped_node_environment = node.environment.to_s.gsub(/%/, '%%') 315 benchmark_str = _("Compiled catalog for %{node} in environment %{environment} in %%{seconds} seconds") % { node: escaped_node_name, environment: escaped_node_environment } 316 profile_str = _("Compiled catalog for %{node} in environment %{environment}") % { node: node.name, environment: node.environment } 317 else 318 benchmark_str = _("Compiled catalog for %{node} in %%{seconds} seconds") % { node: escaped_node_name } 319 profile_str = _("Compiled catalog for %{node}") % { node: node.name } 320 end 321 end 322 config = nil 323 324 benchmark(:notice, benchmark_str) do 325 compile_type = checksum_type ? :static_compile : :compile 326 Puppet::Util::Profiler.profile(profile_str, [:compiler, compile_type, node.environment, node.name]) do 327 begin 328 config = Puppet::Parser::Compiler.compile(node, options[:code_id]) 329 rescue Puppet::Error => detail 330 Puppet.err(detail.to_s) if networked? 331 raise 332 ensure 333 Puppet::Type.clear_misses unless Puppet[:always_retry_plugins] 334 end 335 336 if checksum_type && config.is_a?(model) 337 escaped_node_name = node.name.gsub(/%/, '%%') 338 if node.environment 339 escaped_node_environment = node.environment.to_s.gsub(/%/, '%%') 340 #TRANSLATORS Inlined refers to adding additional metadata 341 benchmark_str = _("Inlined resource metadata into static catalog for %{node} in environment %{environment} in %%{seconds} seconds") % { node: escaped_node_name, environment: escaped_node_environment } 342 #TRANSLATORS Inlined refers to adding additional metadata 343 profile_str = _("Inlined resource metadata into static catalog for %{node} in environment %{environment}") % { node: node.name, environment: node.environment } 344 else 345 #TRANSLATORS Inlined refers to adding additional metadata 346 benchmark_str = _("Inlined resource metadata into static catalog for %{node} in %%{seconds} seconds") % { node: escaped_node_name } 347 #TRANSLATORS Inlined refers to adding additional metadata 348 profile_str = _("Inlined resource metadata into static catalog for %{node}") % { node: node.name } 349 end 350 benchmark(:notice, benchmark_str) do 351 Puppet::Util::Profiler.profile(profile_str, [:compiler, :static_compile_postprocessing, node.environment, node.name]) do 352 inline_metadata(config, checksum_type) 353 end 354 end 355 end 356 end 357 end 358 359 360 config 361 end
@param facts [String] facts in a wire format for decoding @param format [String] a content-type string @return [Puppet::Node::Facts] facts object deserialized from supplied string @api private
# File lib/puppet/indirector/catalog/compiler.rb 105 def convert_wire_facts(facts, format) 106 if format == 'pson' 107 # We unescape here because the corresponding code in Puppet::Configurer::FactHandler encodes with Puppet::Util.uri_query_encode 108 # PSON is deprecated, but continue to accept from older agents 109 return Puppet::Node::Facts.convert_from('pson', CGI.unescape(facts)) 110 elsif format == 'application/json' 111 return Puppet::Node::Facts.convert_from('json', CGI.unescape(facts)) 112 else 113 raise ArgumentError, _("Unsupported facts format") 114 end 115 end
Use indirection to find the node associated with a given request
# File lib/puppet/indirector/catalog/compiler.rb 364 def find_node(name, environment, transaction_uuid, configured_environment, facts) 365 Puppet::Util::Profiler.profile(_("Found node information"), [:compiler, :find_node]) do 366 node = nil 367 begin 368 node = Puppet::Node.indirection.find(name, :environment => environment, 369 :transaction_uuid => transaction_uuid, 370 :configured_environment => configured_environment, 371 :facts => facts) 372 rescue => detail 373 message = _("Failed when searching for node %{name}: %{detail}") % { name: name, detail: detail } 374 Puppet.log_exception(detail, message) 375 raise Puppet::Error, message, detail.backtrace 376 end 377 378 379 # Add any external data to the node. 380 if node 381 add_node_data(node) 382 end 383 node 384 end 385 end
# File lib/puppet/indirector/catalog/compiler.rb 136 def get_content_uri(metadata, source, environment_path) 137 # The static file content server doesn't know how to expand mountpoints, so 138 # we need to do that ourselves from the actual system path of the source file. 139 # This does that, while preserving any user-specified server or port. 140 source_path = Pathname.new(metadata.full_path) 141 path = source_path.relative_path_from(environment_path).to_s 142 source_as_uri = URI.parse(Puppet::Util.uri_encode(source)) 143 server = source_as_uri.host 144 port = ":#{source_as_uri.port}" if source_as_uri.port 145 return "puppet://#{server}#{port}/#{path}" 146 end
Inline file metadata for static catalogs Initially restricted to files sourced from codedir via puppet:/// uri.
# File lib/puppet/indirector/catalog/compiler.rb 193 def inline_metadata(catalog, checksum_type) 194 environment_path = Pathname.new File.join(Puppet[:environmentpath], catalog.environment) 195 environment_path = Puppet::Environments::Directories.real_path(environment_path) 196 list_of_resources = catalog.resources.find_all { |res| res.type == "File" } 197 198 # TODO: get property/parameter defaults if entries are nil in the resource 199 # For now they're hard-coded to match the File type. 200 201 list_of_resources.each do |resource| 202 sources = [resource[:source]].flatten.compact 203 next unless inlineable?(resource, sources) 204 205 # both need to handle multiple sources 206 if resource[:recurse] == true || resource[:recurse] == 'true' || resource[:recurse] == 'remote' 207 # Construct a hash mapping sources to arrays (list of files found recursively) of metadata 208 options = { 209 :environment => catalog.environment_instance, 210 :links => resource[:links] ? resource[:links].to_sym : :manage, 211 :checksum_type => resource[:checksum] ? resource[:checksum].to_sym : checksum_type.to_sym, 212 :source_permissions => resource[:source_permissions] ? resource[:source_permissions].to_sym : :ignore, 213 :recurse => true, 214 :recurselimit => resource[:recurselimit], 215 :max_files => resource[:max_files], 216 :ignore => resource[:ignore], 217 } 218 219 sources_in_environment = true 220 221 source_to_metadatas = {} 222 sources.each do |source| 223 source = Puppet::Type.type(:file).attrclass(:source).normalize(source) 224 225 list_of_data = Puppet::FileServing::Metadata.indirection.search(source, options) 226 if list_of_data 227 basedir_meta = list_of_data.find {|meta| meta.relative_path == '.'} 228 devfail "FileServing::Metadata search should always return the root search path" if basedir_meta.nil? 229 230 if ! inlineable_metadata?(basedir_meta, source, environment_path) 231 # If any source is not in the environment path, skip inlining this resource. 232 log_file_outside_environment 233 sources_in_environment = false 234 break 235 end 236 237 base_content_uri = get_content_uri(basedir_meta, source, environment_path) 238 list_of_data.each do |metadata| 239 if metadata.relative_path == '.' 240 metadata.content_uri = base_content_uri 241 else 242 metadata.content_uri = "#{base_content_uri}/#{metadata.relative_path}" 243 end 244 end 245 246 source_to_metadatas[source] = list_of_data 247 # Optimize for returning less data if sourceselect is first 248 if resource[:sourceselect] == 'first' || resource[:sourceselect].nil? 249 break 250 end 251 end 252 end 253 254 if sources_in_environment && !source_to_metadatas.empty? 255 log_metadata_inlining 256 catalog.recursive_metadata[resource.title] = source_to_metadatas 257 end 258 else 259 options = { 260 :environment => catalog.environment_instance, 261 :links => resource[:links] ? resource[:links].to_sym : :manage, 262 :checksum_type => resource[:checksum] ? resource[:checksum].to_sym : checksum_type.to_sym, 263 :source_permissions => resource[:source_permissions] ? resource[:source_permissions].to_sym : :ignore 264 } 265 266 metadata = nil 267 sources.each do |source| 268 source = Puppet::Type.type(:file).attrclass(:source).normalize(source) 269 270 data = Puppet::FileServing::Metadata.indirection.find(source, options) 271 if data 272 metadata = data 273 metadata.source = source 274 break 275 end 276 end 277 278 raise _("Could not get metadata for %{resource}") % { resource: resource[:source] } unless metadata 279 280 if inlineable_metadata?(metadata, metadata.source, environment_path) 281 metadata.content_uri = get_content_uri(metadata, metadata.source, environment_path) 282 log_metadata_inlining 283 284 # If the file is in the environment directory, we can safely inline 285 catalog.metadata[resource.title] = metadata 286 else 287 # Log a profiler event that we skipped this file because it is not in an environment. 288 log_file_outside_environment 289 end 290 end 291 end 292 end
Helper method to decide if a file resource's metadata can be inlined. Also used to profile/log reasons for not inlining.
# File lib/puppet/indirector/catalog/compiler.rb 150 def inlineable?(resource, sources) 151 case 152 when resource[:ensure] == 'absent' 153 #TRANSLATORS Inlining refers to adding additional metadata (in this case we are not inlining) 154 return Puppet::Util::Profiler.profile(_("Not inlining absent resource"), [:compiler, :static_compile_inlining, :skipped_file_metadata, :absent]) { false } 155 when sources.empty? 156 #TRANSLATORS Inlining refers to adding additional metadata (in this case we are not inlining) 157 return Puppet::Util::Profiler.profile(_("Not inlining resource without sources"), [:compiler, :static_compile_inlining, :skipped_file_metadata, :no_sources]) { false } 158 when (not (sources.all? {|source| source =~ /^puppet:/})) 159 #TRANSLATORS Inlining refers to adding additional metadata (in this case we are not inlining) 160 return Puppet::Util::Profiler.profile(_("Not inlining unsupported source scheme"), [:compiler, :static_compile_inlining, :skipped_file_metadata, :unsupported_scheme]) { false } 161 else 162 return true 163 end 164 end
Return true if metadata is inlineable, meaning the request's source is for the 'modules' mount and the resolved path is of the form:
$codedir/environments/$environment/*/*/files/**
# File lib/puppet/indirector/catalog/compiler.rb 169 def inlineable_metadata?(metadata, source, environment_path) 170 source_as_uri = URI.parse(Puppet::Util.uri_encode(source)) 171 172 location = Puppet::Module::FILETYPES['files'] 173 174 !!(source_as_uri.path =~ /^\/modules\// && 175 metadata.full_path =~ /#{environment_path}\/[^\/]+\/[^\/]+\/#{location}\/.+/) 176 end
Helper method to log file resources that could not be inlined because they fall outside of an environment.
# File lib/puppet/indirector/catalog/compiler.rb 180 def log_file_outside_environment 181 #TRANSLATORS Inlining refers to adding additional metadata (in this case we are not inlining) 182 Puppet::Util::Profiler.profile(_("Not inlining file outside environment"), [:compiler, :static_compile_inlining, :skipped_file_metadata, :file_outside_environment]) { true } 183 end
Helper method to log file resources that were successfully inlined.
# File lib/puppet/indirector/catalog/compiler.rb 186 def log_metadata_inlining 187 #TRANSLATORS Inlining refers to adding additional metadata 188 Puppet::Util::Profiler.profile(_("Inlining file metadata"), [:compiler, :static_compile_inlining, :inlined_file_metadata]) { true } 189 end
Extract the node from the request, or use the request to find the node.
# File lib/puppet/indirector/catalog/compiler.rb 389 def node_from_request(facts, request) 390 node = request.options[:use_node] 391 if node 392 if request.remote? 393 raise Puppet::Error, _("Invalid option use_node for a remote request") 394 else 395 return node 396 end 397 end 398 399 # We rely on our authorization system to determine whether the connected 400 # node is allowed to compile the catalog's node referenced by key. 401 # By default the REST authorization system makes sure only the connected node 402 # can compile his catalog. 403 # This allows for instance monitoring systems or puppet-load to check several 404 # node's catalog with only one certificate and a modification to auth.conf 405 # If no key is provided we can only compile the currently connected node. 406 name = request.key || request.node 407 node = find_node(name, request.environment, request.options[:transaction_uuid], request.options[:configured_environment], facts) 408 if node 409 return node 410 end 411 412 raise ArgumentError, _("Could not find node '%{name}'; cannot compile") % { name: name } 413 end
Initialize our server fact hash; we add these to each client, and they won't change while we're running, so it's safe to cache the values.
See also set_server_facts
in Puppet::Server::Compiler in puppetserver.
# File lib/puppet/indirector/catalog/compiler.rb 419 def set_server_facts 420 @server_facts = {} 421 422 # Add our server Puppet Enterprise version, if available. 423 pe_version_file = '/opt/puppetlabs/server/pe_version' 424 if File.readable?(pe_version_file) and !File.zero?(pe_version_file) 425 @server_facts['pe_serverversion'] = File.read(pe_version_file).chomp 426 end 427 428 # Add our server version to the fact list 429 @server_facts["serverversion"] = Puppet.version.to_s 430 431 # And then add the server name and IP 432 {"servername" => "fqdn", 433 "serverip" => "ipaddress", 434 "serverip6" => "ipaddress6" 435 }.each do |var, fact| 436 value = Facter.value(fact) 437 if !value.nil? 438 @server_facts[var] = value 439 end 440 end 441 442 if @server_facts["servername"].nil? 443 host = Facter.value(:hostname) 444 if host.nil? 445 Puppet.warning _("Could not retrieve fact servername") 446 elsif domain = Facter.value(:domain) #rubocop:disable Lint/AssignmentInCondition 447 @server_facts["servername"] = [host, domain].join(".") 448 else 449 @server_facts["servername"] = host 450 end 451 end 452 453 if @server_facts["serverip"].nil? && @server_facts["serverip6"].nil? 454 Puppet.warning _("Could not retrieve either serverip or serverip6 fact") 455 end 456 end