class Catalog::Compiler

Attributes

code[RW]

Public Class Methods

new() click to toggle source
   # File lib/puppet/indirector/catalog/compiler.rb
84 def initialize
85   Puppet::Util::Profiler.profile(_("Setup server facts for compiling"), [:compiler, :init_server_facts]) do
86     set_server_facts
87   end
88 end

Public Instance Methods

extract_facts_from_request(request) click to toggle source

@param request [Puppet::Indirector::Request] an indirection request

(possibly) containing facts

@return [Puppet::Node::Facts] facts object corresponding to facts in request

   # File lib/puppet/indirector/catalog/compiler.rb
21 def extract_facts_from_request(request)
22   text_facts = request.options[:facts]
23   return unless text_facts
24   format = request.options[:facts_format]
25   unless format
26     raise ArgumentError, _("Facts but no fact format provided for %{request}") % { request: request.key }
27   end
28 
29   Puppet::Util::Profiler.profile(_("Found facts"), [:compiler, :find_facts]) do
30     facts = text_facts.is_a?(Puppet::Node::Facts) ? text_facts :
31                                                     convert_wire_facts(text_facts, format)
32 
33     unless facts.name == request.key
34       raise Puppet::Error, _("Catalog for %{request} was requested with fact definition for the wrong node (%{fact_name}).") % { request: request.key.inspect, fact_name: facts.name.inspect }
35     end
36     return facts
37   end
38 end
filter(catalog) click to toggle source

filter-out a catalog to remove exported resources

   # File lib/puppet/indirector/catalog/compiler.rb
79 def filter(catalog)
80   return catalog.filter { |r| r.virtual? } if catalog.respond_to?(:filter)
81   catalog
82 end
find(request) click to toggle source

Compile a node's catalog.

   # File lib/puppet/indirector/catalog/compiler.rb
47 def find(request)
48   facts = extract_facts_from_request(request)
49 
50   save_facts_from_request(facts, request) if !facts.nil?
51 
52   node = node_from_request(facts, request)
53   node.trusted_data = Puppet.lookup(:trusted_information) { Puppet::Context::TrustedInformation.local(node) }.to_h
54 
55   if node.environment
56     # If the requested environment doesn't match the server specified environment,
57     # as determined by the node terminus, and the request wants us to check for an
58     # environment mismatch, then return an empty catalog with the server-specified
59     # enviroment.
60     if request.remote? && request.options[:check_environment] && node.environment != request.environment
61       return Puppet::Resource::Catalog.new(node.name, node.environment)
62     end
63 
64     node.environment.with_text_domain do
65       envs = Puppet.lookup(:environments)
66       envs.guard(node.environment.name)
67       begin
68         compile(node, request.options)
69       ensure
70         envs.unguard(node.environment.name)
71       end
72     end
73   else
74     compile(node, request.options)
75   end
76 end
networked?() click to toggle source

Is our compiler part of a network, or are we just local?

   # File lib/puppet/indirector/catalog/compiler.rb
91 def networked?
92   Puppet.run_mode.server?
93 end
require_environment?() click to toggle source
   # File lib/puppet/indirector/catalog/compiler.rb
95 def require_environment?
96   false
97 end
save_facts_from_request(facts, request) click to toggle source
   # File lib/puppet/indirector/catalog/compiler.rb
40 def save_facts_from_request(facts, request)
41   Puppet::Node::Facts.indirection.save(facts, nil,
42                                        :environment => request.environment,
43                                        :transaction_uuid => request.options[:transaction_uuid])
44 end

Private Instance Methods

add_node_data(node) click to toggle source

Add any extra data necessary to the node.

    # File lib/puppet/indirector/catalog/compiler.rb
118 def add_node_data(node)
119   # Merge in our server-side facts, so they can be used during compilation.
120   node.add_server_facts(@server_facts)
121 end
common_checksum_type(agent_checksum_type) click to toggle source

Determine which checksum to use; if agent_checksum_type is not nil, use the first entry in it that is also in known_checksum_types. If no match is found, return nil.

    # File lib/puppet/indirector/catalog/compiler.rb
126 def common_checksum_type(agent_checksum_type)
127   if agent_checksum_type
128     agent_checksum_types = agent_checksum_type.split('.').map {|type| type.to_sym}
129     checksum_type = agent_checksum_types.drop_while do |type|
130       not known_checksum_types.include? type
131     end.first
132   end
133   checksum_type
134 end
compile(node, options) click to toggle source

Compile the actual catalog.

    # File lib/puppet/indirector/catalog/compiler.rb
295 def compile(node, options)
296   if node.environment && node.environment.static_catalogs? && options[:static_catalog] && options[:code_id]
297     # Check for errors before compiling the catalog
298     checksum_type = common_checksum_type(options[:checksum_type])
299     raise Puppet::Error, _("Unable to find a common checksum type between agent '%{agent_type}' and master '%{master_type}'.") % { agent_type: options[:checksum_type], master_type: known_checksum_types } unless checksum_type
300   end
301 
302   escaped_node_name = node.name.gsub(/%/, '%%')
303   if checksum_type
304     if node.environment
305       escaped_node_environment = node.environment.to_s.gsub(/%/, '%%')
306       benchmark_str = _("Compiled static catalog for %{node} in environment %{environment} in %%{seconds} seconds") % { node: escaped_node_name, environment: escaped_node_environment }
307       profile_str   = _("Compiled static catalog for %{node} in environment %{environment}") % { node: node.name, environment: node.environment }
308     else
309       benchmark_str = _("Compiled static catalog for %{node} in %%{seconds} seconds") % { node: escaped_node_name }
310       profile_str   = _("Compiled static catalog for %{node}") % { node: node.name }
311     end
312   else
313     if node.environment
314       escaped_node_environment = node.environment.to_s.gsub(/%/, '%%')
315       benchmark_str = _("Compiled catalog for %{node} in environment %{environment} in %%{seconds} seconds") % { node: escaped_node_name, environment: escaped_node_environment }
316       profile_str   = _("Compiled catalog for %{node} in environment %{environment}") % { node: node.name, environment: node.environment }
317     else
318       benchmark_str = _("Compiled catalog for %{node} in %%{seconds} seconds") % { node: escaped_node_name }
319       profile_str   = _("Compiled catalog for %{node}") % { node: node.name }
320     end
321   end
322   config = nil
323 
324   benchmark(:notice, benchmark_str) do
325     compile_type = checksum_type ? :static_compile : :compile
326     Puppet::Util::Profiler.profile(profile_str, [:compiler, compile_type, node.environment, node.name]) do
327       begin
328         config = Puppet::Parser::Compiler.compile(node, options[:code_id])
329       rescue Puppet::Error => detail
330         Puppet.err(detail.to_s) if networked?
331         raise
332       ensure
333         Puppet::Type.clear_misses unless Puppet[:always_retry_plugins]
334       end
335 
336       if checksum_type && config.is_a?(model)
337         escaped_node_name = node.name.gsub(/%/, '%%')
338         if node.environment
339           escaped_node_environment = node.environment.to_s.gsub(/%/, '%%')
340           #TRANSLATORS Inlined refers to adding additional metadata
341           benchmark_str = _("Inlined resource metadata into static catalog for %{node} in environment %{environment} in %%{seconds} seconds") % { node: escaped_node_name, environment: escaped_node_environment }
342           #TRANSLATORS Inlined refers to adding additional metadata
343           profile_str   = _("Inlined resource metadata into static catalog for %{node} in environment %{environment}") % { node: node.name, environment: node.environment }
344         else
345           #TRANSLATORS Inlined refers to adding additional metadata
346           benchmark_str = _("Inlined resource metadata into static catalog for %{node} in %%{seconds} seconds") % { node: escaped_node_name }
347           #TRANSLATORS Inlined refers to adding additional metadata
348           profile_str   = _("Inlined resource metadata into static catalog for %{node}") % { node: node.name }
349         end
350         benchmark(:notice, benchmark_str) do
351           Puppet::Util::Profiler.profile(profile_str, [:compiler, :static_compile_postprocessing, node.environment, node.name]) do
352             inline_metadata(config, checksum_type)
353           end
354         end
355       end
356     end
357   end
358 
359 
360   config
361 end
convert_wire_facts(facts, format) click to toggle source

@param facts [String] facts in a wire format for decoding @param format [String] a content-type string @return [Puppet::Node::Facts] facts object deserialized from supplied string @api private

    # File lib/puppet/indirector/catalog/compiler.rb
105 def convert_wire_facts(facts, format)
106   if format == 'pson'
107     # We unescape here because the corresponding code in Puppet::Configurer::FactHandler encodes with Puppet::Util.uri_query_encode
108     # PSON is deprecated, but continue to accept from older agents
109     return Puppet::Node::Facts.convert_from('pson', CGI.unescape(facts))
110   elsif format == 'application/json'
111     return Puppet::Node::Facts.convert_from('json', CGI.unescape(facts))
112   else
113     raise ArgumentError, _("Unsupported facts format")
114   end
115 end
find_node(name, environment, transaction_uuid, configured_environment, facts) click to toggle source

Use indirection to find the node associated with a given request

    # File lib/puppet/indirector/catalog/compiler.rb
364 def find_node(name, environment, transaction_uuid, configured_environment, facts)
365   Puppet::Util::Profiler.profile(_("Found node information"), [:compiler, :find_node]) do
366     node = nil
367     begin
368       node = Puppet::Node.indirection.find(name, :environment => environment,
369                                            :transaction_uuid => transaction_uuid,
370                                            :configured_environment => configured_environment,
371                                            :facts => facts)
372     rescue => detail
373       message = _("Failed when searching for node %{name}: %{detail}") % { name: name, detail: detail }
374       Puppet.log_exception(detail, message)
375       raise Puppet::Error, message, detail.backtrace
376     end
377 
378 
379     # Add any external data to the node.
380     if node
381       add_node_data(node)
382     end
383     node
384   end
385 end
get_content_uri(metadata, source, environment_path) click to toggle source
    # File lib/puppet/indirector/catalog/compiler.rb
136 def get_content_uri(metadata, source, environment_path)
137   # The static file content server doesn't know how to expand mountpoints, so
138   # we need to do that ourselves from the actual system path of the source file.
139   # This does that, while preserving any user-specified server or port.
140   source_path = Pathname.new(metadata.full_path)
141   path = source_path.relative_path_from(environment_path).to_s
142   source_as_uri = URI.parse(Puppet::Util.uri_encode(source))
143   server = source_as_uri.host
144   port = ":#{source_as_uri.port}" if source_as_uri.port
145   return "puppet://#{server}#{port}/#{path}"
146 end
inline_metadata(catalog, checksum_type) click to toggle source

Inline file metadata for static catalogs Initially restricted to files sourced from codedir via puppet:/// uri.

    # File lib/puppet/indirector/catalog/compiler.rb
193 def inline_metadata(catalog, checksum_type)
194   environment_path = Pathname.new File.join(Puppet[:environmentpath], catalog.environment)
195   environment_path = Puppet::Environments::Directories.real_path(environment_path)
196   list_of_resources = catalog.resources.find_all { |res| res.type == "File" }
197 
198   # TODO: get property/parameter defaults if entries are nil in the resource
199   # For now they're hard-coded to match the File type.
200 
201   list_of_resources.each do |resource|
202     sources = [resource[:source]].flatten.compact
203     next unless inlineable?(resource, sources)
204 
205     # both need to handle multiple sources
206     if resource[:recurse] == true || resource[:recurse] == 'true' || resource[:recurse] == 'remote'
207       # Construct a hash mapping sources to arrays (list of files found recursively) of metadata
208       options = {
209         :environment        => catalog.environment_instance,
210         :links              => resource[:links] ? resource[:links].to_sym : :manage,
211         :checksum_type      => resource[:checksum] ? resource[:checksum].to_sym : checksum_type.to_sym,
212         :source_permissions => resource[:source_permissions] ? resource[:source_permissions].to_sym : :ignore,
213         :recurse            => true,
214         :recurselimit       => resource[:recurselimit],
215         :max_files          => resource[:max_files],
216         :ignore             => resource[:ignore],
217       }
218 
219       sources_in_environment = true
220 
221       source_to_metadatas = {}
222       sources.each do |source|
223         source = Puppet::Type.type(:file).attrclass(:source).normalize(source)
224 
225         list_of_data = Puppet::FileServing::Metadata.indirection.search(source, options)
226         if list_of_data
227           basedir_meta = list_of_data.find {|meta| meta.relative_path == '.'}
228           devfail "FileServing::Metadata search should always return the root search path" if basedir_meta.nil?
229 
230           if ! inlineable_metadata?(basedir_meta, source,  environment_path)
231             # If any source is not in the environment path, skip inlining this resource.
232             log_file_outside_environment
233             sources_in_environment = false
234             break
235           end
236 
237           base_content_uri = get_content_uri(basedir_meta, source, environment_path)
238           list_of_data.each do |metadata|
239             if metadata.relative_path == '.'
240               metadata.content_uri = base_content_uri
241             else
242               metadata.content_uri = "#{base_content_uri}/#{metadata.relative_path}"
243             end
244           end
245 
246           source_to_metadatas[source] = list_of_data
247           # Optimize for returning less data if sourceselect is first
248           if resource[:sourceselect] == 'first' || resource[:sourceselect].nil?
249             break
250           end
251         end
252       end
253 
254       if sources_in_environment && !source_to_metadatas.empty?
255         log_metadata_inlining
256         catalog.recursive_metadata[resource.title] = source_to_metadatas
257       end
258     else
259       options = {
260         :environment        => catalog.environment_instance,
261         :links              => resource[:links] ? resource[:links].to_sym : :manage,
262         :checksum_type      => resource[:checksum] ? resource[:checksum].to_sym : checksum_type.to_sym,
263         :source_permissions => resource[:source_permissions] ? resource[:source_permissions].to_sym : :ignore
264       }
265 
266       metadata = nil
267       sources.each do |source|
268         source = Puppet::Type.type(:file).attrclass(:source).normalize(source)
269 
270         data = Puppet::FileServing::Metadata.indirection.find(source, options)
271         if data
272           metadata = data
273           metadata.source = source
274           break
275         end
276       end
277 
278       raise _("Could not get metadata for %{resource}") % { resource: resource[:source] } unless metadata
279 
280       if inlineable_metadata?(metadata, metadata.source,  environment_path)
281         metadata.content_uri = get_content_uri(metadata, metadata.source, environment_path)
282         log_metadata_inlining
283 
284         # If the file is in the environment directory, we can safely inline
285         catalog.metadata[resource.title] = metadata
286       else
287         # Log a profiler event that we skipped this file because it is not in an environment.
288         log_file_outside_environment
289       end
290     end
291   end
292 end
inlineable?(resource, sources) click to toggle source

Helper method to decide if a file resource's metadata can be inlined. Also used to profile/log reasons for not inlining.

    # File lib/puppet/indirector/catalog/compiler.rb
150 def inlineable?(resource, sources)
151   case
152     when resource[:ensure] == 'absent'
153       #TRANSLATORS Inlining refers to adding additional metadata (in this case we are not inlining)
154       return Puppet::Util::Profiler.profile(_("Not inlining absent resource"), [:compiler, :static_compile_inlining, :skipped_file_metadata, :absent]) { false }
155     when sources.empty?
156       #TRANSLATORS Inlining refers to adding additional metadata (in this case we are not inlining)
157       return Puppet::Util::Profiler.profile(_("Not inlining resource without sources"), [:compiler, :static_compile_inlining, :skipped_file_metadata, :no_sources]) { false }
158     when (not (sources.all? {|source| source =~ /^puppet:/}))
159       #TRANSLATORS Inlining refers to adding additional metadata (in this case we are not inlining)
160       return Puppet::Util::Profiler.profile(_("Not inlining unsupported source scheme"), [:compiler, :static_compile_inlining, :skipped_file_metadata, :unsupported_scheme]) { false }
161     else
162       return true
163   end
164 end
inlineable_metadata?(metadata, source, environment_path) click to toggle source

Return true if metadata is inlineable, meaning the request's source is for the 'modules' mount and the resolved path is of the form:

$codedir/environments/$environment/*/*/files/**
    # File lib/puppet/indirector/catalog/compiler.rb
169 def inlineable_metadata?(metadata, source, environment_path)
170   source_as_uri = URI.parse(Puppet::Util.uri_encode(source))
171 
172   location = Puppet::Module::FILETYPES['files']
173 
174   !!(source_as_uri.path =~ /^\/modules\// &&
175      metadata.full_path =~ /#{environment_path}\/[^\/]+\/[^\/]+\/#{location}\/.+/)
176 end
log_file_outside_environment() click to toggle source

Helper method to log file resources that could not be inlined because they fall outside of an environment.

    # File lib/puppet/indirector/catalog/compiler.rb
180 def log_file_outside_environment
181   #TRANSLATORS Inlining refers to adding additional metadata (in this case we are not inlining)
182   Puppet::Util::Profiler.profile(_("Not inlining file outside environment"), [:compiler, :static_compile_inlining, :skipped_file_metadata, :file_outside_environment]) { true }
183 end
log_metadata_inlining() click to toggle source

Helper method to log file resources that were successfully inlined.

    # File lib/puppet/indirector/catalog/compiler.rb
186 def log_metadata_inlining
187   #TRANSLATORS Inlining refers to adding additional metadata
188   Puppet::Util::Profiler.profile(_("Inlining file metadata"), [:compiler, :static_compile_inlining, :inlined_file_metadata]) { true }
189 end
node_from_request(facts, request) click to toggle source

Extract the node from the request, or use the request to find the node.

    # File lib/puppet/indirector/catalog/compiler.rb
389 def node_from_request(facts, request)
390   node = request.options[:use_node]
391   if node
392     if request.remote?
393       raise Puppet::Error, _("Invalid option use_node for a remote request")
394     else
395       return node
396     end
397   end
398 
399   # We rely on our authorization system to determine whether the connected
400   # node is allowed to compile the catalog's node referenced by key.
401   # By default the REST authorization system makes sure only the connected node
402   # can compile his catalog.
403   # This allows for instance monitoring systems or puppet-load to check several
404   # node's catalog with only one certificate and a modification to auth.conf
405   # If no key is provided we can only compile the currently connected node.
406   name = request.key || request.node
407   node = find_node(name, request.environment, request.options[:transaction_uuid], request.options[:configured_environment], facts)
408   if node
409     return node
410   end
411 
412   raise ArgumentError, _("Could not find node '%{name}'; cannot compile") % { name: name }
413 end
set_server_facts() click to toggle source

Initialize our server fact hash; we add these to each client, and they won't change while we're running, so it's safe to cache the values.

See also set_server_facts in Puppet::Server::Compiler in puppetserver.

    # File lib/puppet/indirector/catalog/compiler.rb
419 def set_server_facts
420   @server_facts = {}
421 
422   # Add our server Puppet Enterprise version, if available.
423   pe_version_file = '/opt/puppetlabs/server/pe_version'
424   if File.readable?(pe_version_file) and !File.zero?(pe_version_file)
425     @server_facts['pe_serverversion'] = File.read(pe_version_file).chomp
426   end
427 
428   # Add our server version to the fact list
429   @server_facts["serverversion"] = Puppet.version.to_s
430 
431   # And then add the server name and IP
432   {"servername" => "fqdn",
433     "serverip"  => "ipaddress",
434     "serverip6" => "ipaddress6"
435   }.each do |var, fact|
436     value = Facter.value(fact)
437     if !value.nil?
438       @server_facts[var] = value
439     end
440   end
441 
442   if @server_facts["servername"].nil?
443     host = Facter.value(:hostname)
444     if host.nil?
445       Puppet.warning _("Could not retrieve fact servername")
446     elsif domain = Facter.value(:domain) #rubocop:disable Lint/AssignmentInCondition
447       @server_facts["servername"] = [host, domain].join(".")
448     else
449       @server_facts["servername"] = host
450     end
451   end
452 
453   if @server_facts["serverip"].nil? && @server_facts["serverip6"].nil?
454     Puppet.warning _("Could not retrieve either serverip or serverip6 fact")
455   end
456 end