class Puppet::Parser::Compiler

Maintain a graph of scopes, along with a bunch of data about the individual catalog we're compiling.

Abstract class for a catalog validator that can be registered with the compiler to run at a certain stage.

Constants

SETTINGS

Attributes

catalog[R]
code_id[RW]

The id of code input to the compiler. @api private

collections[R]
facts[R]
loaders[R]

Access to the configured loaders for 4x @return [Puppet::Pops::Loader::Loaders] the configured loaders @api private

node[R]
qualified_variables[R]
relationships[R]
resources[R]
topscope[R]

Public Class Methods

compile(node, code_id = nil) click to toggle source
   # File lib/puppet/parser/compiler.rb
21 def self.compile(node, code_id = nil)
22   node.environment.check_for_reparse
23 
24   errors = node.environment.validation_errors
25   if !errors.empty?
26     errors.each { |e| Puppet.err(e) } if errors.size > 1
27     errmsg = [
28       _("Compilation has been halted because: %{error}") % { error: errors.first },
29       _("For more information, see https://puppet.com/docs/puppet/latest/environments_about.html"),
30     ]
31     raise(Puppet::Error, errmsg.join(' '))
32   end
33 
34   new(node, :code_id => code_id).compile {|resulting_catalog| resulting_catalog.to_resource }
35 rescue Puppet::ParseErrorWithIssue => detail
36   detail.node = node.name
37   Puppet.log_exception(detail)
38   raise
39 rescue => detail
40   message = _("%{message} on node %{node}") % { message: detail, node: node.name }
41   Puppet.log_exception(detail, message)
42   raise Puppet::Error, message, detail.backtrace
43 end
new(node, code_id: nil) click to toggle source
    # File lib/puppet/parser/compiler.rb
273 def initialize(node, code_id: nil)
274   @node = sanitize_node(node)
275   @code_id = code_id
276   initvars
277   add_catalog_validators
278   # Resolutions of fully qualified variable names
279   @qualified_variables = {}
280 end

Public Instance Methods

add_catalog_validator(catalog_validators) click to toggle source

Add a catalog validator that will run at some stage to this compiler @param catalog_validators [Class<CatalogValidator>] The catalog validator class to add

    # File lib/puppet/parser/compiler.rb
104 def add_catalog_validator(catalog_validators)
105   @catalog_validators << catalog_validators
106   nil
107 end
add_catalog_validators() click to toggle source
    # File lib/puppet/parser/compiler.rb
109 def add_catalog_validators
110   add_catalog_validator(CatalogValidator::RelationshipValidator)
111 end
add_class(name) click to toggle source

Store the fact that we've evaluated a class

    # File lib/puppet/parser/compiler.rb
 98 def add_class(name)
 99   @catalog.add_class(name) unless name == ""
100 end
add_override(override) click to toggle source

Store a resource override.

   # File lib/puppet/parser/compiler.rb
62 def add_override(override)
63   # If possible, merge the override in immediately.
64   resource = @catalog.resource(override.ref)
65   if resource
66     resource.merge(override)
67   else
68     # Otherwise, store the override for later; these
69     # get evaluated in Resource#finish.
70     @resource_overrides[override.ref] << override
71   end
72 end
add_resource(scope, resource) click to toggle source
   # File lib/puppet/parser/compiler.rb
74 def add_resource(scope, resource)
75   @resources << resource
76 
77   # Note that this will fail if the resource is not unique.
78   @catalog.add_resource(resource)
79 
80   if not resource.class? and resource[:stage]
81     #TRANSLATORS "stage" is a keyword in Puppet and should not be translated
82     raise ArgumentError, _("Only classes can set 'stage'; normal resources like %{resource} cannot change run stage") % { resource: resource }
83   end
84 
85   # Stages should not be inside of classes.  They are always a
86   # top-level container, regardless of where they appear in the
87   # manifest.
88   return if resource.stage?
89 
90   # This adds a resource to the class it lexically appears in in the
91   # manifest.
92   unless resource.class?
93     @catalog.add_edge(scope.resource, resource)
94   end
95 end
compile() { |catalog| ... } click to toggle source

Compiler our catalog. This mostly revolves around finding and evaluating classes. This is the main entry into our catalog.

    # File lib/puppet/parser/compiler.rb
122 def compile
123   Puppet.override( @context_overrides , _("For compiling %{node}") % { node: node.name }) do
124     @catalog.environment_instance = environment
125 
126     # Set the client's parameters into the top scope.
127     Puppet::Util::Profiler.profile(_("Compile: Set node parameters"), [:compiler, :set_node_params]) { set_node_parameters }
128 
129     Puppet::Util::Profiler.profile(_("Compile: Created settings scope"), [:compiler, :create_settings_scope]) { create_settings_scope }
130 
131     #TRANSLATORS "main" is a function name and should not be translated
132     Puppet::Util::Profiler.profile(_("Compile: Evaluated main"), [:compiler, :evaluate_main]) { evaluate_main }
133 
134     Puppet::Util::Profiler.profile(_("Compile: Evaluated AST node"), [:compiler, :evaluate_ast_node]) { evaluate_ast_node }
135 
136     Puppet::Util::Profiler.profile(_("Compile: Evaluated node classes"), [:compiler, :evaluate_node_classes]) { evaluate_node_classes }
137 
138     Puppet::Util::Profiler.profile(_("Compile: Evaluated generators"), [:compiler, :evaluate_generators]) { evaluate_generators }
139 
140     Puppet::Util::Profiler.profile(_("Compile: Validate Catalog pre-finish"), [:compiler, :validate_pre_finish]) do
141       validate_catalog(CatalogValidator::PRE_FINISH)
142     end
143 
144     Puppet::Util::Profiler.profile(_("Compile: Finished catalog"), [:compiler, :finish_catalog]) { finish }
145 
146     fail_on_unevaluated
147 
148     Puppet::Util::Profiler.profile(_("Compile: Validate Catalog final"), [:compiler, :validate_final]) do
149       validate_catalog(CatalogValidator::FINAL)
150     end
151 
152     if block_given?
153       yield @catalog
154     else
155       @catalog
156     end
157   end
158 end
context_overrides() click to toggle source

Constructs the overrides for the context

    # File lib/puppet/parser/compiler.rb
165 def context_overrides()
166   {
167     :current_environment => environment,
168     :global_scope => @topscope,             # 4x placeholder for new global scope
169     :loaders  => @loaders,                  # 4x loaders
170   }
171 end
environment() click to toggle source

Return the node's environment.

    # File lib/puppet/parser/compiler.rb
176 def environment
177   node.environment
178 end
evaluate_ast_node() click to toggle source

If ast nodes are enabled, then see if we can find and evaluate one.

@api private

    # File lib/puppet/parser/compiler.rb
206 def evaluate_ast_node
207   krt = environment.known_resource_types
208   return unless krt.nodes? #ast_nodes?
209 
210   # Now see if we can find the node.
211   astnode = nil
212   @node.names.each do |name|
213     astnode = krt.node(name.to_s.downcase)
214     break if astnode
215   end
216 
217   unless (astnode ||= krt.node("default"))
218     raise Puppet::ParseError, _("Could not find node statement with name 'default' or '%{names}'") % { names: node.names.join(", ") }
219   end
220 
221   # Create a resource to model this node, and then add it to the list
222   # of resources.
223   resource = astnode.ensure_in_catalog(topscope)
224 
225   resource.evaluate
226 
227   @node_scope = topscope.class_scope(astnode)
228 end
evaluate_classes(classes, scope, lazy_evaluate = true) click to toggle source

Evaluates each specified class in turn. If there are any classes that can't be found, an error is raised. This method really just creates resource objects that point back to the classes, and then the resources are themselves evaluated later in the process.

    # File lib/puppet/parser/compiler.rb
235 def evaluate_classes(classes, scope, lazy_evaluate = true)
236   raise Puppet::DevError, _("No source for scope passed to evaluate_classes") unless scope.source
237   class_parameters = nil
238   # if we are a param class, save the classes hash
239   # and transform classes to be the keys
240   if classes.class == Hash
241     class_parameters = classes
242     classes = classes.keys
243   end
244 
245   hostclasses = classes.collect do |name|
246     environment.known_resource_types.find_hostclass(name) or raise Puppet::Error, _("Could not find class %{name} for %{node}") % { name: name, node: node.name }
247   end
248 
249   if class_parameters
250     resources = ensure_classes_with_parameters(scope, hostclasses, class_parameters)
251     if !lazy_evaluate
252       resources.each(&:evaluate)
253     end
254 
255     resources
256   else
257     already_included, newly_included = ensure_classes_without_parameters(scope, hostclasses)
258     if !lazy_evaluate
259       newly_included.each(&:evaluate)
260     end
261 
262     already_included + newly_included
263   end
264 end
evaluate_node_classes() click to toggle source

Evaluate all of the classes specified by the node. Classes with parameters are evaluated as if they were declared. Classes without parameters or with an empty set of parameters are evaluated as if they were included. This means classes with an empty set of parameters won't conflict even if the class has already been included.

    # File lib/puppet/parser/compiler.rb
185 def evaluate_node_classes
186   if @node.classes.is_a? Hash
187     classes_with_params, classes_without_params = @node.classes.partition {|name,params| params and !params.empty?}
188 
189     # The results from Hash#partition are arrays of pairs rather than hashes,
190     # so we have to convert to the forms evaluate_classes expects (Hash, and
191     # Array of class names)
192     classes_with_params = Hash[classes_with_params]
193     classes_without_params.map!(&:first)
194   else
195     classes_with_params = {}
196     classes_without_params = @node.classes
197   end
198 
199   evaluate_classes(classes_with_params, @node_scope || topscope)
200   evaluate_classes(classes_without_params, @node_scope || topscope)
201 end
evaluate_relationships() click to toggle source
    # File lib/puppet/parser/compiler.rb
266 def evaluate_relationships
267   @relationships.each { |rel| rel.evaluate(catalog) }
268 end
newscope(parent, options = {}) click to toggle source

Create a new scope, with either a specified parent scope or using the top scope.

    # File lib/puppet/parser/compiler.rb
284 def newscope(parent, options = {})
285   parent ||= topscope
286   scope = Puppet::Parser::Scope.new(self, **options)
287   scope.parent = parent
288   scope
289 end
resource_overrides(resource) click to toggle source

Return any overrides for the given resource.

    # File lib/puppet/parser/compiler.rb
292 def resource_overrides(resource)
293   @resource_overrides[resource.ref]
294 end
validate_catalog(validation_stage) click to toggle source
    # File lib/puppet/parser/compiler.rb
160 def validate_catalog(validation_stage)
161   @catalog_validators.select { |vclass| vclass.validation_stage?(validation_stage) }.each { |vclass| vclass.new(@catalog).validate }
162 end
with_context_overrides(description = '', &block) click to toggle source
    # File lib/puppet/parser/compiler.rb
116 def with_context_overrides(description = '', &block)
117   Puppet.override( @context_overrides , description, &block)
118 end

Protected Instance Methods

evaluate_generators() click to toggle source

Iterate over collections and resources until we're sure that the whole compile is evaluated. This is necessary because both collections and defined resources can generate new resources, which themselves could be defined resources.

    # File lib/puppet/parser/compiler.rb
367 def evaluate_generators
368   count = 0
369   loop do
370     done = true
371 
372     Puppet::Util::Profiler.profile(_("Iterated (%{count}) on generators") % { count: count + 1 }, [:compiler, :iterate_on_generators]) do
373       # Call collections first, then definitions.
374       done = false if evaluate_collections
375       done = false if evaluate_definitions
376     end
377 
378     break if done
379 
380     count += 1
381 
382     if count > 1000
383       raise Puppet::ParseError, _("Somehow looped more than 1000 times while evaluating host catalog")
384     end
385   end
386 end
finish() click to toggle source

Make sure all of our resources and such have done any last work necessary.

    # File lib/puppet/parser/compiler.rb
432 def finish
433   evaluate_relationships
434 
435   resources.each do |resource|
436     # Add in any resource overrides.
437     overrides = resource_overrides(resource)
438     if overrides
439       overrides.each do |over|
440         resource.merge(over)
441       end
442 
443       # Remove the overrides, so that the configuration knows there
444       # are none left.
445       overrides.clear
446     end
447 
448     resource.finish if resource.respond_to?(:finish)
449   end
450 
451   add_resource_metaparams
452 end

Private Instance Methods

add_resource_metaparams() click to toggle source
    # File lib/puppet/parser/compiler.rb
455 def add_resource_metaparams
456   main = catalog.resource(:class, :main)
457   unless main
458     #TRANSLATORS "main" is a function name and should not be translated
459     raise _("Couldn't find main")
460   end
461 
462   names = Puppet::Type.metaparams.select do |name|
463     !Puppet::Parser::Resource.relationship_parameter?(name)
464   end
465   data = {}
466   catalog.walk(main, :out) do |source, target|
467     source_data = data[source] || metaparams_as_data(source, names)
468     if source_data
469       # only store anything in the data hash if we've actually got
470       # data
471       data[source] ||= source_data
472       source_data.each do |param, value|
473         target[param] = value if target[param].nil?
474       end
475       data[target] = source_data.merge(metaparams_as_data(target, names))
476     end
477 
478     target.merge_tags_from(source)
479   end
480 end
create_settings_scope() click to toggle source
    # File lib/puppet/parser/compiler.rb
584 def create_settings_scope
585   settings_type = create_settings_type
586   settings_resource = Puppet::Parser::Resource.new('class', SETTINGS, :scope => @topscope)
587 
588   @catalog.add_resource(settings_resource)
589   settings_type.evaluate_code(settings_resource)
590   settings_resource.instance_variable_set(:@evaluated, true) # Prevents settings from being reevaluated
591 
592   scope = @topscope.class_scope(settings_type)
593   scope.merge_settings(environment.name)
594 end
create_settings_type() click to toggle source
    # File lib/puppet/parser/compiler.rb
596 def create_settings_type
597   environment.lock.synchronize do
598     resource_types = environment.known_resource_types
599     settings_type = resource_types.hostclass(SETTINGS)
600     if settings_type.nil?
601       settings_type = Puppet::Resource::Type.new(:hostclass, SETTINGS)
602       resource_types.add(settings_type)
603     end
604 
605     settings_type
606   end
607 end
ensure_classes_with_parameters(scope, hostclasses, parameters) click to toggle source
    # File lib/puppet/parser/compiler.rb
298 def ensure_classes_with_parameters(scope, hostclasses, parameters)
299   hostclasses.collect do |klass|
300     klass.ensure_in_catalog(scope, parameters[klass.name] || {})
301   end
302 end
ensure_classes_without_parameters(scope, hostclasses) click to toggle source
    # File lib/puppet/parser/compiler.rb
304 def ensure_classes_without_parameters(scope, hostclasses)
305   already_included = []
306   newly_included = []
307   hostclasses.each do |klass|
308     class_scope = scope.class_scope(klass)
309     if class_scope
310       already_included << class_scope.resource
311     else
312       newly_included << klass.ensure_in_catalog(scope)
313     end
314   end
315 
316   [already_included, newly_included]
317 end
evaluate_collections() click to toggle source

Evaluate our collections and return true if anything returned an object. The 'true' is used to continue a loop, so it's important.

    # File lib/puppet/parser/compiler.rb
321 def evaluate_collections
322   return false if @collections.empty?
323 
324   exceptwrap do
325     # We have to iterate over a dup of the array because
326     # collections can delete themselves from the list, which
327     # changes its length and causes some collections to get missed.
328     Puppet::Util::Profiler.profile(_("Evaluated collections"), [:compiler, :evaluate_collections]) do
329       found_something = false
330       @collections.dup.each do |collection|
331         found_something = true if collection.evaluate
332       end
333       found_something
334     end
335   end
336 end
evaluate_definitions() click to toggle source

Make sure all of our resources have been evaluated into native resources. We return true if any resources have, so that we know to continue the evaluate_generators loop.

    # File lib/puppet/parser/compiler.rb
341 def evaluate_definitions
342   exceptwrap do
343     Puppet::Util::Profiler.profile(_("Evaluated definitions"), [:compiler, :evaluate_definitions]) do
344       urs = unevaluated_resources.each do |resource|
345         begin
346           resource.evaluate
347         rescue Puppet::Pops::Evaluator::PuppetStopIteration => detail
348           # needs to be handled specifically as the error has the file/line/position where this
349           # occurred rather than the resource
350           fail(Puppet::Pops::Issues::RUNTIME_ERROR, detail, {:detail => detail.message}, detail)
351 
352         rescue Puppet::Error => e
353           # PuppetError has the ability to wrap an exception, if so, use the wrapped exception's
354           # call stack instead
355           fail(Puppet::Pops::Issues::RUNTIME_ERROR, resource, {:detail => e.message}, e.original || e)
356         end
357       end
358       !urs.empty?
359     end
360   end
361 end
evaluate_main() click to toggle source

Find and evaluate our main object, if possible.

    # File lib/puppet/parser/compiler.rb
390 def evaluate_main
391   krt = environment.known_resource_types
392   @main = krt.find_hostclass('') || krt.add(Puppet::Resource::Type.new(:hostclass, ''))
393   @topscope.source = @main
394   @main_resource = Puppet::Parser::Resource.new('class', :main, :scope => @topscope, :source => @main)
395   @topscope.resource = @main_resource
396 
397   add_resource(@topscope, @main_resource)
398 
399   @main_resource.evaluate
400 end
fail_on_unevaluated() click to toggle source

Make sure the entire catalog is evaluated.

    # File lib/puppet/parser/compiler.rb
403 def fail_on_unevaluated
404   fail_on_unevaluated_overrides
405   fail_on_unevaluated_resource_collections
406 end
fail_on_unevaluated_overrides() click to toggle source

If there are any resource overrides remaining, then we could not find the resource they were supposed to override, so we want to throw an exception.

    # File lib/puppet/parser/compiler.rb
411 def fail_on_unevaluated_overrides
412   remaining = @resource_overrides.values.flatten.collect(&:ref)
413 
414   if !remaining.empty?
415     raise Puppet::ParseError, _("Could not find resource(s) %{resources} for overriding") % { resources: remaining.join(', ') }
416   end
417 end
fail_on_unevaluated_resource_collections() click to toggle source

Make sure there are no remaining collections that are waiting for resources that have not yet been instantiated. If this occurs it is an error (missing resource - it could not be realized).

    # File lib/puppet/parser/compiler.rb
423 def fail_on_unevaluated_resource_collections
424   remaining = @collections.collect(&:unresolved_resources).flatten.compact
425   if !remaining.empty?
426     raise Puppet::ParseError, _("Failed to realize virtual resources %{resources}") % { resources: remaining.join(', ') }
427   end
428 end
initvars() click to toggle source

Set up all of our internal variables.

    # File lib/puppet/parser/compiler.rb
498 def initvars
499   # The list of overrides.  This is used to cache overrides on objects
500   # that don't exist yet.  We store an array of each override.
501   @resource_overrides = Hash.new do |overs, ref|
502     overs[ref] = []
503   end
504 
505   # The list of collections that have been created.  This is a global list,
506   # but they each refer back to the scope that created them.
507   @collections = []
508 
509   # The list of relationships to evaluate.
510   @relationships = []
511 
512   # For maintaining the relationship between scopes and their resources.
513   @catalog = Puppet::Resource::Catalog.new(@node.name, @node.environment, @code_id)
514 
515   # MOVED HERE - SCOPE IS NEEDED (MOVE-SCOPE)
516   # Create the initial scope, it is needed early
517   @topscope = Puppet::Parser::Scope.new(self)
518 
519   # Initialize loaders and Pcore
520   @loaders = Puppet::Pops::Loaders.new(environment)
521 
522   # Need to compute overrides here, and remember them, because we are about to
523   # enter the magic zone of known_resource_types and initial import.
524   # Expensive entries in the context are bound lazily.
525   @context_overrides = context_overrides()
526 
527   # This construct ensures that initial import (triggered by instantiating
528   # the structure 'known_resource_types') has a configured context
529   # It cannot survive the initvars method, and is later reinstated
530   # as part of compiling...
531   #
532   Puppet.override( @context_overrides , _("For initializing compiler")) do
533     # THE MAGIC STARTS HERE ! This triggers parsing, loading etc.
534     @catalog.version = environment.known_resource_types.version
535     @loaders.pre_load
536   end
537 
538   @catalog.add_resource(Puppet::Parser::Resource.new("stage", :main, :scope => @topscope))
539 
540   # local resource array to maintain resource ordering
541   @resources = []
542 
543   # Make sure any external node classes are in our class list
544   if @node.classes.class == Hash
545     @catalog.add_class(*@node.classes.keys)
546   else
547     @catalog.add_class(*@node.classes)
548   end
549 
550   @catalog_validators = []
551 end
metaparams_as_data(resource, params) click to toggle source
    # File lib/puppet/parser/compiler.rb
482 def metaparams_as_data(resource, params)
483   data = nil
484   params.each do |param|
485     unless resource[param].nil?
486       # Because we could be creating a hash for every resource,
487       # and we actually probably don't often have any data here at all,
488       # we're optimizing a bit by only creating a hash if there's
489       # any data to put in it.
490       data ||= {}
491       data[param] = resource[param]
492     end
493   end
494   data
495 end
sanitize_node(node) click to toggle source
    # File lib/puppet/parser/compiler.rb
553 def sanitize_node(node)
554   node.sanitize
555   node
556 end
set_node_parameters() click to toggle source

Set the node's parameters into the top-scope as variables.

    # File lib/puppet/parser/compiler.rb
559 def set_node_parameters
560   node.parameters.each do |param, value|
561     # We don't want to set @topscope['environment'] from the parameters,
562     # instead we want to get that from the node's environment itself in
563     # case a custom node terminus has done any mucking about with
564     # node.parameters.
565     next if param.to_s == 'environment'
566     # Ensure node does not leak Symbol instances in general
567     @topscope[param.to_s] = value.is_a?(Symbol) ? value.to_s : value
568   end
569   @topscope['environment'] = node.environment.name.to_s
570 
571   # These might be nil.
572   catalog.client_version = node.parameters["clientversion"]
573   catalog.server_version = node.parameters["serverversion"]
574   @topscope.set_trusted(node.trusted_data)
575 
576   @topscope.set_server_facts(node.server_facts)
577 
578   facts_hash = node.facts.nil? ? {} : node.facts.values
579   @topscope.set_facts(facts_hash)
580 end
unevaluated_resources() click to toggle source

Return an array of all of the unevaluated resources. These will be definitions, which need to get evaluated into native resources.

    # File lib/puppet/parser/compiler.rb
611 def unevaluated_resources
612   # The order of these is significant for speed due to short-circuiting
613   resources.reject { |resource| resource.evaluated? or resource.virtual? or resource.builtin_type? }
614 end