class Puppet::Parser::Compiler
Maintain a graph of scopes, along with a bunch of data about the individual catalog we're compiling.
Abstract class for a catalog validator that can be registered with the compiler to run at a certain stage.
Constants
- SETTINGS
Attributes
The id of code input to the compiler. @api private
Access to the configured loaders for 4x @return [Puppet::Pops::Loader::Loaders] the configured loaders @api private
Public Class Methods
# File lib/puppet/parser/compiler.rb 21 def self.compile(node, code_id = nil) 22 node.environment.check_for_reparse 23 24 errors = node.environment.validation_errors 25 if !errors.empty? 26 errors.each { |e| Puppet.err(e) } if errors.size > 1 27 errmsg = [ 28 _("Compilation has been halted because: %{error}") % { error: errors.first }, 29 _("For more information, see https://puppet.com/docs/puppet/latest/environments_about.html"), 30 ] 31 raise(Puppet::Error, errmsg.join(' ')) 32 end 33 34 new(node, :code_id => code_id).compile {|resulting_catalog| resulting_catalog.to_resource } 35 rescue Puppet::ParseErrorWithIssue => detail 36 detail.node = node.name 37 Puppet.log_exception(detail) 38 raise 39 rescue => detail 40 message = _("%{message} on node %{node}") % { message: detail, node: node.name } 41 Puppet.log_exception(detail, message) 42 raise Puppet::Error, message, detail.backtrace 43 end
# File lib/puppet/parser/compiler.rb 273 def initialize(node, code_id: nil) 274 @node = sanitize_node(node) 275 @code_id = code_id 276 initvars 277 add_catalog_validators 278 # Resolutions of fully qualified variable names 279 @qualified_variables = {} 280 end
Public Instance Methods
Add a catalog validator that will run at some stage to this compiler @param catalog_validators [Class<CatalogValidator>] The catalog validator class to add
# File lib/puppet/parser/compiler.rb 104 def add_catalog_validator(catalog_validators) 105 @catalog_validators << catalog_validators 106 nil 107 end
# File lib/puppet/parser/compiler.rb 109 def add_catalog_validators 110 add_catalog_validator(CatalogValidator::RelationshipValidator) 111 end
Store the fact that we've evaluated a class
# File lib/puppet/parser/compiler.rb 98 def add_class(name) 99 @catalog.add_class(name) unless name == "" 100 end
Store a resource override.
# File lib/puppet/parser/compiler.rb 62 def add_override(override) 63 # If possible, merge the override in immediately. 64 resource = @catalog.resource(override.ref) 65 if resource 66 resource.merge(override) 67 else 68 # Otherwise, store the override for later; these 69 # get evaluated in Resource#finish. 70 @resource_overrides[override.ref] << override 71 end 72 end
# File lib/puppet/parser/compiler.rb 74 def add_resource(scope, resource) 75 @resources << resource 76 77 # Note that this will fail if the resource is not unique. 78 @catalog.add_resource(resource) 79 80 if not resource.class? and resource[:stage] 81 #TRANSLATORS "stage" is a keyword in Puppet and should not be translated 82 raise ArgumentError, _("Only classes can set 'stage'; normal resources like %{resource} cannot change run stage") % { resource: resource } 83 end 84 85 # Stages should not be inside of classes. They are always a 86 # top-level container, regardless of where they appear in the 87 # manifest. 88 return if resource.stage? 89 90 # This adds a resource to the class it lexically appears in in the 91 # manifest. 92 unless resource.class? 93 @catalog.add_edge(scope.resource, resource) 94 end 95 end
Compiler
our catalog. This mostly revolves around finding and evaluating classes. This is the main entry into our catalog.
# File lib/puppet/parser/compiler.rb 122 def compile 123 Puppet.override( @context_overrides , _("For compiling %{node}") % { node: node.name }) do 124 @catalog.environment_instance = environment 125 126 # Set the client's parameters into the top scope. 127 Puppet::Util::Profiler.profile(_("Compile: Set node parameters"), [:compiler, :set_node_params]) { set_node_parameters } 128 129 Puppet::Util::Profiler.profile(_("Compile: Created settings scope"), [:compiler, :create_settings_scope]) { create_settings_scope } 130 131 #TRANSLATORS "main" is a function name and should not be translated 132 Puppet::Util::Profiler.profile(_("Compile: Evaluated main"), [:compiler, :evaluate_main]) { evaluate_main } 133 134 Puppet::Util::Profiler.profile(_("Compile: Evaluated AST node"), [:compiler, :evaluate_ast_node]) { evaluate_ast_node } 135 136 Puppet::Util::Profiler.profile(_("Compile: Evaluated node classes"), [:compiler, :evaluate_node_classes]) { evaluate_node_classes } 137 138 Puppet::Util::Profiler.profile(_("Compile: Evaluated generators"), [:compiler, :evaluate_generators]) { evaluate_generators } 139 140 Puppet::Util::Profiler.profile(_("Compile: Validate Catalog pre-finish"), [:compiler, :validate_pre_finish]) do 141 validate_catalog(CatalogValidator::PRE_FINISH) 142 end 143 144 Puppet::Util::Profiler.profile(_("Compile: Finished catalog"), [:compiler, :finish_catalog]) { finish } 145 146 fail_on_unevaluated 147 148 Puppet::Util::Profiler.profile(_("Compile: Validate Catalog final"), [:compiler, :validate_final]) do 149 validate_catalog(CatalogValidator::FINAL) 150 end 151 152 if block_given? 153 yield @catalog 154 else 155 @catalog 156 end 157 end 158 end
Constructs the overrides for the context
# File lib/puppet/parser/compiler.rb 165 def context_overrides() 166 { 167 :current_environment => environment, 168 :global_scope => @topscope, # 4x placeholder for new global scope 169 :loaders => @loaders, # 4x loaders 170 } 171 end
Return the node's environment.
# File lib/puppet/parser/compiler.rb 176 def environment 177 node.environment 178 end
If ast nodes are enabled, then see if we can find and evaluate one.
@api private
# File lib/puppet/parser/compiler.rb 206 def evaluate_ast_node 207 krt = environment.known_resource_types 208 return unless krt.nodes? #ast_nodes? 209 210 # Now see if we can find the node. 211 astnode = nil 212 @node.names.each do |name| 213 astnode = krt.node(name.to_s.downcase) 214 break if astnode 215 end 216 217 unless (astnode ||= krt.node("default")) 218 raise Puppet::ParseError, _("Could not find node statement with name 'default' or '%{names}'") % { names: node.names.join(", ") } 219 end 220 221 # Create a resource to model this node, and then add it to the list 222 # of resources. 223 resource = astnode.ensure_in_catalog(topscope) 224 225 resource.evaluate 226 227 @node_scope = topscope.class_scope(astnode) 228 end
Evaluates each specified class in turn. If there are any classes that can't be found, an error is raised. This method really just creates resource objects that point back to the classes, and then the resources are themselves evaluated later in the process.
# File lib/puppet/parser/compiler.rb 235 def evaluate_classes(classes, scope, lazy_evaluate = true) 236 raise Puppet::DevError, _("No source for scope passed to evaluate_classes") unless scope.source 237 class_parameters = nil 238 # if we are a param class, save the classes hash 239 # and transform classes to be the keys 240 if classes.class == Hash 241 class_parameters = classes 242 classes = classes.keys 243 end 244 245 hostclasses = classes.collect do |name| 246 environment.known_resource_types.find_hostclass(name) or raise Puppet::Error, _("Could not find class %{name} for %{node}") % { name: name, node: node.name } 247 end 248 249 if class_parameters 250 resources = ensure_classes_with_parameters(scope, hostclasses, class_parameters) 251 if !lazy_evaluate 252 resources.each(&:evaluate) 253 end 254 255 resources 256 else 257 already_included, newly_included = ensure_classes_without_parameters(scope, hostclasses) 258 if !lazy_evaluate 259 newly_included.each(&:evaluate) 260 end 261 262 already_included + newly_included 263 end 264 end
Evaluate all of the classes specified by the node. Classes with parameters are evaluated as if they were declared. Classes without parameters or with an empty set of parameters are evaluated as if they were included. This means classes with an empty set of parameters won't conflict even if the class has already been included.
# File lib/puppet/parser/compiler.rb 185 def evaluate_node_classes 186 if @node.classes.is_a? Hash 187 classes_with_params, classes_without_params = @node.classes.partition {|name,params| params and !params.empty?} 188 189 # The results from Hash#partition are arrays of pairs rather than hashes, 190 # so we have to convert to the forms evaluate_classes expects (Hash, and 191 # Array of class names) 192 classes_with_params = Hash[classes_with_params] 193 classes_without_params.map!(&:first) 194 else 195 classes_with_params = {} 196 classes_without_params = @node.classes 197 end 198 199 evaluate_classes(classes_with_params, @node_scope || topscope) 200 evaluate_classes(classes_without_params, @node_scope || topscope) 201 end
# File lib/puppet/parser/compiler.rb 266 def evaluate_relationships 267 @relationships.each { |rel| rel.evaluate(catalog) } 268 end
Create a new scope, with either a specified parent scope or using the top scope.
# File lib/puppet/parser/compiler.rb 284 def newscope(parent, options = {}) 285 parent ||= topscope 286 scope = Puppet::Parser::Scope.new(self, **options) 287 scope.parent = parent 288 scope 289 end
Return any overrides for the given resource.
# File lib/puppet/parser/compiler.rb 292 def resource_overrides(resource) 293 @resource_overrides[resource.ref] 294 end
# File lib/puppet/parser/compiler.rb 160 def validate_catalog(validation_stage) 161 @catalog_validators.select { |vclass| vclass.validation_stage?(validation_stage) }.each { |vclass| vclass.new(@catalog).validate } 162 end
# File lib/puppet/parser/compiler.rb 116 def with_context_overrides(description = '', &block) 117 Puppet.override( @context_overrides , description, &block) 118 end
Protected Instance Methods
Iterate over collections and resources until we're sure that the whole compile is evaluated. This is necessary because both collections and defined resources can generate new resources, which themselves could be defined resources.
# File lib/puppet/parser/compiler.rb 367 def evaluate_generators 368 count = 0 369 loop do 370 done = true 371 372 Puppet::Util::Profiler.profile(_("Iterated (%{count}) on generators") % { count: count + 1 }, [:compiler, :iterate_on_generators]) do 373 # Call collections first, then definitions. 374 done = false if evaluate_collections 375 done = false if evaluate_definitions 376 end 377 378 break if done 379 380 count += 1 381 382 if count > 1000 383 raise Puppet::ParseError, _("Somehow looped more than 1000 times while evaluating host catalog") 384 end 385 end 386 end
Make sure all of our resources and such have done any last work necessary.
# File lib/puppet/parser/compiler.rb 432 def finish 433 evaluate_relationships 434 435 resources.each do |resource| 436 # Add in any resource overrides. 437 overrides = resource_overrides(resource) 438 if overrides 439 overrides.each do |over| 440 resource.merge(over) 441 end 442 443 # Remove the overrides, so that the configuration knows there 444 # are none left. 445 overrides.clear 446 end 447 448 resource.finish if resource.respond_to?(:finish) 449 end 450 451 add_resource_metaparams 452 end
Private Instance Methods
# File lib/puppet/parser/compiler.rb 455 def add_resource_metaparams 456 main = catalog.resource(:class, :main) 457 unless main 458 #TRANSLATORS "main" is a function name and should not be translated 459 raise _("Couldn't find main") 460 end 461 462 names = Puppet::Type.metaparams.select do |name| 463 !Puppet::Parser::Resource.relationship_parameter?(name) 464 end 465 data = {} 466 catalog.walk(main, :out) do |source, target| 467 source_data = data[source] || metaparams_as_data(source, names) 468 if source_data 469 # only store anything in the data hash if we've actually got 470 # data 471 data[source] ||= source_data 472 source_data.each do |param, value| 473 target[param] = value if target[param].nil? 474 end 475 data[target] = source_data.merge(metaparams_as_data(target, names)) 476 end 477 478 target.merge_tags_from(source) 479 end 480 end
# File lib/puppet/parser/compiler.rb 584 def create_settings_scope 585 settings_type = create_settings_type 586 settings_resource = Puppet::Parser::Resource.new('class', SETTINGS, :scope => @topscope) 587 588 @catalog.add_resource(settings_resource) 589 settings_type.evaluate_code(settings_resource) 590 settings_resource.instance_variable_set(:@evaluated, true) # Prevents settings from being reevaluated 591 592 scope = @topscope.class_scope(settings_type) 593 scope.merge_settings(environment.name) 594 end
# File lib/puppet/parser/compiler.rb 596 def create_settings_type 597 environment.lock.synchronize do 598 resource_types = environment.known_resource_types 599 settings_type = resource_types.hostclass(SETTINGS) 600 if settings_type.nil? 601 settings_type = Puppet::Resource::Type.new(:hostclass, SETTINGS) 602 resource_types.add(settings_type) 603 end 604 605 settings_type 606 end 607 end
# File lib/puppet/parser/compiler.rb 298 def ensure_classes_with_parameters(scope, hostclasses, parameters) 299 hostclasses.collect do |klass| 300 klass.ensure_in_catalog(scope, parameters[klass.name] || {}) 301 end 302 end
# File lib/puppet/parser/compiler.rb 304 def ensure_classes_without_parameters(scope, hostclasses) 305 already_included = [] 306 newly_included = [] 307 hostclasses.each do |klass| 308 class_scope = scope.class_scope(klass) 309 if class_scope 310 already_included << class_scope.resource 311 else 312 newly_included << klass.ensure_in_catalog(scope) 313 end 314 end 315 316 [already_included, newly_included] 317 end
Evaluate our collections and return true if anything returned an object. The 'true' is used to continue a loop, so it's important.
# File lib/puppet/parser/compiler.rb 321 def evaluate_collections 322 return false if @collections.empty? 323 324 exceptwrap do 325 # We have to iterate over a dup of the array because 326 # collections can delete themselves from the list, which 327 # changes its length and causes some collections to get missed. 328 Puppet::Util::Profiler.profile(_("Evaluated collections"), [:compiler, :evaluate_collections]) do 329 found_something = false 330 @collections.dup.each do |collection| 331 found_something = true if collection.evaluate 332 end 333 found_something 334 end 335 end 336 end
Make sure all of our resources have been evaluated into native resources. We return true if any resources have, so that we know to continue the evaluate_generators
loop.
# File lib/puppet/parser/compiler.rb 341 def evaluate_definitions 342 exceptwrap do 343 Puppet::Util::Profiler.profile(_("Evaluated definitions"), [:compiler, :evaluate_definitions]) do 344 urs = unevaluated_resources.each do |resource| 345 begin 346 resource.evaluate 347 rescue Puppet::Pops::Evaluator::PuppetStopIteration => detail 348 # needs to be handled specifically as the error has the file/line/position where this 349 # occurred rather than the resource 350 fail(Puppet::Pops::Issues::RUNTIME_ERROR, detail, {:detail => detail.message}, detail) 351 352 rescue Puppet::Error => e 353 # PuppetError has the ability to wrap an exception, if so, use the wrapped exception's 354 # call stack instead 355 fail(Puppet::Pops::Issues::RUNTIME_ERROR, resource, {:detail => e.message}, e.original || e) 356 end 357 end 358 !urs.empty? 359 end 360 end 361 end
Find and evaluate our main object, if possible.
# File lib/puppet/parser/compiler.rb 390 def evaluate_main 391 krt = environment.known_resource_types 392 @main = krt.find_hostclass('') || krt.add(Puppet::Resource::Type.new(:hostclass, '')) 393 @topscope.source = @main 394 @main_resource = Puppet::Parser::Resource.new('class', :main, :scope => @topscope, :source => @main) 395 @topscope.resource = @main_resource 396 397 add_resource(@topscope, @main_resource) 398 399 @main_resource.evaluate 400 end
Make sure the entire catalog is evaluated.
# File lib/puppet/parser/compiler.rb 403 def fail_on_unevaluated 404 fail_on_unevaluated_overrides 405 fail_on_unevaluated_resource_collections 406 end
If there are any resource overrides remaining, then we could not find the resource they were supposed to override, so we want to throw an exception.
# File lib/puppet/parser/compiler.rb 411 def fail_on_unevaluated_overrides 412 remaining = @resource_overrides.values.flatten.collect(&:ref) 413 414 if !remaining.empty? 415 raise Puppet::ParseError, _("Could not find resource(s) %{resources} for overriding") % { resources: remaining.join(', ') } 416 end 417 end
Make sure there are no remaining collections that are waiting for resources that have not yet been instantiated. If this occurs it is an error (missing resource - it could not be realized).
# File lib/puppet/parser/compiler.rb 423 def fail_on_unevaluated_resource_collections 424 remaining = @collections.collect(&:unresolved_resources).flatten.compact 425 if !remaining.empty? 426 raise Puppet::ParseError, _("Failed to realize virtual resources %{resources}") % { resources: remaining.join(', ') } 427 end 428 end
Set up all of our internal variables.
# File lib/puppet/parser/compiler.rb 498 def initvars 499 # The list of overrides. This is used to cache overrides on objects 500 # that don't exist yet. We store an array of each override. 501 @resource_overrides = Hash.new do |overs, ref| 502 overs[ref] = [] 503 end 504 505 # The list of collections that have been created. This is a global list, 506 # but they each refer back to the scope that created them. 507 @collections = [] 508 509 # The list of relationships to evaluate. 510 @relationships = [] 511 512 # For maintaining the relationship between scopes and their resources. 513 @catalog = Puppet::Resource::Catalog.new(@node.name, @node.environment, @code_id) 514 515 # MOVED HERE - SCOPE IS NEEDED (MOVE-SCOPE) 516 # Create the initial scope, it is needed early 517 @topscope = Puppet::Parser::Scope.new(self) 518 519 # Initialize loaders and Pcore 520 @loaders = Puppet::Pops::Loaders.new(environment) 521 522 # Need to compute overrides here, and remember them, because we are about to 523 # enter the magic zone of known_resource_types and initial import. 524 # Expensive entries in the context are bound lazily. 525 @context_overrides = context_overrides() 526 527 # This construct ensures that initial import (triggered by instantiating 528 # the structure 'known_resource_types') has a configured context 529 # It cannot survive the initvars method, and is later reinstated 530 # as part of compiling... 531 # 532 Puppet.override( @context_overrides , _("For initializing compiler")) do 533 # THE MAGIC STARTS HERE ! This triggers parsing, loading etc. 534 @catalog.version = environment.known_resource_types.version 535 @loaders.pre_load 536 end 537 538 @catalog.add_resource(Puppet::Parser::Resource.new("stage", :main, :scope => @topscope)) 539 540 # local resource array to maintain resource ordering 541 @resources = [] 542 543 # Make sure any external node classes are in our class list 544 if @node.classes.class == Hash 545 @catalog.add_class(*@node.classes.keys) 546 else 547 @catalog.add_class(*@node.classes) 548 end 549 550 @catalog_validators = [] 551 end
# File lib/puppet/parser/compiler.rb 482 def metaparams_as_data(resource, params) 483 data = nil 484 params.each do |param| 485 unless resource[param].nil? 486 # Because we could be creating a hash for every resource, 487 # and we actually probably don't often have any data here at all, 488 # we're optimizing a bit by only creating a hash if there's 489 # any data to put in it. 490 data ||= {} 491 data[param] = resource[param] 492 end 493 end 494 data 495 end
# File lib/puppet/parser/compiler.rb 553 def sanitize_node(node) 554 node.sanitize 555 node 556 end
Set the node's parameters into the top-scope as variables.
# File lib/puppet/parser/compiler.rb 559 def set_node_parameters 560 node.parameters.each do |param, value| 561 # We don't want to set @topscope['environment'] from the parameters, 562 # instead we want to get that from the node's environment itself in 563 # case a custom node terminus has done any mucking about with 564 # node.parameters. 565 next if param.to_s == 'environment' 566 # Ensure node does not leak Symbol instances in general 567 @topscope[param.to_s] = value.is_a?(Symbol) ? value.to_s : value 568 end 569 @topscope['environment'] = node.environment.name.to_s 570 571 # These might be nil. 572 catalog.client_version = node.parameters["clientversion"] 573 catalog.server_version = node.parameters["serverversion"] 574 @topscope.set_trusted(node.trusted_data) 575 576 @topscope.set_server_facts(node.server_facts) 577 578 facts_hash = node.facts.nil? ? {} : node.facts.values 579 @topscope.set_facts(facts_hash) 580 end
Return an array of all of the unevaluated resources. These will be definitions, which need to get evaluated into native resources.
# File lib/puppet/parser/compiler.rb 611 def unevaluated_resources 612 # The order of these is significant for speed due to short-circuiting 613 resources.reject { |resource| resource.evaluated? or resource.virtual? or resource.builtin_type? } 614 end