class Puppet::Pops::Lookup::LookupAdapter
A LookupAdapter
is a specialized DataAdapter
that uses its hash to store data providers. It also remembers the compiler that it is attached to and maintains a cache of _lookup options_ retrieved from the data providers associated with the compiler's environment.
@api private
Constants
- CONVERT_TO
- GLOBAL_ENV_MERGE
- HASH
- LOOKUP_OPTIONS_PATTERN_START
- LOOKUP_OPTIONS_PREFIX
- MERGE
- NEW
- PROVIDER_STACK
Public Class Methods
# File lib/puppet/pops/lookup/lookup_adapter.rb 22 def self.create_adapter(compiler) 23 new(compiler) 24 end
Puppet::Pops::Lookup::DataAdapter::new
# File lib/puppet/pops/lookup/lookup_adapter.rb 26 def initialize(compiler) 27 super() 28 @compiler = compiler 29 @lookup_options = {} 30 # Get a KeyRecorder from context, and set a "null recorder" if not defined 31 @key_recorder = Puppet.lookup(:lookup_key_recorder) { KeyRecorder.singleton } 32 end
Public Instance Methods
Performs a possible conversion of the result of calling `the_lookup` lambda The conversion takes place if there is a 'convert_to' key in the lookup_options If there is no conversion, the result of calling `the_lookup` is returned otherwise the successfully converted value. Errors are raised if the convert_to is faulty (bad type string, or if a call to new(T, <args>) fails.
@param key [String] The key to lookup @param lookup_options [Hash] a hash of options @param lookup_invocation [Invocation] the lookup invocation @param the_lookup [Lambda] zero arg lambda that performs the lookup of a value @return [Object] the looked up value, or converted value if there was conversion @throw :no_such_key when the object is not found (if thrown by `the_lookup`)
# File lib/puppet/pops/lookup/lookup_adapter.rb 97 def convert_result(key, lookup_options, lookup_invocation, the_lookup) 98 result = the_lookup.call 99 convert_to = lookup_options[CONVERT_TO] 100 return result if convert_to.nil? 101 102 convert_to = convert_to.is_a?(Array) ? convert_to : [convert_to] 103 if convert_to[0].is_a?(String) 104 begin 105 convert_to[0] = Puppet::Pops::Types::TypeParser.singleton.parse(convert_to[0]) 106 rescue StandardError => e 107 raise Puppet::DataBinding::LookupError, 108 _("Invalid data type in lookup_options for key '%{key}' could not parse '%{source}', error: '%{msg}") % 109 { key: key, source: convert_to[0], msg: e.message} 110 end 111 end 112 begin 113 result = lookup_invocation.scope.call_function(NEW, [convert_to[0], result, *convert_to[1..-1]]) 114 # TRANSLATORS 'lookup_options', 'convert_to' and args_string variable should not be translated, 115 args_string = Puppet::Pops::Types::StringConverter.singleton.convert(convert_to) 116 lookup_invocation.report_text { _("Applying convert_to lookup_option with arguments %{args}") % { args: args_string } } 117 rescue StandardError => e 118 raise Puppet::DataBinding::LookupError, 119 _("The convert_to lookup_option for key '%{key}' raised error: %{msg}") % 120 { key: key, msg: e.message} 121 end 122 result 123 end
# File lib/puppet/pops/lookup/lookup_adapter.rb 245 def extract_lookup_options_for_key(key, options) 246 return nil if options.nil? 247 248 rk = key.root_key 249 key_opts = options[0] 250 unless key_opts.nil? 251 key_opt = key_opts[rk] 252 return key_opt unless key_opt.nil? 253 end 254 255 patterns = options[1] 256 patterns.each_pair { |pattern, value| return value if pattern =~ rk } unless patterns.nil? 257 nil 258 end
@return [Pathname] the full path of the hiera.yaml config file
# File lib/puppet/pops/lookup/lookup_adapter.rb 268 def global_hiera_config_path 269 @global_hiera_config_path ||= Pathname.new(Puppet.settings[:hiera_config]) 270 end
# File lib/puppet/pops/lookup/lookup_adapter.rb 279 def global_only? 280 instance_variable_defined?(:@global_only) ? @global_only : false 281 end
@param lookup_invocation [Puppet::Pops::Lookup::Invocation] the lookup invocation @return [Boolean] `true` if an environment data provider version 5 is configured
# File lib/puppet/pops/lookup/lookup_adapter.rb 262 def has_environment_data_provider?(lookup_invocation) 263 ep = env_provider(lookup_invocation) 264 ep.nil? ? false : ep.config(lookup_invocation).version >= 5 265 end
Performs a lookup using global, environment, and module data providers. Merge the result using the given merge strategy. If the merge strategy is nil, then an attempt is made to find merge options in the `lookup_options` hash for an entry associated with the key. If no options are found, the no merge is performed and the first found entry is returned.
@param key [String] The key to lookup @param lookup_invocation [Invocation] the lookup invocation @param merge [MergeStrategy,String,Hash{String => Object},nil] Merge strategy, merge strategy name, strategy and options hash, or nil (implies “first found”) @return [Object] the found object @throw :no_such_key when the object is not found
# File lib/puppet/pops/lookup/lookup_adapter.rb 45 def lookup(key, lookup_invocation, merge) 46 # The 'lookup_options' key is reserved and not found as normal data 47 if key == LOOKUP_OPTIONS || key.start_with?(LOOKUP_OPTIONS_PREFIX) 48 lookup_invocation.with(:invalid_key, LOOKUP_OPTIONS) do 49 throw :no_such_key 50 end 51 end 52 53 # Record that the key was looked up. This will record all keys for which a lookup is performed 54 # except 'lookup_options' (since that is illegal from a user perspective, 55 # and from an impact perspective is always looked up). 56 @key_recorder.record(key) 57 58 key = LookupKey.new(key) 59 lookup_invocation.lookup(key, key.module_name) do 60 if lookup_invocation.only_explain_options? 61 catch(:no_such_key) { do_lookup(LookupKey::LOOKUP_OPTIONS, lookup_invocation, HASH) } 62 nil 63 else 64 lookup_options = lookup_lookup_options(key, lookup_invocation) || {} 65 66 if merge.nil? 67 # Used cached lookup_options 68 # merge = lookup_merge_options(key, lookup_invocation) 69 merge = lookup_options[MERGE] 70 lookup_invocation.report_merge_source(LOOKUP_OPTIONS) unless merge.nil? 71 end 72 convert_result(key.to_s, lookup_options, lookup_invocation, lambda do 73 lookup_invocation.with(:data, key.to_s) do 74 catch(:no_such_key) { return do_lookup(key, lookup_invocation, merge) } 75 throw :no_such_key if lookup_invocation.global_only? 76 key.dig(lookup_invocation, lookup_default_in_module(key, lookup_invocation)) 77 end 78 end) 79 end 80 end 81 end
# File lib/puppet/pops/lookup/lookup_adapter.rb 178 def lookup_default_in_module(key, lookup_invocation) 179 module_name = lookup_invocation.module_name 180 181 # Do not attempt to do a lookup in a module unless the name is qualified. 182 throw :no_such_key if module_name.nil? 183 184 provider = module_provider(lookup_invocation, module_name) 185 throw :no_such_key if provider.nil? || !provider.config(lookup_invocation).has_default_hierarchy? 186 187 lookup_invocation.with(:scope, "Searching default_hierarchy of module \"#{module_name}\"") do 188 merge_strategy = nil 189 if merge_strategy.nil? 190 @module_default_lookup_options ||= {} 191 options = @module_default_lookup_options.fetch(module_name) do |k| 192 meta_invocation = Invocation.new(lookup_invocation.scope) 193 meta_invocation.lookup(LookupKey::LOOKUP_OPTIONS, k) do 194 opts = nil 195 lookup_invocation.with(:scope, "Searching for \"#{LookupKey::LOOKUP_OPTIONS}\"") do 196 catch(:no_such_key) do 197 opts = compile_patterns( 198 validate_lookup_options( 199 provider.key_lookup_in_default(LookupKey::LOOKUP_OPTIONS, meta_invocation, MergeStrategy.strategy(HASH)), k)) 200 end 201 end 202 @module_default_lookup_options[k] = opts 203 end 204 end 205 lookup_options = extract_lookup_options_for_key(key, options) 206 merge_strategy = lookup_options[MERGE] unless lookup_options.nil? 207 end 208 209 lookup_invocation.with(:scope, "Searching for \"#{key}\"") do 210 provider.key_lookup_in_default(key, lookup_invocation, merge_strategy) 211 end 212 end 213 end
# File lib/puppet/pops/lookup/lookup_adapter.rb 125 def lookup_global(key, lookup_invocation, merge_strategy) 126 # hiera_xxx will always use global_provider regardless of data_binding_terminus setting 127 terminus = lookup_invocation.hiera_xxx_call? ? :hiera : Puppet[:data_binding_terminus] 128 case terminus 129 when :hiera, 'hiera' 130 provider = global_provider(lookup_invocation) 131 throw :no_such_key if provider.nil? 132 provider.key_lookup(key, lookup_invocation, merge_strategy) 133 when :none, 'none', '', nil 134 # If global lookup is disabled, immediately report as not found 135 lookup_invocation.report_not_found(key) 136 throw :no_such_key 137 else 138 lookup_invocation.with(:global, terminus) do 139 catch(:no_such_key) do 140 return lookup_invocation.report_found(key, Puppet::DataBinding.indirection.find(key.root_key, 141 {:environment => environment, :variables => lookup_invocation.scope, :merge => merge_strategy})) 142 end 143 lookup_invocation.report_not_found(key) 144 throw :no_such_key 145 end 146 end 147 rescue Puppet::DataBinding::LookupError => detail 148 raise detail unless detail.issue_code.nil? 149 error = Puppet::Error.new(_("Lookup of key '%{key}' failed: %{detail}") % { key: lookup_invocation.top_key, detail: detail.message }) 150 error.set_backtrace(detail.backtrace) 151 raise error 152 end
# File lib/puppet/pops/lookup/lookup_adapter.rb 154 def lookup_in_environment(key, lookup_invocation, merge_strategy) 155 provider = env_provider(lookup_invocation) 156 throw :no_such_key if provider.nil? 157 provider.key_lookup(key, lookup_invocation, merge_strategy) 158 end
# File lib/puppet/pops/lookup/lookup_adapter.rb 160 def lookup_in_module(key, lookup_invocation, merge_strategy) 161 module_name = lookup_invocation.module_name 162 163 # Do not attempt to do a lookup in a module unless the name is qualified. 164 throw :no_such_key if module_name.nil? 165 166 provider = module_provider(lookup_invocation, module_name) 167 if provider.nil? 168 if environment.module(module_name).nil? 169 lookup_invocation.report_module_not_found(module_name) 170 else 171 lookup_invocation.report_module_provider_not_found(module_name) 172 end 173 throw :no_such_key 174 end 175 provider.key_lookup(key, lookup_invocation, merge_strategy) 176 end
Retrieve the lookup options that match the given `name`.
@param key [LookupKey] The key for which we want lookup options @param lookup_invocation [Puppet::Pops::Lookup::Invocation] the lookup invocation @return [String,Hash,nil] The found lookup options or nil
# File lib/puppet/pops/lookup/lookup_adapter.rb 232 def lookup_lookup_options(key, lookup_invocation) 233 module_name = key.module_name 234 235 # Retrieve the options for the module. We use nil as a key in case we have no module 236 if !@lookup_options.include?(module_name) 237 options = retrieve_lookup_options(module_name, lookup_invocation, MergeStrategy.strategy(HASH)) 238 @lookup_options[module_name] = options 239 else 240 options = @lookup_options[module_name] 241 end 242 extract_lookup_options_for_key(key, options) 243 end
Retrieve the merge options that match the given `name`.
@param key [LookupKey] The key for which we want merge options @param lookup_invocation [Invocation] the lookup invocation @return [String,Hash,nil] The found merge options or nil
# File lib/puppet/pops/lookup/lookup_adapter.rb 221 def lookup_merge_options(key, lookup_invocation) 222 lookup_options = lookup_lookup_options(key, lookup_invocation) 223 lookup_options.nil? ? nil : lookup_options[MERGE] 224 end
@param path [String] the absolute path name of the global hiera.yaml file. @return [LookupAdapter] self
# File lib/puppet/pops/lookup/lookup_adapter.rb 274 def set_global_hiera_config_path(path) 275 @global_hiera_config_path = Pathname.new(path) 276 self 277 end
Instructs the lookup framework to only perform lookups in the global layer @return [LookupAdapter] self
# File lib/puppet/pops/lookup/lookup_adapter.rb 285 def set_global_only 286 @global_only = true 287 self 288 end
Private Instance Methods
# File lib/puppet/pops/lookup/lookup_adapter.rb 312 def compile_patterns(options) 313 return nil if options.nil? 314 key_options = {} 315 pattern_options = {} 316 options.each_pair do |key, value| 317 if key.start_with?(LOOKUP_OPTIONS_PATTERN_START) 318 pattern_options[Regexp.compile(key)] = value 319 else 320 key_options[key] = value 321 end 322 end 323 [key_options.empty? ? nil : key_options, pattern_options.empty? ? nil : pattern_options] 324 end
# File lib/puppet/pops/lookup/lookup_adapter.rb 326 def do_lookup(key, lookup_invocation, merge) 327 if lookup_invocation.global_only? 328 key.dig(lookup_invocation, lookup_global(key, lookup_invocation, merge)) 329 else 330 merge_strategy = Puppet::Pops::MergeStrategy.strategy(merge) 331 key.dig(lookup_invocation, 332 merge_strategy.lookup(PROVIDER_STACK, lookup_invocation) { |m| send(m, key, lookup_invocation, merge_strategy) }) 333 end 334 end
Retrieve and cache lookup options specific to the environment of the compiler that this adapter is attached to (i.e. a merge of global and environment lookup options).
# File lib/puppet/pops/lookup/lookup_adapter.rb 379 def env_lookup_options(lookup_invocation, merge_strategy) 380 if !instance_variable_defined?(:@env_lookup_options) 381 global_options = global_lookup_options(lookup_invocation, merge_strategy) 382 @env_only_lookup_options = nil 383 catch(:no_such_key) { @env_only_lookup_options = validate_lookup_options(lookup_in_environment(LookupKey::LOOKUP_OPTIONS, lookup_invocation, merge_strategy), nil) } 384 if global_options.nil? 385 @env_lookup_options = @env_only_lookup_options 386 elsif @env_only_lookup_options.nil? 387 @env_lookup_options = global_options 388 else 389 @env_lookup_options = merge_strategy.merge(global_options, @env_only_lookup_options) 390 end 391 end 392 @env_lookup_options 393 end
# File lib/puppet/pops/lookup/lookup_adapter.rb 400 def env_provider(lookup_invocation) 401 @env_provider = initialize_env_provider(lookup_invocation) unless instance_variable_defined?(:@env_provider) 402 @env_provider 403 end
@return [Puppet::Node::Environment] the environment of the compiler that this adapter is associated with
# File lib/puppet/pops/lookup/lookup_adapter.rb 517 def environment 518 @compiler.environment 519 end
Retrieve and cache the global lookup options
# File lib/puppet/pops/lookup/lookup_adapter.rb 369 def global_lookup_options(lookup_invocation, merge_strategy) 370 if !instance_variable_defined?(:@global_lookup_options) 371 @global_lookup_options = nil 372 catch(:no_such_key) { @global_lookup_options = validate_lookup_options(lookup_global(LookupKey::LOOKUP_OPTIONS, lookup_invocation, merge_strategy), nil) } 373 end 374 @global_lookup_options 375 end
# File lib/puppet/pops/lookup/lookup_adapter.rb 395 def global_provider(lookup_invocation) 396 @global_provider = GlobalDataProvider.new unless instance_variable_defined?(:@global_provider) 397 @global_provider 398 end
# File lib/puppet/pops/lookup/lookup_adapter.rb 461 def initialize_env_provider(lookup_invocation) 462 env_conf = environment.configuration 463 return nil if env_conf.nil? || env_conf.path_to_env.nil? 464 465 # Get the name of the data provider from the environment's configuration 466 provider_name = env_conf.environment_data_provider 467 env_path = Pathname(env_conf.path_to_env) 468 config_path = env_path + HieraConfig::CONFIG_FILE_NAME 469 470 ep = nil 471 if config_path.exist? 472 ep = EnvironmentDataProvider.new 473 # A version 5 hiera.yaml trumps any data provider setting in the environment.conf 474 ep_config = ep.config(lookup_invocation) 475 if ep_config.nil? 476 ep = nil 477 elsif ep_config.version >= 5 478 unless provider_name.nil? || Puppet[:strict] == :off 479 Puppet.warn_once('deprecations', 'environment.conf#data_provider', 480 _("Defining environment_data_provider='%{provider_name}' in environment.conf is deprecated") % { provider_name: provider_name }, env_path + 'environment.conf') 481 482 unless provider_name == 'hiera' 483 Puppet.warn_once('deprecations', 'environment.conf#data_provider_overridden', 484 _("The environment_data_provider='%{provider_name}' setting is ignored since '%{config_path}' version >= 5") % { provider_name: provider_name, config_path: config_path }, env_path + 'environment.conf') 485 end 486 end 487 provider_name = nil 488 end 489 end 490 491 if provider_name.nil? 492 ep 493 else 494 unless Puppet[:strict] == :off 495 msg = _("Defining environment_data_provider='%{provider_name}' in environment.conf is deprecated.") % { provider_name: provider_name } 496 msg += " " + _("A '%{hiera_config}' file should be used instead") % { hiera_config: HieraConfig::CONFIG_FILE_NAME } if ep.nil? 497 Puppet.warn_once('deprecations', 'environment.conf#data_provider', msg, env_path + 'environment.conf') 498 end 499 500 case provider_name 501 when 'none' 502 nil 503 when 'hiera' 504 # Use hiera.yaml or default settings if it is missing 505 ep || EnvironmentDataProvider.new 506 when 'function' 507 ep = EnvironmentDataProvider.new 508 ep.config = HieraConfigV5.v4_function_config(env_path, 'environment::data', ep) 509 ep 510 else 511 raise Puppet::Error.new(_("Environment '%{env}', cannot find environment_data_provider '%{provider}'") % { env: environment.name, provider: provider_name }) 512 end 513 end 514 end
# File lib/puppet/pops/lookup/lookup_adapter.rb 414 def initialize_module_provider(lookup_invocation, module_name) 415 mod = environment.module(module_name) 416 return nil if mod.nil? 417 418 metadata = mod.metadata 419 provider_name = metadata.nil? ? nil : metadata['data_provider'] 420 421 mp = nil 422 if mod.has_hiera_conf? 423 mp = ModuleDataProvider.new(module_name) 424 # A version 5 hiera.yaml trumps a data provider setting in the module 425 mp_config = mp.config(lookup_invocation) 426 if mp_config.nil? 427 mp = nil 428 elsif mp_config.version >= 5 429 unless provider_name.nil? || Puppet[:strict] == :off 430 Puppet.warn_once('deprecations', "metadata.json#data_provider-#{module_name}", 431 _("Defining \"data_provider\": \"%{name}\" in metadata.json is deprecated. It is ignored since a '%{config}' with version >= 5 is present") % { name: provider_name, config: HieraConfig::CONFIG_FILE_NAME }, mod.metadata_file) 432 end 433 provider_name = nil 434 end 435 end 436 437 if provider_name.nil? 438 mp 439 else 440 unless Puppet[:strict] == :off 441 msg = _("Defining \"data_provider\": \"%{name}\" in metadata.json is deprecated.") % { name: provider_name } 442 msg += " " + _("A '%{hiera_config}' file should be used instead") % { hiera_config: HieraConfig::CONFIG_FILE_NAME } if mp.nil? 443 Puppet.warn_once('deprecations', "metadata.json#data_provider-#{module_name}", msg, mod.metadata_file) 444 end 445 446 case provider_name 447 when 'none' 448 nil 449 when 'hiera' 450 mp || ModuleDataProvider.new(module_name) 451 when 'function' 452 mp = ModuleDataProvider.new(module_name) 453 mp.config = HieraConfig.v4_function_config(Pathname(mod.path), "#{module_name}::data", mp) 454 mp 455 else 456 raise Puppet::Error.new(_("Environment '%{env}', cannot find module_data_provider '%{provider}'")) % { env: environment.name, provider: provider_name } 457 end 458 end 459 end
# File lib/puppet/pops/lookup/lookup_adapter.rb 405 def module_provider(lookup_invocation, module_name) 406 # Test if the key is present for the given module_name. It might be there even if the 407 # value is nil (which indicates that no module provider is configured for the given name) 408 unless self.include?(module_name) 409 self[module_name] = initialize_module_provider(lookup_invocation, module_name) 410 end 411 self[module_name] 412 end
Retrieve lookup options that applies when using a specific module (i.e. a merge of the pre-cached `env_lookup_options` and the module specific data)
# File lib/puppet/pops/lookup/lookup_adapter.rb 340 def retrieve_lookup_options(module_name, lookup_invocation, merge_strategy) 341 meta_invocation = Invocation.new(lookup_invocation.scope) 342 meta_invocation.lookup(LookupKey::LOOKUP_OPTIONS, lookup_invocation.module_name) do 343 meta_invocation.with(:meta, LOOKUP_OPTIONS) do 344 if meta_invocation.global_only? 345 compile_patterns(global_lookup_options(meta_invocation, merge_strategy)) 346 else 347 opts = env_lookup_options(meta_invocation, merge_strategy) 348 unless module_name.nil? 349 # Store environment options at key nil. This removes the need for an additional lookup for keys that are not prefixed. 350 @lookup_options[nil] = compile_patterns(opts) unless @lookup_options.include?(nil) 351 catch(:no_such_key) do 352 module_opts = validate_lookup_options(lookup_in_module(LookupKey::LOOKUP_OPTIONS, meta_invocation, merge_strategy), module_name) 353 opts = if opts.nil? 354 module_opts 355 else 356 merge_strategy.lookup([GLOBAL_ENV_MERGE, "Module #{lookup_invocation.module_name}"], meta_invocation) do |n| 357 meta_invocation.with(:scope, n) { meta_invocation.report_found(LOOKUP_OPTIONS, n == GLOBAL_ENV_MERGE ? opts : module_opts) } 358 end 359 end 360 end 361 end 362 compile_patterns(opts) 363 end 364 end 365 end 366 end
# File lib/puppet/pops/lookup/lookup_adapter.rb 294 def validate_lookup_options(options, module_name) 295 raise Puppet::DataBinding::LookupError.new(_("value of %{opts} must be a hash") % { opts: LOOKUP_OPTIONS }) unless options.is_a?(Hash) unless options.nil? 296 return options if module_name.nil? 297 298 pfx = "#{module_name}::" 299 options.each_pair do |key, value| 300 if key.start_with?(LOOKUP_OPTIONS_PATTERN_START) 301 unless key[1..pfx.length] == pfx 302 raise Puppet::DataBinding::LookupError.new(_("all %{opts} patterns must match a key starting with module name '%{module_name}'") % { opts: LOOKUP_OPTIONS, module_name: module_name }) 303 end 304 else 305 unless key.start_with?(pfx) 306 raise Puppet::DataBinding::LookupError.new(_("all %{opts} keys must start with module name '%{module_name}'") % { opts: LOOKUP_OPTIONS, module_name: module_name }) 307 end 308 end 309 end 310 end