(PUP-9323) Resolve deferred values at resource evaluation time

Allow deferred values to be resolved lazily as each resource is evaluated.
This also means validation, munging and unmunging are delayed until the
resolved value is set back on the parameter or property.

The deferred evaluation is similar to how Puppet::Type#eval_generate works, but
since we're not modifying the graph, we don't need to worry about containment
edges, tags, noop, etc.

Author: joshcooper

lib/puppet/parameter.rb

@@ -415,6 +415,8 @@ def unsafe_munge(value)
   # @return [Object] the unmunged value
   #
   def unmunge(value)
+    return value if value.is_a?(Puppet::Pops::Evaluator::DeferredValue)
+
     unsafe_unmunge(value)
   end
 
@@ -435,6 +437,8 @@ def unsafe_unmunge(value)
   # @return [Object] the munged (internal) value
   #
   def munge(value)
+    return value if value.is_a?(Puppet::Pops::Evaluator::DeferredValue)
+
     begin
       ret = unsafe_munge(value)
     rescue Puppet::Error => detail
@@ -468,6 +472,8 @@ def unsafe_validate(value)
   # @api public
   #
   def validate(value)
+    return if value.is_a?(Puppet::Pops::Evaluator::DeferredValue)
+
     begin
       unsafe_validate(value)
     rescue ArgumentError => detail

lib/puppet/pops/evaluator/deferred_resolver.rb

@@ -3,6 +3,16 @@
 module Puppet::Pops
 module Evaluator
 
+class DeferredValue
+  def initialize(proc)
+    @proc = proc
+  end
+
+  def resolve
+    @proc.call
+  end
+end
+
 # Utility class to help resolve instances of Puppet::Pops::Types::PDeferredType::Deferred
 #
 class DeferredResolver
@@ -20,9 +30,9 @@ class DeferredResolver
   #  are to be mixed into the scope
   # @return [nil] does not return anything - the catalog is modified as a side effect
   #
-  def self.resolve_and_replace(facts, catalog, environment = catalog.environment_instance)
-    compiler = Puppet::Parser::ScriptCompiler.new(environment, catalog.name, true)
-    resolver = new(compiler)
+  def self.resolve_and_replace(facts, catalog, environment = catalog.environment_instance, preprocess_deferred = true)
+    compiler = Puppet::Parser::ScriptCompiler.new(environment, catalog.name, preprocess_deferred)
+    resolver = new(compiler, preprocess_deferred)
     resolver.set_facts_variable(facts)
     # TODO:
     #    # When scripting the trusted data are always local, but set them anyway
@@ -53,11 +63,12 @@ def self.resolve(value, compiler)
     resolver.resolve(value)
   end
 
-  def initialize(compiler)
+  def initialize(compiler, preprocess_deferred = true)
     @compiler = compiler
     # Always resolve in top scope
     @scope = @compiler.topscope
     @deferred_class = Puppet::Pops::Types::TypeFactory.deferred.implementation_class
+    @preprocess_deferred = preprocess_deferred
   end
 
   # @param facts [Puppet::Node::Facts] the facts to set in $facts in the compiler's topscope
@@ -106,6 +117,24 @@ def resolve(x)
     end
   end
 
+  def resolve_lazy_args(x)
+    if x.is_a?(DeferredValue)
+      x.resolve
+    elsif x.is_a?(Array)
+      x.map {|v| resolve_lazy_args(v) }
+    elsif x.is_a?(Hash)
+      result = {}
+      x.each_pair {|k,v| result[k] = resolve_lazy_args(v) }
+      result
+    elsif x.is_a?(Puppet::Pops::Types::PSensitiveType::Sensitive)
+      # rewrap in a new Sensitive after resolving any nested deferred values
+      Puppet::Pops::Types::PSensitiveType::Sensitive.new(resolve_lazy_args(x.unwrap))
+    else
+      x
+    end
+  end
+  private :resolve_lazy_args
+
   def resolve_future(f)
     # If any of the arguments to a future is a future it needs to be resolved first
     func_name = f.name
@@ -117,8 +146,19 @@ def resolve_future(f)
       mapped_arguments.insert(0, @scope[var_name])
     end
 
-    # call the function (name in deferred, or 'dig' for a variable)
-    @scope.call_function(func_name, mapped_arguments)
+    if @preprocess_deferred
+      # call the function (name in deferred, or 'dig' for a variable)
+      @scope.call_function(func_name, mapped_arguments)
+    else
+      # call the function later
+      DeferredValue.new(
+        Proc.new {
+          # deferred functions can have nested deferred arguments
+          resolved_arguments = mapped_arguments.map { |arg| resolve_lazy_args(arg) }
+          @scope.call_function(func_name, resolved_arguments)
+        }
+      )
+    end
   end
 
   def map_arguments(args)

lib/puppet/transaction.rb

@@ -276,6 +276,7 @@ def apply(resource, ancestor = nil)
 
   # Evaluate a single resource.
   def eval_resource(resource, ancestor = nil)
+    resolve_resource(resource)
     propagate_failure(resource)
     if skip?(resource)
       resource_status(resource).skipped = true
@@ -464,6 +465,20 @@ def split_qualified_tags?
   public :skip?
   public :missing_tags?
 
+  def resolve_resource(resource)
+    return unless catalog.host_config?
+
+    resource.eachparameter do |param|
+      if param.value.instance_of?(Puppet::Pops::Evaluator::DeferredValue)
+        # Puppet::Parameter#value= triggers validation and munging. Puppet::Property#value=
+        # overrides the method, but also triggers validation and munging, since we're
+        # setting the desired/should value.
+        resolved = param.value.resolve
+        # resource.notice("Resolved deferred value to #{resolved}")
+        param.value = resolved
+      end
+    end
+  end
 end
 
 require_relative 'transaction/report'

spec/unit/pops/evaluator/deferred_resolver_spec.rb

@@ -17,4 +17,30 @@
 
     expect(catalog.resource(:notify, 'deferred')[:message]).to eq('1:2:3')
   end
+
+  it 'lazily resolves deferred values in a catalog' do
+    catalog = compile_to_catalog(<<~END)
+      notify { "deferred":
+        message => Deferred("join", [[1,2,3], ":"])
+      }
+    END
+    described_class.resolve_and_replace(facts, catalog, environment, false)
+
+    deferred = catalog.resource(:notify, 'deferred')[:message]
+    expect(deferred.resolve).to eq('1:2:3')
+  end
+
+  it 'lazily resolves nested deferred values in a catalog' do
+    catalog = compile_to_catalog(<<~END)
+      $args = Deferred("inline_epp", ["<%= 'a,b,c' %>"])
+      notify { "deferred":
+        message => Deferred("split", [$args, ","])
+      }
+    END
+    described_class.resolve_and_replace(facts, catalog, environment, false)
+
+    deferred = catalog.resource(:notify, 'deferred')[:message]
+    expect(deferred.resolve).to eq(["a", "b", "c"])
+  end
+
 end