Merge pull request #3539 from sebastian-miclea/PDB-5135

(PDB-5135) Notify when resource_event size limit is reached

Author: Filipovici-Andrei

puppet/lib/puppet/reports/puppetdb.rb

@@ -190,6 +190,19 @@ def build_events_list(events)
   # @api private
   def resource_status_to_hash(resource_status)
     defaulted_corrective_change = defined?(resource_status.corrective_change) ? resource_status.corrective_change : nil
+    index_size = resource_event_index_size(resource_status)
+
+    if  index_size > 2500 && index_size <= 2704
+      Puppet.warning("resource_event on #{self.host} is very large. Values (resource timestamp + title + type) \
+are #{index_size} bytes and the hard limit is 2704. Caused by Resource type: \"#{resource_status.resource_type}\" \
+with title: \"#{resource_status.title.to_s}\" in #{resource_status.file}:#{resource_status.line}")
+    end
+
+    if index_size > 2704
+      Puppet.warning("resource_event on #{self.host} could not be saved. Values (resource timestamp + title + type) are larger \
+than index total size of 2704. Caused by Resource type: \"#{resource_status.resource_type}\" with title: \
+\"#{resource_status.title.to_s}\" in #{resource_status.file}:#{resource_status.line}")
+    end
     {
       "skipped"           => resource_status.skipped,
       "timestamp"         => Puppet::Util::Puppetdb.to_wire_time(resource_status.time),
@@ -203,6 +216,17 @@ def resource_status_to_hash(resource_status)
     }
   end
 
+  # Get the size of the columns that will populate the resource_events_resource_timestamp_xxxxxz index.
+  # resource_events_resource_timestamp_xxxxxz index is comprised of 3 columns (resource timestamp, title and type)
+  # and it's maximum size is 2704 bytes. So the size of the 3 resource attributes combined, must not exceed 2704 bytes.
+  #
+  # @return Boolean
+  # @api private
+  def resource_event_index_size(resource_status)
+    # 8 - represents the timestamp which has the same size all the time
+    8 + resource_status.resource_type.bytesize + resource_status.title.to_s.bytesize
+  end
+
   # Helper method for accessing the puppetdb configuration
   #
   # @api private

puppet/spec/unit/reports/puppetdb_spec.rb

@@ -20,7 +20,6 @@
   }
 
   context "#process" do
-
     let(:http) { mock "http" }
     let(:httpok) { Net::HTTPOK.new('1.1', 200, '') }
 
@@ -367,7 +366,57 @@ def without_producer_timestamp(json_body)
         end
 
       end
+    end
+  end
+
+  context '#resource_status_to_hash' do
+    before do
+      subject.host = 'localhost'
+    end
+    let(:resource) do
+      stub('resource',
+        { pathbuilder: ['foo', 'bar', 'baz'],
+          path: 'foo',
+          file: 'foo',
+          line: 1,
+          tags: [],
+          provider: 'foo',
+          skipped: false,
+          containment_path: '\\',
+          events: [],
+          time: DateTime.now,
+          type: 'File',
+          resource_type: 'File',
+          title: huge_title,
+          merge_into: nil })
+    end
+
+    context 'large resources' do
+      let(:huge_title) { (0...2501).map { ('a'..'z').to_a[rand(26)] }.join }
+      let (:status) do
+        Puppet::Resource::Status.new(resource)
+      end
+
+      it 'large resources will warn about the index size' do
+        Puppet.expects(:warning).with("resource_event on localhost is very large. Values (resource timestamp + title + type) \
+are 2513 bytes and the hard limit is 2704. Caused by Resource type: \"File\" with title: \"#{huge_title}\" in foo:1")
+        subject.add_resource_status(status)
+        subject.resource_status_to_hash(resource)
+      end
+    end
+
+    context "oversized resource" do
+      let(:huge_title) { (0...2704).map { ('a'..'z').to_a[rand(26)] }.join }
+      let (:status) do
+        Puppet::Resource::Status.new(resource)
+      end
 
+      it 'large resources will warn about the index size' do
+        Puppet.expects(:warning).with("resource_event on localhost could not be saved. Values (resource timestamp + title + type) \
+are larger than index total size of 2704. Caused by Resource type: \"File\" with title: \"#{huge_title}\" in foo:1")
+        subject.add_resource_status(status)
+        subject.resource_status_to_hash(resource)
+      end
     end
   end
 end