From 33029bbe189608177bcdca02a50d3b438e90ec0a Mon Sep 17 00:00:00 2001 From: Adrian Parreiras Horta Date: Fri, 17 Oct 2025 10:36:11 -0700 Subject: [PATCH 1/9] Add new management params These params control managing the Telegraf class, service, and outputs. It also adds one to manage token authentication. --- data/common.yaml | 2 + hiera-rspec.yaml | 11 +++ manifests/telegraf/agent.pp | 185 +++++++++++++++++++++--------------- spec/classes/agent_spec.rb | 86 +++++++++++++++++ spec/data/rspec.yaml | 2 + spec/spec_helper.rb | 6 +- 6 files changed, 213 insertions(+), 79 deletions(-) create mode 100644 hiera-rspec.yaml create mode 100644 spec/data/rspec.yaml diff --git a/data/common.yaml b/data/common.yaml index 3092a64..813fd70 100644 --- a/data/common.yaml +++ b/data/common.yaml @@ -1,3 +1,5 @@ --- puppet_operational_dashboards::telegraf::agent::manage_repo: false +puppet_operational_dashboards::telegraf::agent::manage_class: true +puppet_operational_dashboards::telegraf::agent::use_token_auth: true puppet_operational_dashboards::telegraf::agent::version: '1.29.4-1' diff --git a/hiera-rspec.yaml b/hiera-rspec.yaml new file mode 100644 index 0000000..4d6745a --- /dev/null +++ b/hiera-rspec.yaml @@ -0,0 +1,11 @@ +--- +version: 5 + +defaults: # Used for any hierarchy level that omits these keys. + datadir: spec/data # This path is relative to hiera.yaml's directory. + data_hash: yaml_data # Use the built-in YAML backend. + +hierarchy: + - name: 'rspec' + path: 'rspec.yaml' + diff --git a/manifests/telegraf/agent.pp b/manifests/telegraf/agent.pp index 9f441d3..6b85ff2 100644 --- a/manifests/telegraf/agent.pp +++ b/manifests/telegraf/agent.pp @@ -23,10 +23,16 @@ # Whether to manage Telegraf ssl configuration. # @param manage_repo # Whether to install Telegraf from a repository. +# @param manage_class +# Whether to manage the Telegraf class and related resources +# @param use_token_auth +# Whether to set up and use token based auth to InfluxDB # @param manage_archive # Whether to install Telegraf from an archive source. # @param manage_user # Whether to manage the telegraf user when installing from archive. +# @param manage_outputs +# Whether to manage the telegraf outputs # @param ssl_cert_file # SSL certificate to be used by the telegraf service. # @param ssl_key_file @@ -43,6 +49,8 @@ # Skip verification of SSL certificate. # @param version # Version of the Telegraf package to install. +# @param influxdb_version +# Which version of InfluxDB to use in the inputs. Currently supports versions 1 and 2 # @param archive_location # URL containing an archive source for the telegraf package. Defaults to downloading $version from dl.influxdata.com # @param archive_install_dir @@ -88,7 +96,10 @@ # Template format to use for puppet template toml or yaml config class puppet_operational_dashboards::telegraf::agent ( String $version, + Enum['v1', 'v2'] $influxdb_version = 'v2', Boolean $manage_repo, + Boolean $manage_class, + Boolean $use_token_auth, Optional[Sensitive[String]] $token = $puppet_operational_dashboards::telegraf_token, String $token_name = $puppet_operational_dashboards::telegraf_token_name, String $influxdb_token_file = $puppet_operational_dashboards::influxdb_token_file, @@ -102,6 +113,7 @@ Boolean $insecure_skip_verify = true, Boolean $manage_archive = !$manage_repo, Boolean $manage_user = true, + Boolean $manage_outputs = true, Stdlib::Absolutepath $ssl_cert_file = "/etc/puppetlabs/puppet/ssl/certs/${trusted['certname']}.pem", Stdlib::Absolutepath $ssl_key_file ="/etc/puppetlabs/puppet/ssl/private_keys/${trusted['certname']}.pem", Stdlib::Absolutepath $ssl_ca_file ='/etc/puppetlabs/puppet/ssl/certs/ca.pem', @@ -152,31 +164,6 @@ } $influxdb_uri = "${protocol}://${influxdb_host}:${influxdb_port}" - $influxdb_v2 = $use_ssl ? { - true => { - 'influxdb_v2' => [ - { - 'tls_ca' => '/etc/telegraf/ca.pem', - 'tls_cert' => '/etc/telegraf/cert.pem', - 'insecure_skip_verify' => $insecure_skip_verify, - 'bucket' => $influxdb_bucket, - 'organization' => $influxdb_org, - 'token' => '$INFLUX_TOKEN', - 'urls' => [$influxdb_uri], - } - ], - }, - false => { - 'influxdb_v2' => [ - { - 'bucket' => $influxdb_bucket, - 'organization' => $influxdb_org, - 'token' => '$INFLUX_TOKEN', - 'urls' => [$influxdb_uri], - } - ], - }, - } $database = if $include_pe_metrics { 'pe-puppetdb' @@ -192,18 +179,61 @@ notify => Service['telegraf'], } - class { 'telegraf': - ensure => $version, - manage_repo => $manage_repo, - manage_archive => $manage_archive, - manage_user => $manage_user, - archive_location => $archive_location, - archive_install_dir => $archive_install_dir, - interval => $collection_interval, - hostname => '', - manage_service => false, - outputs => $influxdb_v2, - notify => Service['telegraf'], + if $manage_class { + if $manage_outputs { + $influxdb_v2 = $use_ssl ? { + true => { + 'influxdb_v2' => [ + { + 'tls_ca' => '/etc/telegraf/ca.pem', + 'tls_cert' => '/etc/telegraf/cert.pem', + 'insecure_skip_verify' => $insecure_skip_verify, + 'bucket' => $influxdb_bucket, + 'organization' => $influxdb_org, + 'token' => '$INFLUX_TOKEN', + 'urls' => [$influxdb_uri], + } + ], + }, + false => { + 'influxdb_v2' => [ + { + 'bucket' => $influxdb_bucket, + 'organization' => $influxdb_org, + 'token' => '$INFLUX_TOKEN', + 'urls' => [$influxdb_uri], + } + ], + }, + } + class { 'telegraf': + ensure => $version, + manage_repo => $manage_repo, + manage_archive => $manage_archive, + manage_user => $manage_user, + archive_location => $archive_location, + archive_install_dir => $archive_install_dir, + interval => $collection_interval, + hostname => '', + manage_service => false, + outputs => $influxdb_v2, + notify => Service['telegraf'], + } + } + else { + class { 'telegraf': + ensure => $version, + manage_repo => $manage_repo, + manage_archive => $manage_archive, + manage_user => $manage_user, + archive_location => $archive_location, + archive_install_dir => $archive_install_dir, + interval => $collection_interval, + hostname => '', + manage_service => false, + notify => Service['telegraf'], + } + } } if $use_ssl and $manage_ssl { @@ -257,44 +287,53 @@ } } - file { '/etc/systemd/system/telegraf.service.d': - ensure => directory, - owner => 'telegraf', - group => 'telegraf', - mode => '0700', - require => Class['telegraf::install'], - } + if $use_token_auth { + # Only require the override file containing the token if using token auth + Puppet_operational_dashboards::Telegraf::Config { + require => File['/etc/systemd/system/telegraf.service.d/override.conf'], + } - if $token { - file { '/etc/systemd/system/telegraf.service.d/override.conf': - ensure => file, - content => inline_epp(file('influxdb/telegraf_environment_file.epp'), { token => $token }), - notify => [ - Exec['puppet_telegraf_daemon_reload'], - Service['telegraf'] - ], + file { '/etc/systemd/system/telegraf.service.d': + ensure => directory, + owner => 'telegraf', + group => 'telegraf', + mode => '0700', + require => Class['telegraf::install'], } - } - else { - $token_vars = { - token => Sensitive(Deferred('influxdb::retrieve_token', [$influxdb_uri, $token_name, $influxdb_token_file, $use_system_store])), + + if $token { + file { '/etc/systemd/system/telegraf.service.d/override.conf': + ensure => file, + content => inline_epp(file('influxdb/telegraf_environment_file.epp'), { token => $token }), + notify => [ + Exec['puppet_telegraf_daemon_reload'], + Service['telegraf'] + ], + } } - file { '/etc/systemd/system/telegraf.service.d/override.conf': - ensure => file, - content => Deferred('inline_epp', [file('influxdb/telegraf_environment_file.epp'), $token_vars]), - notify => [ - Exec['puppet_telegraf_daemon_reload'], - Service['telegraf'], - ], + else { + $token_vars = { + token => Sensitive(Deferred('influxdb::retrieve_token', [$influxdb_uri, $token_name, $influxdb_token_file, $use_system_store])), + } + file { '/etc/systemd/system/telegraf.service.d/override.conf': + ensure => file, + content => Deferred('inline_epp', [file('influxdb/telegraf_environment_file.epp'), $token_vars]), + notify => [ + Exec['puppet_telegraf_daemon_reload'], + Service['telegraf'], + ], + } } } - service { 'telegraf': - ensure => running, - require => [ - Class['telegraf::install'], - Exec['puppet_telegraf_daemon_reload'], - ], + if $manage_class { + service { 'telegraf': + ensure => running, + require => [ + Class['telegraf::install'], + Exec['puppet_telegraf_daemon_reload'], + ], + } } if $collection_method == 'all' { @@ -304,7 +343,6 @@ protocol => $protocol, http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, - require => File['/etc/systemd/system/telegraf.service.d/override.conf'], } } @@ -314,7 +352,6 @@ protocol => $protocol, http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, - require => File['/etc/systemd/system/telegraf.service.d/override.conf'], } } @@ -324,7 +361,6 @@ protocol => $protocol, http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, - require => File['/etc/systemd/system/telegraf.service.d/override.conf'], } } @@ -371,7 +407,6 @@ protocol => $protocol, http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, - require => File['/etc/systemd/system/telegraf.service.d/override.conf'], } } } @@ -388,7 +423,6 @@ protocol => $protocol, http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, - require => File['/etc/systemd/system/telegraf.service.d/override.conf'], } } @@ -398,7 +432,6 @@ protocol => $protocol, http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, - require => File['/etc/systemd/system/telegraf.service.d/override.conf'], } if $include_pe_metrics { puppet_operational_dashboards::telegraf::config { 'pcp': @@ -406,7 +439,6 @@ protocol => $protocol, http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, - require => File['/etc/systemd/system/telegraf.service.d/override.conf'], } } } @@ -417,7 +449,6 @@ protocol => $protocol, http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, - require => File['/etc/systemd/system/telegraf.service.d/override.conf'], } } diff --git a/spec/classes/agent_spec.rb b/spec/classes/agent_spec.rb index 3d35aa8..b282c57 100644 --- a/spec/classes/agent_spec.rb +++ b/spec/classes/agent_spec.rb @@ -439,6 +439,92 @@ it { is_expected.to contain_telegraf__input('pcp_metrics') } it { is_expected.to contain_telegraf__input('orchestrator_metrics') } end + + context 'when not managing telegraf class' do + let(:params) do + { + token: RSpec::Puppet::Sensitive.new(nil), + token_name: 'puppet telegraf token', + influxdb_token_file: '/root/.influxdb_token', + influxdb_host: 'localhost.foo.com', + influxdb_port: 8086, + influxdb_bucket: 'puppet_data', + influxdb_org: 'puppetlabs', + use_ssl: true, + use_system_store: false, + collection_method: 'local', + local_services: ['puppetserver'], + include_pe_metrics: false, + template_format: 'toml', + manage_class: true, + } + end + + it { + is_expected.to compile.with_all_deps + } + end + + context 'when not managing outputs' do + let(:params) do + { + token: RSpec::Puppet::Sensitive.new(nil), + token_name: 'puppet telegraf token', + influxdb_token_file: '/root/.influxdb_token', + influxdb_host: 'localhost.foo.com', + influxdb_port: 8086, + influxdb_bucket: 'puppet_data', + influxdb_org: 'puppetlabs', + use_ssl: true, + use_system_store: false, + collection_method: 'local', + local_services: ['puppetserver'], + include_pe_metrics: false, + template_format: 'toml', + manage_class: true, + manage_outputs: false, + } + end + + it { + is_expected.to compile.with_all_deps + + # Default for telegraf::outputs is set in spec/data/rspec.yaml + # If we are not managing it, it should use the default + is_expected.to contain_class('telegraf').with( + outputs: {}, + ) + } + end + + + context 'when not using token auth' do + let(:params) do + { + token: RSpec::Puppet::Sensitive.new(nil), + token_name: 'puppet telegraf token', + influxdb_token_file: '/root/.influxdb_token', + influxdb_host: 'localhost.foo.com', + influxdb_port: 8086, + influxdb_bucket: 'puppet_data', + influxdb_org: 'puppetlabs', + use_ssl: true, + use_system_store: false, + collection_method: 'local', + local_services: ['puppetserver'], + include_pe_metrics: false, + template_format: 'toml', + use_token_auth: false, + } + end + + it { + is_expected.to compile.with_all_deps + + is_expected.to_not contain_file('/etc/systemd/system/telegraf.service.d') + is_expected.to_not contain_file('/etc/systemd/system/telegraf.service.d/override.conf') + } + end end end end diff --git a/spec/data/rspec.yaml b/spec/data/rspec.yaml new file mode 100644 index 0000000..0d07b25 --- /dev/null +++ b/spec/data/rspec.yaml @@ -0,0 +1,2 @@ +--- +telegraf::outputs: {} diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 6820ceb..062eb00 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -25,7 +25,8 @@ next unless File.exist?(f) && File.readable?(f) && File.size?(f) begin - default_facts.merge!(YAML.safe_load(File.read(f), permitted_classes: [], permitted_symbols: [], aliases: true)) + require 'deep_merge' + default_facts.deep_merge!(YAML.safe_load_file(f, permitted_classes: [], permitted_symbols: [], aliases: true)) rescue StandardError => e RSpec.configuration.reporter.message "WARNING: Unable to load #{f}: #{e}" end @@ -33,11 +34,12 @@ # read default_facts and merge them over what is provided by facterdb default_facts.each do |fact, value| - add_custom_fact fact, value + add_custom_fact fact, value, merge_facts: true end RSpec.configure do |c| c.default_facts = default_facts + c.hiera_config = 'hiera-rspec.yaml' c.before :each do # set to strictest setting for testing # by default Puppet runs at warning level From 7991a2e2cb9d03a0e170d38d0b7a872c45edd975 Mon Sep 17 00:00:00 2001 From: Adrian Parreiras Horta Date: Fri, 17 Oct 2025 11:57:32 -0700 Subject: [PATCH 2/9] Add support for InfluxDB 1 This is still supported with enterprise InfluxDB --- manifests/telegraf/agent.pp | 16 ++++++++---- spec/classes/agent_spec.rb | 50 ++++++++++++++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/manifests/telegraf/agent.pp b/manifests/telegraf/agent.pp index 6b85ff2..5c3e99e 100644 --- a/manifests/telegraf/agent.pp +++ b/manifests/telegraf/agent.pp @@ -164,7 +164,6 @@ } $influxdb_uri = "${protocol}://${influxdb_host}:${influxdb_port}" - $database = if $include_pe_metrics { 'pe-puppetdb' } @@ -181,9 +180,16 @@ if $manage_class { if $manage_outputs { - $influxdb_v2 = $use_ssl ? { + $output_version = if $influxdb_version == 'v2' { + 'influxdb_v2' + } + else { + 'influxdb' + } + + $influxdb_outputs = $use_ssl ? { true => { - 'influxdb_v2' => [ + $output_version => [ { 'tls_ca' => '/etc/telegraf/ca.pem', 'tls_cert' => '/etc/telegraf/cert.pem', @@ -196,7 +202,7 @@ ], }, false => { - 'influxdb_v2' => [ + $output_version => [ { 'bucket' => $influxdb_bucket, 'organization' => $influxdb_org, @@ -216,7 +222,7 @@ interval => $collection_interval, hostname => '', manage_service => false, - outputs => $influxdb_v2, + outputs => $influxdb_outputs, notify => Service['telegraf'], } } diff --git a/spec/classes/agent_spec.rb b/spec/classes/agent_spec.rb index b282c57..f6a0ea3 100644 --- a/spec/classes/agent_spec.rb +++ b/spec/classes/agent_spec.rb @@ -78,6 +78,51 @@ } end + context 'when using influxdb v1' do + let(:params) do + { + token: RSpec::Puppet::Sensitive.new(nil), + token_name: 'puppet telegraf token', + influxdb_token_file: '/root/.influxdb_token', + influxdb_host: 'localhost.foo.com', + influxdb_port: 8086, + influxdb_bucket: 'puppet_data', + influxdb_org: 'puppetlabs', + use_ssl: true, + use_system_store: false, + manage_ssl: true, + include_pe_metrics: true, + template_format: 'toml', + influxdb_version: 'v1' + } + end + let(:influxdb_v1) do + { + 'influxdb' => [ + { + 'tls_ca' => '/etc/telegraf/ca.pem', + 'tls_cert' => '/etc/telegraf/cert.pem', + 'insecure_skip_verify' => true, + 'bucket' => 'puppet_data', + 'organization' => 'puppetlabs', + 'token' => '$INFLUX_TOKEN', + 'urls' => ['https://localhost.foo.com:8086'] + }, + ], + } + end + + it { + is_expected.to contain_class('telegraf').with( + ensure: '1.29.4-1', + archive_location: 'https://dl.influxdata.com/telegraf/releases/telegraf-1.29.4_linux_amd64.tar.gz', + interval: '10m', + manage_service: false, + outputs: influxdb_v1, + ) + } + end + context 'when using postgres password auth' do let(:params) do { @@ -497,7 +542,6 @@ } end - context 'when not using token auth' do let(:params) do { @@ -521,8 +565,8 @@ it { is_expected.to compile.with_all_deps - is_expected.to_not contain_file('/etc/systemd/system/telegraf.service.d') - is_expected.to_not contain_file('/etc/systemd/system/telegraf.service.d/override.conf') + is_expected.not_to contain_file('/etc/systemd/system/telegraf.service.d') + is_expected.not_to contain_file('/etc/systemd/system/telegraf.service.d/override.conf') } end end From b95cd9f76a508c0bf8818f2e4a6ef8691c393a14 Mon Sep 17 00:00:00 2001 From: Adrian Parreiras Horta Date: Wed, 22 Oct 2025 14:37:23 -0700 Subject: [PATCH 3/9] Make file-sync metrics configurable This commit adds $include_pe_metrics to the defined types and templates to allow for configuring whether we want to include file sync metrics. --- manifests/telegraf/agent.pp | 8 ++++++++ manifests/telegraf/config.pp | 5 ++++- ...puppet_operational_dashboards_telegraf_config_spec.rb | 2 ++ templates/orchestrator_metrics.toml.epp | 7 ++++++- templates/orchestrator_metrics.yaml.epp | 7 ++++++- templates/pcp_metrics.toml.epp | 7 ++++++- templates/pcp_metrics.yaml.epp | 7 ++++++- templates/puppetdb_jvm_metrics.toml.epp | 7 ++++++- templates/puppetdb_jvm_metrics.yaml.epp | 7 ++++++- templates/puppetdb_metrics.toml.epp | 7 ++++++- templates/puppetdb_metrics.yaml.epp | 7 ++++++- templates/puppetserver_metrics.toml.epp | 9 ++++++++- templates/puppetserver_metrics.yaml.epp | 9 ++++++++- 13 files changed, 78 insertions(+), 11 deletions(-) diff --git a/manifests/telegraf/agent.pp b/manifests/telegraf/agent.pp index 5c3e99e..8cb6339 100644 --- a/manifests/telegraf/agent.pp +++ b/manifests/telegraf/agent.pp @@ -349,6 +349,7 @@ protocol => $protocol, http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, + include_pe_metrics => $include_pe_metrics, } } @@ -358,6 +359,7 @@ protocol => $protocol, http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, + include_pe_metrics => $include_pe_metrics, } } @@ -367,6 +369,7 @@ protocol => $protocol, http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, + include_pe_metrics => $include_pe_metrics, } } @@ -413,6 +416,7 @@ protocol => $protocol, http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, + include_pe_metrics => $include_pe_metrics, } } } @@ -429,6 +433,7 @@ protocol => $protocol, http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, + include_pe_metrics => $include_pe_metrics, } } @@ -438,6 +443,7 @@ protocol => $protocol, http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, + include_pe_metrics => $include_pe_metrics, } if $include_pe_metrics { puppet_operational_dashboards::telegraf::config { 'pcp': @@ -445,6 +451,7 @@ protocol => $protocol, http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, + include_pe_metrics => $include_pe_metrics, } } } @@ -455,6 +462,7 @@ protocol => $protocol, http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, + include_pe_metrics => $include_pe_metrics, } } diff --git a/manifests/telegraf/config.pp b/manifests/telegraf/config.pp index ba35fd5..f6026cc 100644 --- a/manifests/telegraf/config.pp +++ b/manifests/telegraf/config.pp @@ -11,6 +11,8 @@ # Timeout for HTTP Telegraf inputs. Might be usefull in huge environments with slower API responses # @param template_format # Template format to use for puppet template toml or yaml config +# @param include_pe_metrics +# Whether to include Filesync metrics in Puppetserver define puppet_operational_dashboards::telegraf::config ( Array[String[1]] $hosts, Enum['https', 'http'] $protocol, @@ -18,6 +20,7 @@ String $service = $title, Enum['present', 'absent'] $ensure = 'present', Enum['yaml','toml'] $template_format = 'toml', + Boolean $include_pe_metrics, ) { unless $service in ['puppetserver', 'puppetdb', 'puppetdb_jvm', 'orchestrator', 'pcp'] { fail("Unknown service type ${service}") @@ -43,7 +46,7 @@ $inputs = epp( "puppet_operational_dashboards/${service}_metrics.${template_format}.epp", - { urls => $urls, protocol => $protocol, http_timeout_seconds => $http_timeout_seconds } + { urls => $urls, protocol => $protocol, http_timeout_seconds => $http_timeout_seconds, include_pe_metrics => $include_pe_metrics } ) $_inputs = $template_format ? { diff --git a/spec/defines/puppet_operational_dashboards_telegraf_config_spec.rb b/spec/defines/puppet_operational_dashboards_telegraf_config_spec.rb index 2c33240..6ceb21e 100644 --- a/spec/defines/puppet_operational_dashboards_telegraf_config_spec.rb +++ b/spec/defines/puppet_operational_dashboards_telegraf_config_spec.rb @@ -9,6 +9,7 @@ protocol: 'https', http_timeout_seconds: 5, hosts: ['localhost.foo.com'], + include_pe_metrics: true, } end @@ -135,6 +136,7 @@ protocol: 'http', http_timeout_seconds: 5, hosts: ['localhost.foo.com'], + include_pe_metrics: true, } end diff --git a/templates/orchestrator_metrics.toml.epp b/templates/orchestrator_metrics.toml.epp index 8d4729a..83d29d7 100644 --- a/templates/orchestrator_metrics.toml.epp +++ b/templates/orchestrator_metrics.toml.epp @@ -1,4 +1,9 @@ -<%- | Array[String] $urls, Enum['https', 'http'] $protocol, Integer[1] $http_timeout_seconds | -%> +<%- | + Array[String] $urls, + Enum['https', 'http'] $protocol, + Integer[1] $http_timeout_seconds, + Boolean $include_pe_metrics, + | -%> <% if $protocol == 'https' { -%> tls_cert = "/etc/telegraf/puppet_cert.pem" tls_key = "/etc/telegraf/puppet_key.pem" diff --git a/templates/orchestrator_metrics.yaml.epp b/templates/orchestrator_metrics.yaml.epp index d4600a8..fec5a37 100644 --- a/templates/orchestrator_metrics.yaml.epp +++ b/templates/orchestrator_metrics.yaml.epp @@ -1,4 +1,9 @@ -<%- | Array[String] $urls, Enum['https', 'http'] $protocol, Integer[1] $http_timeout_seconds | -%> +<%- | + Array[String] $urls, + Enum['https', 'http'] $protocol, + Integer[1] $http_timeout_seconds, + Boolean $include_pe_metrics, + | -%> <% if $protocol == 'https' { -%> tls_cert: "/etc/telegraf/puppet_cert.pem" tls_key: "/etc/telegraf/puppet_key.pem" diff --git a/templates/pcp_metrics.toml.epp b/templates/pcp_metrics.toml.epp index b11fa73..2064f8b 100644 --- a/templates/pcp_metrics.toml.epp +++ b/templates/pcp_metrics.toml.epp @@ -1,4 +1,9 @@ -<%- | Array[String] $urls, Enum['https', 'http'] $protocol, Integer[1] $http_timeout_seconds | -%> +<%- | + Array[String] $urls, + Enum['https', 'http'] $protocol, + Integer[1] $http_timeout_seconds, + Boolean $include_pe_metrics, + | -%> <% if $protocol == 'https' { -%> tls_cert = "/etc/telegraf/puppet_cert.pem" tls_key = "/etc/telegraf/puppet_key.pem" diff --git a/templates/pcp_metrics.yaml.epp b/templates/pcp_metrics.yaml.epp index a76110e..274968d 100644 --- a/templates/pcp_metrics.yaml.epp +++ b/templates/pcp_metrics.yaml.epp @@ -1,4 +1,9 @@ -<%- | Array[String] $urls, Enum['https', 'http'] $protocol, Integer[1] $http_timeout_seconds | -%> +<%- | + Array[String] $urls, + Enum['https', 'http'] $protocol, + Integer[1] $http_timeout_seconds, + Boolean $include_pe_metrics, + | -%> <% if $protocol == 'https' { -%> tls_cert: "/etc/telegraf/puppet_cert.pem" tls_key: "/etc/telegraf/puppet_key.pem" diff --git a/templates/puppetdb_jvm_metrics.toml.epp b/templates/puppetdb_jvm_metrics.toml.epp index 3a70a95..e920122 100644 --- a/templates/puppetdb_jvm_metrics.toml.epp +++ b/templates/puppetdb_jvm_metrics.toml.epp @@ -1,4 +1,9 @@ -<%- | Array[String] $urls, Enum['https', 'http'] $protocol, Integer[1] $http_timeout_seconds | -%> +<%- | + Array[String] $urls, + Enum['https', 'http'] $protocol, + Integer[1] $http_timeout_seconds, + Boolean $include_pe_metrics, + | -%> <% if $protocol == 'https' { -%> tls_cert = "/etc/telegraf/puppet_cert.pem" tls_key = "/etc/telegraf/puppet_key.pem" diff --git a/templates/puppetdb_jvm_metrics.yaml.epp b/templates/puppetdb_jvm_metrics.yaml.epp index 3357a79..14690e0 100644 --- a/templates/puppetdb_jvm_metrics.yaml.epp +++ b/templates/puppetdb_jvm_metrics.yaml.epp @@ -1,4 +1,9 @@ -<%- | Array[String] $urls, Enum['https', 'http'] $protocol, Integer[1] $http_timeout_seconds | -%> +<%- | + Array[String] $urls, + Enum['https', 'http'] $protocol, + Integer[1] $http_timeout_seconds, + Boolean $include_pe_metrics, + | -%> <% if $protocol == 'https' { -%> tls_cert: /etc/telegraf/puppet_cert.pem tls_key: /etc/telegraf/puppet_key.pem diff --git a/templates/puppetdb_metrics.toml.epp b/templates/puppetdb_metrics.toml.epp index d190231..e9e2528 100644 --- a/templates/puppetdb_metrics.toml.epp +++ b/templates/puppetdb_metrics.toml.epp @@ -1,4 +1,9 @@ -<%- | Array[String] $urls, Enum['https', 'http'] $protocol, Integer[1] $http_timeout_seconds | -%> +<%- | + Array[String] $urls, + Enum['https', 'http'] $protocol, + Integer[1] $http_timeout_seconds, + Boolean $include_pe_metrics, + | -%> <% if $protocol == 'https' { -%> tls_cert = "/etc/telegraf/puppet_cert.pem" tls_key = "/etc/telegraf/puppet_key.pem" diff --git a/templates/puppetdb_metrics.yaml.epp b/templates/puppetdb_metrics.yaml.epp index e5939fb..e50a37a 100644 --- a/templates/puppetdb_metrics.yaml.epp +++ b/templates/puppetdb_metrics.yaml.epp @@ -1,4 +1,9 @@ -<%- | Array[String] $urls, Enum['https', 'http'] $protocol, Integer[1] $http_timeout_seconds | -%> +<%- | + Array[String] $urls, + Enum['https', 'http'] $protocol, + Integer[1] $http_timeout_seconds, + Boolean $include_pe_metrics, + | -%> <% if $protocol == 'https' { -%> tls_cert: /etc/telegraf/puppet_cert.pem tls_key: /etc/telegraf/puppet_key.pem diff --git a/templates/puppetserver_metrics.toml.epp b/templates/puppetserver_metrics.toml.epp index 47ad1e1..bbccefc 100644 --- a/templates/puppetserver_metrics.toml.epp +++ b/templates/puppetserver_metrics.toml.epp @@ -1,4 +1,9 @@ -<%- | Array[String] $urls, Enum['https', 'http'] $protocol, Integer[1] $http_timeout_seconds | -%> +<%- | + Array[String] $urls, + Enum['https', 'http'] $protocol, + Integer[1] $http_timeout_seconds, + Boolean $include_pe_metrics + | -%> <% if $protocol == 'https' { -%> tls_cert = "/etc/telegraf/puppet_cert.pem" tls_key = "/etc/telegraf/puppet_key.pem" @@ -41,6 +46,7 @@ urls = <%= $urls %> [[json_v2.object]] path = 'jruby-metrics.status.experimental.metrics' disable_prepend_keys = false +<% if $include_pe_metrics { -%> [[json_v2.object]] path = "@this" disable_prepend_keys = false @@ -49,3 +55,4 @@ urls = <%= $urls %> path = "@this" disable_prepend_keys = false included_keys = ['broker-service_status_metrics_puppetlabs.pcp.on-close_rates_1', 'broker-service_status_metrics_puppetlabs.pcp.on-close_rates_5', 'broker-service_status_metrics_puppetlabs.pcp.on-close_rates_15', 'broker-service_status_metrics_puppetlabs.pcp.on-close_rates_total', 'broker-service_status_metrics_puppetlabs.pcp.on-close_mean', 'broker-service_status_metrics_puppetlabs.pcp.on-close_std-dev', 'broker-service_status_metrics_puppetlabs.pcp.on-close_percentiles', 'broker-service_status_metrics_puppetlabs.pcp.on-close_percentiles_0.75', 'broker-service_status_metrics_puppetlabs.pcp.on-close_percentiles_0.95', 'broker-service_status_metrics_puppetlabs.pcp.on-close_percentiles_0.99', 'broker-service_status_metrics_puppetlabs.pcp.on-close_percentiles_0.999', 'broker-service_status_metrics_puppetlabs.pcp.on-close_percentiles_1.0', 'broker-service_status_metrics_puppetlabs.pcp.on-close_largest', 'broker-service_status_metrics_puppetlabs.pcp.on-close_smallest', 'broker-service_status_metrics_puppetlabs.pcp.on-connect_rates_1', 'broker-service_status_metrics_puppetlabs.pcp.on-connect_rates_5', 'broker-service_status_metrics_puppetlabs.pcp.on-connect_rates_15', 'broker-service_status_metrics_puppetlabs.pcp.on-connect_rates_total', 'broker-service_status_metrics_puppetlabs.pcp.on-connect_mean', 'broker-service_status_metrics_puppetlabs.pcp.on-connect_std-dev', 'broker-service_status_metrics_puppetlabs.pcp.on-connect_percentiles', 'broker-service_status_metrics_puppetlabs.pcp.on-connect_percentiles_0.75', 'broker-service_status_metrics_puppetlabs.pcp.on-connect_percentiles_0.95', 'broker-service_status_metrics_puppetlabs.pcp.on-connect_percentiles_0.99', 'broker-service_status_metrics_puppetlabs.pcp.on-connect_percentiles_0.999', 'broker-service_status_metrics_puppetlabs.pcp.on-connect_percentiles_1.0', 'broker-service_status_metrics_puppetlabs.pcp.on-connect_largest', 'broker-service_status_metrics_puppetlabs.pcp.on-connect_smallest', 'broker-service_status_metrics_puppetlabs.pcp.on-message_rates_1', 'broker-service_status_metrics_puppetlabs.pcp.on-message_rates_5', 'broker-service_status_metrics_puppetlabs.pcp.on-message_rates_15', 'broker-service_status_metrics_puppetlabs.pcp.on-message_rates_total', 'broker-service_status_metrics_puppetlabs.pcp.on-message_mean', 'broker-service_status_metrics_puppetlabs.pcp.on-message_std-dev', 'broker-service_status_metrics_puppetlabs.pcp.on-message_percentiles', 'broker-service_status_metrics_puppetlabs.pcp.on-message_percentiles_0.75', 'broker-service_status_metrics_puppetlabs.pcp.on-message_percentiles_0.95', 'broker-service_status_metrics_puppetlabs.pcp.on-message_percentiles_0.99', 'broker-service_status_metrics_puppetlabs.pcp.on-message_percentiles_0.999', 'broker-service_status_metrics_puppetlabs.pcp.on-message_percentiles_1.0', 'broker-service_status_metrics_puppetlabs.pcp.on-message_largest', 'broker-service_status_metrics_puppetlabs.pcp.on-message_smallest', 'broker-service_status_metrics_puppetlabs.pcp.on-send_rates_1', 'broker-service_status_metrics_puppetlabs.pcp.on-send_rates_5', 'broker-service_status_metrics_puppetlabs.pcp.on-send_rates_15', 'broker-service_status_metrics_puppetlabs.pcp.on-send_rates_total', 'broker-service_status_metrics_puppetlabs.pcp.on-send_mean', 'broker-service_status_metrics_puppetlabs.pcp.on-send_std-dev', 'broker-service_status_metrics_puppetlabs.pcp.on-send_percentiles', 'broker-service_status_metrics_puppetlabs.pcp.on-send_percentiles_0.75', 'broker-service_status_metrics_puppetlabs.pcp.on-send_percentiles_0.95', 'broker-service_status_metrics_puppetlabs.pcp.on-send_percentiles_0.99', 'broker-service_status_metrics_puppetlabs.pcp.on-send_percentiles_0.999', 'broker-service_status_metrics_puppetlabs.pcp.on-send_percentiles_1.0', 'broker-service_status_metrics_puppetlabs.pcp.on-send_largest', 'broker-service_status_metrics_puppetlabs.pcp.on-send_smallest'] +<% } -%> diff --git a/templates/puppetserver_metrics.yaml.epp b/templates/puppetserver_metrics.yaml.epp index b5aba0e..cb55b27 100644 --- a/templates/puppetserver_metrics.yaml.epp +++ b/templates/puppetserver_metrics.yaml.epp @@ -1,4 +1,9 @@ -<%- | Array[String] $urls, Enum['https', 'http'] $protocol, Integer[1] $http_timeout_seconds | -%> +<%- | + Array[String] $urls, + Enum['https', 'http'] $protocol, + Integer[1] $http_timeout_seconds, + Boolean $include_pe_metrics, + | -%> <% if $protocol == 'https' { -%> tls_cert: /etc/telegraf/puppet_cert.pem tls_key: /etc/telegraf/puppet_key.pem @@ -41,6 +46,7 @@ json_v2: - metric - path: jruby-metrics.status.experimental.metrics disable_prepend_keys: false +<% if $include_pe_metrics { -%> - path: '@this' disable_prepend_keys: false included_keys: @@ -158,3 +164,4 @@ json_v2: - broker-service_status_metrics_puppetlabs.pcp.on-send_percentiles_1.0 - broker-service_status_metrics_puppetlabs.pcp.on-send_largest - broker-service_status_metrics_puppetlabs.pcp.on-send_smallest +<% } -%> From 4ffbe792dddf60e35b63f76e7d5a10be4b4d9b3b Mon Sep 17 00:00:00 2001 From: Adrian Parreiras Horta Date: Wed, 22 Oct 2025 15:44:37 -0700 Subject: [PATCH 4/9] Add parameter for extra options to Telegraf inputs This commit adds a new parameter for passing additional hash elements to the options parameter of the Telegraf inputs used in this module. --- manifests/telegraf/agent.pp | 9 +++++ manifests/telegraf/config.pp | 8 ++++- ...ational_dashboards_telegraf_config_spec.rb | 33 +++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/manifests/telegraf/agent.pp b/manifests/telegraf/agent.pp index 8cb6339..e69ac07 100644 --- a/manifests/telegraf/agent.pp +++ b/manifests/telegraf/agent.pp @@ -146,6 +146,7 @@ 'sslcert' => '/etc/telegraf/puppet_cert.pem', 'sslrootcert' => '/etc/telegraf/puppet_ca.pem', }, + Optional[Hash] $extra_input_options = undef, Enum['yaml','toml'] $template_format = $puppet_operational_dashboards::template_format, ) { unless [$puppetserver_hosts, $puppetdb_hosts, $postgres_hosts, $profiles, $local_services].any |$service| { $service } { @@ -350,6 +351,7 @@ http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, include_pe_metrics => $include_pe_metrics, + extra_input_options => $extra_input_options, } } @@ -360,6 +362,7 @@ http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, include_pe_metrics => $include_pe_metrics, + extra_input_options => $extra_input_options, } } @@ -370,6 +373,7 @@ http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, include_pe_metrics => $include_pe_metrics, + extra_input_options => $extra_input_options, } } @@ -417,6 +421,7 @@ http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, include_pe_metrics => $include_pe_metrics, + extra_input_options => $extra_input_options, } } } @@ -434,6 +439,7 @@ http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, include_pe_metrics => $include_pe_metrics, + extra_input_options => $extra_input_options, } } @@ -444,6 +450,7 @@ http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, include_pe_metrics => $include_pe_metrics, + extra_input_options => $extra_input_options, } if $include_pe_metrics { puppet_operational_dashboards::telegraf::config { 'pcp': @@ -452,6 +459,7 @@ http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, include_pe_metrics => $include_pe_metrics, + extra_input_options => $extra_input_options, } } } @@ -463,6 +471,7 @@ http_timeout_seconds => $http_timeout_seconds, template_format => $template_format, include_pe_metrics => $include_pe_metrics, + extra_input_options => $extra_input_options, } } diff --git a/manifests/telegraf/config.pp b/manifests/telegraf/config.pp index f6026cc..da8bda5 100644 --- a/manifests/telegraf/config.pp +++ b/manifests/telegraf/config.pp @@ -21,6 +21,7 @@ Enum['present', 'absent'] $ensure = 'present', Enum['yaml','toml'] $template_format = 'toml', Boolean $include_pe_metrics, + Optional[Hash] $extra_input_options = undef, ) { unless $service in ['puppetserver', 'puppetdb', 'puppetdb_jvm', 'orchestrator', 'pcp'] { fail("Unknown service type ${service}") @@ -56,7 +57,12 @@ telegraf::input { "${service}_metrics": plugin_type => 'http', - options => [$_inputs], + options => if $extra_input_options { + [$_inputs + $extra_input_options] + } + else { + [$_inputs] + }, } # Create processors.strings.rename entries to rename full url to hostname diff --git a/spec/defines/puppet_operational_dashboards_telegraf_config_spec.rb b/spec/defines/puppet_operational_dashboards_telegraf_config_spec.rb index 6ceb21e..e9c39a2 100644 --- a/spec/defines/puppet_operational_dashboards_telegraf_config_spec.rb +++ b/spec/defines/puppet_operational_dashboards_telegraf_config_spec.rb @@ -1,5 +1,6 @@ require 'spec_helper' require 'json' +require 'pry' describe 'puppet_operational_dashboards::telegraf::config' do let(:facts) { { os: { family: 'RedHat' } } } @@ -169,4 +170,36 @@ ) } end + + context 'when passing extra input options' do + let(:title) { 'puppetserver' } + let(:puppetserver_epp) do + JSON.parse(File.read('./spec/fixtures/defines/puppetserver_metrics.json')) + end + let(:params) do + { + ensure: 'present', + protocol: 'https', + http_timeout_seconds: 5, + hosts: ['localhost.foo.com'], + include_pe_metrics: true, + extra_input_options: { + tags: { + 'foo' => 'bar', + } + } + } + + end + + it { + is_expected.to compile + + # Testing that the Telegraf input contains identical Ruby objects was turning out to be difficult + # It was returning an error but saying the diffs were identical, so instead we just check the file + is_expected.to contain_file('/etc/telegraf/telegraf.d/puppetserver_metrics.conf').with_content( + %r{\[inputs\.http\.tags]\nfoo = \"bar\"} + ) + } + end end From e7a46039b12d970059f9640726fc045984ff7683 Mon Sep 17 00:00:00 2001 From: Adrian Parreiras Horta Date: Fri, 24 Oct 2025 10:25:01 -0700 Subject: [PATCH 5/9] Lint and release prep --- CHANGELOG.md | 11 +++ README.md | 67 +++++++++++++++++++ REFERENCE.md | 59 +++++++++++++++- functions/pe_profiles_on_host.pp | 7 +- manifests/telegraf/agent.pp | 4 +- manifests/telegraf/config.pp | 14 ++-- ...ational_dashboards_telegraf_config_spec.rb | 3 +- 7 files changed, 154 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d9e91d..3462151 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org). +## [v2.4.0](https://github.com/puppetlabs/puppet_operational_dashboards/tree/v2.3.0) + +[Full Changelog](https://github.com/puppetlabs/puppet_operational_dashboards/compare/v2.3.0...v2.4.0) + +### Added + +- Add new management params +- Add support for InfluxDB 1 in Telegraf outputs +- Make file-sync metrics configurable +- Add parameter for extra options to Telegraf inputs + ## [v2.3.0](https://github.com/puppetlabs/puppet_operational_dashboards/tree/v2.3.0) (2024-02-05) [Full Changelog](https://github.com/puppetlabs/puppet_operational_dashboards/compare/v2.2.0...v2.3.0) diff --git a/README.md b/README.md index c24d6bd..8723c25 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ - [Beginning with puppet_operational_dashboards](#beginning-with-puppet_operational_dashboards) - [Installing on Puppet Enterprise](#installing-on-puppet-enterprise) - [Installing on Puppet Open Source](#installing-on-puppet-open-source) + - [Advanced Telegraf Configuration](#advanced-telegraf-configuration) - [What puppet_operational_dashboards affects](#what-puppet_operational_dashboards-affects) - [Usage](#usage) - [Evaluation order](#evaluation-order) @@ -221,6 +222,72 @@ The easiest way to get started using this module is by including the `puppet_ope include puppet_operational_dashboards ``` +#### Advanced Telegraf Configuration + +If you already have an existing setup and defined configuration for Telegraf and InfluxDB, it is possible to only configure the Telegraf inputs needed for collection of Puppet metrics via the `puppet_operational_dashboards::telegraf::agent` class. The main use case for this is if you already manage Telegraf and InfluxDB in Puppet code, and you only want to use this module to collect and ship Puppet metrics to your existing InfluxDB. To do so, you may apply the `puppet_operational_dashboards::telegraf::agent` class to either: + +* Each Puppet infrastructure node to collect metrics locally +* A single node to remotely collect metrics from all infrastructure nodes + +For the first case of local collection, set `collection_method: 'local'` and `local_services` to an array of Puppet services to collect from, for example `local_services: ['puppetserver', 'puppetdb']`. + +For the second case of remote collection, use `collection_method: 'all'` and set each of the following as needed: + +* puppetserver_hosts +* orchestrator_hosts +* puppetdb_hosts +* postgres_hosts + +In either case, you can control whether this class will manage the Telegraf class and service with: + +```bash +puppet_operational_dashboards::telegraf::agent::manage_class +``` + +Whether to manage Telegraf outputs with: + +```bash +puppet_operational_dashboards::telegraf::agent::manage_outputs +``` + +And whether to configure token authentication in the inputs with: +```bash +puppet_operational_dashboards::telegraf::agent::use_token_auth +``` + +You may also pass additional options to the Telegraf inputs, for example, if you need to add a tag. The following hiera data will add an additional `[inputs.http.tags]` element with a key/value pair of `foo = bar`: + +```bash +puppet_operational_dashboards::telegraf::agent::extra_input_options: + tags: + foo: 'bar' +``` + +The following is a complete example to configure a Telegraf input for collecting Puppet server metrics locally with an additional tag: + +```bash +puppet_operational_dashboards::telegraf::agent::local_services: + - 'puppetserver' +puppet_operational_dashboards::telegraf::agent::token_name: 'puppetlabs' +puppet_operational_dashboards::telegraf::agent::influxdb_token_file: '/root/.influxdb_token' +puppet_operational_dashboards::telegraf::agent::influxdb_bucket: 'puppet_data' +puppet_operational_dashboards::telegraf::agent::influxdb_org: 'puppetlabs' +puppet_operational_dashboards::telegraf::agent::include_pe_metrics: false +puppet_operational_dashboards::telegraf::agent::manage_repo: false +puppet_operational_dashboards::telegraf::agent::manage_class: false +puppet_operational_dashboards::telegraf::agent::influxdb_host: 'foo.bar.com' +puppet_operational_dashboards::telegraf::agent::influxdb_port: 8086 +puppet_operational_dashboards::telegraf::agent::use_token_auth: false +puppet_operational_dashboards::telegraf::agent::manage_outputs: false +puppet_operational_dashboards::telegraf::agent::collection_method: 'local' +puppet_operational_dashboards::telegraf::agent::use_ssl: true +puppet_operational_dashboards::telegraf::agent::use_system_store: false +puppet_operational_dashboards::telegraf::agent::template_format: 'yaml' +puppet_operational_dashboards::telegraf::agent::extra_input_options: + tags: + foo: 'bar' +``` + #### What puppet_operational_dashboards affects Installing the module will: diff --git a/REFERENCE.md b/REFERENCE.md index 6d05f3a..31a76bc 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -620,8 +620,11 @@ The following parameters are available in the `puppet_operational_dashboards::te * [`use_system_store`](#-puppet_operational_dashboards--telegraf--agent--use_system_store) * [`manage_ssl`](#-puppet_operational_dashboards--telegraf--agent--manage_ssl) * [`manage_repo`](#-puppet_operational_dashboards--telegraf--agent--manage_repo) +* [`manage_class`](#-puppet_operational_dashboards--telegraf--agent--manage_class) +* [`use_token_auth`](#-puppet_operational_dashboards--telegraf--agent--use_token_auth) * [`manage_archive`](#-puppet_operational_dashboards--telegraf--agent--manage_archive) * [`manage_user`](#-puppet_operational_dashboards--telegraf--agent--manage_user) +* [`manage_outputs`](#-puppet_operational_dashboards--telegraf--agent--manage_outputs) * [`ssl_cert_file`](#-puppet_operational_dashboards--telegraf--agent--ssl_cert_file) * [`ssl_key_file`](#-puppet_operational_dashboards--telegraf--agent--ssl_key_file) * [`ssl_ca_file`](#-puppet_operational_dashboards--telegraf--agent--ssl_ca_file) @@ -630,6 +633,7 @@ The following parameters are available in the `puppet_operational_dashboards::te * [`puppet_ssl_ca_file`](#-puppet_operational_dashboards--telegraf--agent--puppet_ssl_ca_file) * [`insecure_skip_verify`](#-puppet_operational_dashboards--telegraf--agent--insecure_skip_verify) * [`version`](#-puppet_operational_dashboards--telegraf--agent--version) +* [`influxdb_version`](#-puppet_operational_dashboards--telegraf--agent--influxdb_version) * [`archive_location`](#-puppet_operational_dashboards--telegraf--agent--archive_location) * [`archive_install_dir`](#-puppet_operational_dashboards--telegraf--agent--archive_install_dir) * [`collection_method`](#-puppet_operational_dashboards--telegraf--agent--collection_method) @@ -648,6 +652,7 @@ The following parameters are available in the `puppet_operational_dashboards::te * [`telegraf_postgres_password`](#-puppet_operational_dashboards--telegraf--agent--telegraf_postgres_password) * [`postgres_port`](#-puppet_operational_dashboards--telegraf--agent--postgres_port) * [`postgres_options`](#-puppet_operational_dashboards--telegraf--agent--postgres_options) +* [`extra_input_options`](#-puppet_operational_dashboards--telegraf--agent--extra_input_options) * [`template_format`](#-puppet_operational_dashboards--telegraf--agent--template_format) ##### `token` @@ -720,6 +725,18 @@ Data type: `Boolean` Whether to install Telegraf from a repository. +##### `manage_class` + +Data type: `Boolean` + +Whether to manage the Telegraf class and related resources + +##### `use_token_auth` + +Data type: `Boolean` + +Whether to set up and use token based auth to InfluxDB + ##### `manage_archive` Data type: `Boolean` @@ -736,6 +753,14 @@ Whether to manage the telegraf user when installing from archive. Default value: `true` +##### `manage_outputs` + +Data type: `Boolean` + +Whether to manage the telegraf outputs + +Default value: `true` + ##### `ssl_cert_file` Data type: `Stdlib::Absolutepath` @@ -798,6 +823,14 @@ Data type: `String` Version of the Telegraf package to install. +##### `influxdb_version` + +Data type: `Enum['v1', 'v2']` + +Which version of InfluxDB to use in the inputs. Currently supports versions 1 and 2 + +Default value: `'v2'` + ##### `archive_location` Data type: `String` @@ -956,6 +989,14 @@ Default value: } ``` +##### `extra_input_options` + +Data type: `Optional[Hash]` + +Optional hash of extra values to pass to each telegraf::input declared in this module + +Default value: `undef` + ##### `template_format` Data type: `Enum['yaml','toml']` @@ -980,6 +1021,8 @@ The following parameters are available in the `puppet_operational_dashboards::te * [`ensure`](#-puppet_operational_dashboards--telegraf--config--ensure) * [`http_timeout_seconds`](#-puppet_operational_dashboards--telegraf--config--http_timeout_seconds) * [`template_format`](#-puppet_operational_dashboards--telegraf--config--template_format) +* [`include_pe_metrics`](#-puppet_operational_dashboards--telegraf--config--include_pe_metrics) +* [`extra_input_options`](#-puppet_operational_dashboards--telegraf--config--extra_input_options) ##### `service` @@ -1023,6 +1066,20 @@ Template format to use for puppet template toml or yaml config Default value: `'toml'` +##### `include_pe_metrics` + +Data type: `Boolean` + +Whether to include Filesync metrics in Puppetserver + +##### `extra_input_options` + +Data type: `Optional[Hash]` + +Optional hash of extra values to pass to each telegraf::input declared in this module + +Default value: `undef` + ## Functions ### `puppet_operational_dashboards::hosts_with_profile` @@ -1194,7 +1251,7 @@ Default value: ```puppet [{ 'type' => 'expire', - 'everySeconds' => 3456000, + 'everySeconds' => 7776000, 'shardGroupDurationSeconds' => 604800, }] ``` diff --git a/functions/pe_profiles_on_host.pp b/functions/pe_profiles_on_host.pp index 7a63fc0..7efd12f 100644 --- a/functions/pe_profiles_on_host.pp +++ b/functions/pe_profiles_on_host.pp @@ -10,7 +10,12 @@ function puppet_operational_dashboards::pe_profiles_on_host() >> Array[String] { $hosts = puppetdb_query("resources[title] { type = 'Class' and certname = '${trusted['certname']}' and - title in ['Puppet_enterprise::Profile::Puppetdb', 'Puppet_enterprise::Profile::Master', 'Puppet_enterprise::Profile::Database', 'Puppet_enterprise::Profile::Orchestrator'] and + title in [ + 'Puppet_enterprise::Profile::Puppetdb', + 'Puppet_enterprise::Profile::Master', + 'Puppet_enterprise::Profile::Database', + 'Puppet_enterprise::Profile::Orchestrator' + ] and nodes { deactivated is null and expired is null } }").map |$nodes| { $nodes['title'] } } else { diff --git a/manifests/telegraf/agent.pp b/manifests/telegraf/agent.pp index e69ac07..0a80791 100644 --- a/manifests/telegraf/agent.pp +++ b/manifests/telegraf/agent.pp @@ -92,14 +92,16 @@ # Port for the Telegraf client to use in the postgres connection string # @param postgres_options # Hash of options for the Telegraf client to use as connection parameters in the postgres connection string +# @param extra_input_options +# Optional hash of extra values to pass to each telegraf::input declared in this module # @param template_format # Template format to use for puppet template toml or yaml config class puppet_operational_dashboards::telegraf::agent ( String $version, - Enum['v1', 'v2'] $influxdb_version = 'v2', Boolean $manage_repo, Boolean $manage_class, Boolean $use_token_auth, + Enum['v1', 'v2'] $influxdb_version = 'v2', Optional[Sensitive[String]] $token = $puppet_operational_dashboards::telegraf_token, String $token_name = $puppet_operational_dashboards::telegraf_token_name, String $influxdb_token_file = $puppet_operational_dashboards::influxdb_token_file, diff --git a/manifests/telegraf/config.pp b/manifests/telegraf/config.pp index da8bda5..03b9b72 100644 --- a/manifests/telegraf/config.pp +++ b/manifests/telegraf/config.pp @@ -13,14 +13,16 @@ # Template format to use for puppet template toml or yaml config # @param include_pe_metrics # Whether to include Filesync metrics in Puppetserver +# @param extra_input_options +# Optional hash of extra values to pass to each telegraf::input declared in this module define puppet_operational_dashboards::telegraf::config ( + Boolean $include_pe_metrics, Array[String[1]] $hosts, Enum['https', 'http'] $protocol, Integer[1] $http_timeout_seconds, String $service = $title, Enum['present', 'absent'] $ensure = 'present', Enum['yaml','toml'] $template_format = 'toml', - Boolean $include_pe_metrics, Optional[Hash] $extra_input_options = undef, ) { unless $service in ['puppetserver', 'puppetdb', 'puppetdb_jvm', 'orchestrator', 'pcp'] { @@ -58,11 +60,11 @@ telegraf::input { "${service}_metrics": plugin_type => 'http', options => if $extra_input_options { - [$_inputs + $extra_input_options] - } - else { - [$_inputs] - }, + [$_inputs + $extra_input_options] + } + else { + [$_inputs] + }, } # Create processors.strings.rename entries to rename full url to hostname diff --git a/spec/defines/puppet_operational_dashboards_telegraf_config_spec.rb b/spec/defines/puppet_operational_dashboards_telegraf_config_spec.rb index e9c39a2..ae46ed1 100644 --- a/spec/defines/puppet_operational_dashboards_telegraf_config_spec.rb +++ b/spec/defines/puppet_operational_dashboards_telegraf_config_spec.rb @@ -189,7 +189,6 @@ } } } - end it { @@ -198,7 +197,7 @@ # Testing that the Telegraf input contains identical Ruby objects was turning out to be difficult # It was returning an error but saying the diffs were identical, so instead we just check the file is_expected.to contain_file('/etc/telegraf/telegraf.d/puppetserver_metrics.conf').with_content( - %r{\[inputs\.http\.tags]\nfoo = \"bar\"} + %r{\[inputs\.http\.tags\]\nfoo = \"bar\"}, ) } end From 22173128f6ef645da1179793ef4e8b5a841e4bdd Mon Sep 17 00:00:00 2001 From: Adrian Parreiras Horta Date: Fri, 24 Oct 2025 10:38:19 -0700 Subject: [PATCH 6/9] Update examples in README to yaml format --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8723c25..f1bc1ef 100644 --- a/README.md +++ b/README.md @@ -240,24 +240,24 @@ For the second case of remote collection, use `collection_method: 'all'` and set In either case, you can control whether this class will manage the Telegraf class and service with: -```bash +```yaml puppet_operational_dashboards::telegraf::agent::manage_class ``` Whether to manage Telegraf outputs with: -```bash +```yaml puppet_operational_dashboards::telegraf::agent::manage_outputs ``` And whether to configure token authentication in the inputs with: -```bash +```yaml puppet_operational_dashboards::telegraf::agent::use_token_auth ``` You may also pass additional options to the Telegraf inputs, for example, if you need to add a tag. The following hiera data will add an additional `[inputs.http.tags]` element with a key/value pair of `foo = bar`: -```bash +```yaml puppet_operational_dashboards::telegraf::agent::extra_input_options: tags: foo: 'bar' @@ -265,7 +265,7 @@ puppet_operational_dashboards::telegraf::agent::extra_input_options: The following is a complete example to configure a Telegraf input for collecting Puppet server metrics locally with an additional tag: -```bash +```yaml puppet_operational_dashboards::telegraf::agent::local_services: - 'puppetserver' puppet_operational_dashboards::telegraf::agent::token_name: 'puppetlabs' From 78e9560717ae7fa6e4598e81e724ddb2b858fbe0 Mon Sep 17 00:00:00 2001 From: Adrian Parreiras Horta Date: Thu, 30 Oct 2025 11:31:56 -0700 Subject: [PATCH 7/9] Pass extra input options to postgres input --- manifests/telegraf/agent.pp | 14 ++++++++++-- spec/classes/agent_spec.rb | 43 +++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/manifests/telegraf/agent.pp b/manifests/telegraf/agent.pp index 0a80791..30cd58f 100644 --- a/manifests/telegraf/agent.pp +++ b/manifests/telegraf/agent.pp @@ -400,7 +400,12 @@ telegraf::input { "postgres_${pg_host}": plugin_type => 'postgresql_extensible', - options => [$_inputs], + options => if $extra_input_options { + [$_inputs + $extra_input_options] + } + else { + [$_inputs] + }, } } } @@ -497,7 +502,12 @@ telegraf::input { "postgres_${trusted['certname']}": plugin_type => 'postgresql_extensible', - options => [$_inputs], + options => if $extra_input_options { + [$_inputs + $extra_input_options] + } + else { + [$_inputs] + }, } } } diff --git a/spec/classes/agent_spec.rb b/spec/classes/agent_spec.rb index f6a0ea3..882b860 100644 --- a/spec/classes/agent_spec.rb +++ b/spec/classes/agent_spec.rb @@ -569,6 +569,49 @@ is_expected.not_to contain_file('/etc/systemd/system/telegraf.service.d/override.conf') } end + + context 'when passing extra input options' do + let(:params) do + { + token: RSpec::Puppet::Sensitive.new(nil), + token_name: 'puppet telegraf token', + influxdb_token_file: '/root/.influxdb_token', + influxdb_host: 'localhost.foo.com', + influxdb_port: 8086, + influxdb_bucket: 'puppet_data', + influxdb_org: 'puppetlabs', + use_ssl: true, + use_system_store: false, + collection_method: 'local', + local_services: ['puppetserver', 'postgres'], + include_pe_metrics: false, + template_format: 'toml', + use_token_auth: false, + postgres_options: { + 'sslmode': 'verify-full', + 'sslkey': '/etc/telegraf/puppet_key.pem', + 'sslcert': '/etc/telegraf/puppet_cert.pem', + 'sslrootcert': '/etc/telegraf/puppet_ca.pem', + }, + postgres_hosts: ['localhost.foo.com'], + extra_input_options: { + tags: { + 'foo' => 'bar', + }, + }, + } + end + + it { + is_expected.to compile + + # Testing that the Telegraf input contains identical Ruby objects was turning out to be difficult + # It was returning an error but saying the diffs were identical, so instead we just check the file + is_expected.to contain_file('/etc/telegraf/telegraf.d/postgres_localhost.foo.com.conf').with_content( + %r{\[inputs\.postgresql_extensible\.tags\]\nfoo = \"bar\"}, + ) + } + end end end end From 1a2f573805d67e0e8e654783a370e0d11c14321c Mon Sep 17 00:00:00 2001 From: Adrian Parreiras Horta Date: Thu, 30 Oct 2025 12:59:37 -0700 Subject: [PATCH 8/9] Fix postgres templates This commit fixes the hardcoded database name and removed the version string. This is not needed anymore on newer postgresql versions --- templates/postgres.toml.epp | 7 +------ templates/postgres.yaml.epp | 7 +------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/templates/postgres.toml.epp b/templates/postgres.toml.epp index 62505a0..b758920 100644 --- a/templates/postgres.toml.epp +++ b/templates/postgres.toml.epp @@ -12,29 +12,24 @@ address = "postgres://<%=$telegraf_user%>:<%=$password.unwrap%>@<%=$certname%>:< <% else { -%> address = "postgres://<%=$telegraf_user%>@<%=$certname%>:<%=$port%>/<%=$database%>?<%=$connection_params%>" <% } -%> -databases = ["pe-puppetdb"] +databases = ["<%=$database%>"] outputaddress = "<%=$certname%>" [[query]] sqlquery = "SELECT * FROM pg_stat_database" -version = 901 withdbname = false [[query]] tagvalue = "table_name" -version = 901 withdbname = false sqlquery = "SELECT current_database() AS datname, total_bytes AS total , table_name , index_bytes AS index , toast_bytes AS toast , table_bytes AS table FROM ( SELECT *, total_bytes-index_bytes-coalesce(toast_bytes,0) AS table_bytes FROM ( SELECT c.oid,nspname AS table_schema, relname AS table_name , c.reltuples AS row_estimate , pg_total_relation_size(c.oid) AS total_bytes , pg_indexes_size(c.oid) AS index_bytes , pg_total_relation_size(reltoastrelid) AS toast_bytes FROM pg_class c LEFT JOIN pg_namespace n ON n.oid = c.relnamespace WHERE relkind = 'r' AND nspname NOT IN ('pg_catalog', 'information_schema')) a) a" [[query]] sqlquery = "SELECT current_database() AS datname, relname as table, autovacuum_count, vacuum_count, n_live_tup, n_dead_tup FROM pg_stat_user_tables" tagvalue = "table" -version = 901 withdbname = false [[query]] sqlquery = "SELECT current_database() AS datname, a.indexrelname as index, pg_relation_size(a.indexrelid) as size_bytes, idx_scan, idx_tup_read, idx_tup_fetch, idx_blks_read, idx_blks_hit from pg_stat_user_indexes a join pg_statio_user_indexes b on a.indexrelid = b.indexrelid;" tagvalue = "index" -version = 901 withdbname = false [[query]] sqlquery = "SELECT current_database() AS datname, relname as table, heap_blks_read, heap_blks_hit, idx_blks_read, idx_blks_hit, toast_blks_read, toast_blks_hit, tidx_blks_read, tidx_blks_hit FROM pg_statio_user_tables" tagvalue = "table" -version = 901 withdbname = false diff --git a/templates/postgres.yaml.epp b/templates/postgres.yaml.epp index 0402e40..a5832b4 100644 --- a/templates/postgres.yaml.epp +++ b/templates/postgres.yaml.epp @@ -15,14 +15,12 @@ address: >- postgres://<%=$telegraf_user%>@<%=$certname%>:<%=$port%>/<%=$database%>?<%=$connection_params%> <% } -%> databases: - - pe-puppetdb + - <%=$database%> outputaddress: <%=$certname%> query: - sqlquery: SELECT * FROM pg_stat_database - version: 901 withdbname: false - tagvalue: table_name - version: 901 withdbname: false sqlquery: >- SELECT current_database() AS datname, total_bytes AS total , table_name , @@ -38,7 +36,6 @@ query: SELECT current_database() AS datname, relname as table, autovacuum_count, vacuum_count, n_live_tup, n_dead_tup FROM pg_stat_user_tables tagvalue: table - version: 901 withdbname: false - sqlquery: >- SELECT current_database() AS datname, a.indexrelname as index, @@ -46,12 +43,10 @@ query: idx_tup_fetch, idx_blks_read, idx_blks_hit from pg_stat_user_indexes a join pg_statio_user_indexes b on a.indexrelid = b.indexrelid; tagvalue: index - version: 901 withdbname: false - sqlquery: >- SELECT current_database() AS datname, relname as table, heap_blks_read, heap_blks_hit, idx_blks_read, idx_blks_hit, toast_blks_read, toast_blks_hit, tidx_blks_read, tidx_blks_hit FROM pg_statio_user_tables tagvalue: table - version: 901 withdbname: false From 5a9c57f1ff67c152bc72dd2be46db7d8c9d9d36b Mon Sep 17 00:00:00 2001 From: Adrian Parreiras Horta Date: Thu, 30 Oct 2025 13:08:21 -0700 Subject: [PATCH 9/9] Use localhost as the hostname for local postgres --- manifests/telegraf/agent.pp | 2 +- spec/classes/agent_spec.rb | 15 +++------------ 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/manifests/telegraf/agent.pp b/manifests/telegraf/agent.pp index 30cd58f..0b4da05 100644 --- a/manifests/telegraf/agent.pp +++ b/manifests/telegraf/agent.pp @@ -487,7 +487,7 @@ $inputs = epp( "puppet_operational_dashboards/postgres.${template_format}.epp", { - certname => $trusted['certname'], + certname => 'localhost', telegraf_user => $telegraf_user, password => $telegraf_postgres_password, database => $database, diff --git a/spec/classes/agent_spec.rb b/spec/classes/agent_spec.rb index 882b860..d492f42 100644 --- a/spec/classes/agent_spec.rb +++ b/spec/classes/agent_spec.rb @@ -157,22 +157,17 @@ 'outputaddress' => 'localhost.foo.com', 'query' => [ { 'sqlquery' => 'SELECT * FROM pg_stat_database', - 'version' => 901, 'withdbname' => false }, { 'tagvalue' => 'table_name', - 'version' => 901, 'withdbname' => false, 'sqlquery' => "SELECT current_database() AS datname, total_bytes AS total , table_name , index_bytes AS index , toast_bytes AS toast , table_bytes AS table FROM ( SELECT *, total_bytes-index_bytes-coalesce(toast_bytes,0) AS table_bytes FROM ( SELECT c.oid,nspname AS table_schema, relname AS table_name , c.reltuples AS row_estimate , pg_total_relation_size(c.oid) AS total_bytes , pg_indexes_size(c.oid) AS index_bytes , pg_total_relation_size(reltoastrelid) AS toast_bytes FROM pg_class c LEFT JOIN pg_namespace n ON n.oid = c.relnamespace WHERE relkind = 'r' AND nspname NOT IN ('pg_catalog', 'information_schema')) a) a" }, { 'sqlquery' => 'SELECT current_database() AS datname, relname as table, autovacuum_count, vacuum_count, n_live_tup, n_dead_tup FROM pg_stat_user_tables', 'tagvalue' => 'table', - 'version' => 901, 'withdbname' => false }, { 'sqlquery' => 'SELECT current_database() AS datname, a.indexrelname as index, pg_relation_size(a.indexrelid) as size_bytes, idx_scan, idx_tup_read, idx_tup_fetch, idx_blks_read, idx_blks_hit from pg_stat_user_indexes a join pg_statio_user_indexes b on a.indexrelid = b.indexrelid;', 'tagvalue' => 'index', - 'version' => 901, 'withdbname' => false }, { 'sqlquery' => 'SELECT current_database() AS datname, relname as table, heap_blks_read, heap_blks_hit, idx_blks_read, idx_blks_hit, toast_blks_read, toast_blks_hit, tidx_blks_read, tidx_blks_hit FROM pg_statio_user_tables', 'tagvalue' => 'table', - 'version' => 901, 'withdbname' => false }, ] }] @@ -204,33 +199,29 @@ }, postgres_hosts: ['localhost.foo.com'], template_format: 'toml', + include_pe_metrics: false, } end it { # rubocop:disable Layout/LineLength options = [{ - 'address' => "postgres://telegraf@localhost.foo.com:5432/pe-puppetdb?#{params[:postgres_options].map { |k, v| "#{k}=#{v}" }.join('&').chomp}", - 'databases' => ['pe-puppetdb'], + 'address' => "postgres://telegraf@localhost.foo.com:5432/puppetdb?#{params[:postgres_options].map { |k, v| "#{k}=#{v}" }.join('&').chomp}", + 'databases' => ['puppetdb'], 'outputaddress' => 'localhost.foo.com', 'query' => [ { 'sqlquery' => 'SELECT * FROM pg_stat_database', - 'version' => 901, 'withdbname' => false }, { 'tagvalue' => 'table_name', - 'version' => 901, 'withdbname' => false, 'sqlquery' => "SELECT current_database() AS datname, total_bytes AS total , table_name , index_bytes AS index , toast_bytes AS toast , table_bytes AS table FROM ( SELECT *, total_bytes-index_bytes-coalesce(toast_bytes,0) AS table_bytes FROM ( SELECT c.oid,nspname AS table_schema, relname AS table_name , c.reltuples AS row_estimate , pg_total_relation_size(c.oid) AS total_bytes , pg_indexes_size(c.oid) AS index_bytes , pg_total_relation_size(reltoastrelid) AS toast_bytes FROM pg_class c LEFT JOIN pg_namespace n ON n.oid = c.relnamespace WHERE relkind = 'r' AND nspname NOT IN ('pg_catalog', 'information_schema')) a) a" }, { 'sqlquery' => 'SELECT current_database() AS datname, relname as table, autovacuum_count, vacuum_count, n_live_tup, n_dead_tup FROM pg_stat_user_tables', 'tagvalue' => 'table', - 'version' => 901, 'withdbname' => false }, { 'sqlquery' => 'SELECT current_database() AS datname, a.indexrelname as index, pg_relation_size(a.indexrelid) as size_bytes, idx_scan, idx_tup_read, idx_tup_fetch, idx_blks_read, idx_blks_hit from pg_stat_user_indexes a join pg_statio_user_indexes b on a.indexrelid = b.indexrelid;', 'tagvalue' => 'index', - 'version' => 901, 'withdbname' => false }, { 'sqlquery' => 'SELECT current_database() AS datname, relname as table, heap_blks_read, heap_blks_hit, idx_blks_read, idx_blks_hit, toast_blks_read, toast_blks_hit, tidx_blks_read, tidx_blks_hit FROM pg_statio_user_tables', 'tagvalue' => 'table', - 'version' => 901, 'withdbname' => false }, ] }]