feat: drop scrape_errors_total metric with a configurable interval (burningalchemist#551)

* feat: add scrape_error_drop_interval param to clear error metrics

* refactor: extract logic from main

* style: wording

Author: burningalchemist

cmd/sql_exporter/main.go

@@ -16,6 +16,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	info "github.com/prometheus/client_golang/prometheus/collectors/version"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/promlog"
 	"github.com/prometheus/common/version"
 	"github.com/prometheus/exporter-toolkit/web"
@@ -87,12 +88,17 @@ func main() {
 	}
 
 	klog.Warningf("Starting SQL exporter %s %s", version.Info(), version.BuildContext())
-
 	exporter, err := sql_exporter.NewExporter(*configFile)
 	if err != nil {
 		klog.Fatalf("Error creating exporter: %s", err)
 	}
 
+	// Start the scrape_errors_total metric drop ticker if configured.
+	startScrapeErrorsDropTicker(exporter, exporter.Config().Globals.ScrapeErrorDropInterval)
+
+	// Start signal handler to reload collector and target data.
+	signalHandler(exporter, *configFile)
+
 	// Setup and start webserver.
 	http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { http.Error(w, "OK", http.StatusOK) })
 	http.HandleFunc("/", HomeHandlerFunc(*metricsPath))
@@ -102,24 +108,9 @@ func main() {
 	http.Handle("/sql_exporter_metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{}))
 	// Expose refresh handler to reload collectors and targets
 	if *enableReload {
-		http.HandleFunc("/reload", reloadHandler(exporter))
+		http.HandleFunc("/reload", reloadHandler(exporter, *configFile))
 	}
 
-	// Handle SIGHUP for reloading the configuration
-	go func() {
-		c := make(chan os.Signal, 1)
-		signal.Notify(c, syscall.SIGHUP)
-		for {
-			<-c
-			err := sql_exporter.Reload(exporter, configFile)
-			if err != nil {
-				klog.Error(err)
-				continue
-			}
-		}
-	}()
-
-	// Start the web server
 	server := &http.Server{Addr: *listenAddress, ReadHeaderTimeout: httpReadHeaderTimeout}
 	if err := web.ListenAndServe(server, &web.FlagConfig{
 		WebListenAddresses: &([]string{*listenAddress}),
@@ -129,14 +120,43 @@ func main() {
 	}
 }
 
-// reloadHandler returns a handler that reloads collectors and targets
-func reloadHandler(e sql_exporter.Exporter) func(http.ResponseWriter, *http.Request) {
+// reloadHandler returns a handler that reloads collector and target data.
+func reloadHandler(e sql_exporter.Exporter, configFile string) func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
-		err := sql_exporter.Reload(e, configFile)
-		if err != nil {
+		if err := sql_exporter.Reload(e, &configFile); err != nil {
+			klog.Error(err)
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
 		w.WriteHeader(http.StatusOK)
 	}
 }
+
+// signalHandler listens for SIGHUP signals and reloads the collector and target data.
+func signalHandler(e sql_exporter.Exporter, configFile string) {
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP)
+	go func() {
+		for range c {
+			if err := sql_exporter.Reload(e, &configFile); err != nil {
+				klog.Error(err)
+			}
+		}
+	}()
+}
+
+// startScrapeErrorsDropTicker starts a ticker that periodically drops scrape error metrics.
+func startScrapeErrorsDropTicker(exporter sql_exporter.Exporter, interval model.Duration) {
+	if interval <= 0 {
+		return
+	}
+
+	ticker := time.NewTicker(time.Duration(interval))
+	klog.Warning("Started scrape_errors_total metrics drop ticker: ", interval)
+	go func() {
+		defer ticker.Stop()
+		for range ticker.C {
+			exporter.DropErrorMetrics()
+		}
+	}()
+}

config/global_config.go

@@ -9,12 +9,14 @@ import (
 
 // GlobalConfig contains globally applicable defaults.
 type GlobalConfig struct {
-	MinInterval     model.Duration `yaml:"min_interval" env:"MIN_INTERVAL"`                       // minimum interval between query executions, default is 0
-	ScrapeTimeout   model.Duration `yaml:"scrape_timeout" env:"SCRAPE_TIMEOUT"`                   // per-scrape timeout, global
-	TimeoutOffset   model.Duration `yaml:"scrape_timeout_offset" env:"SCRAPE_TIMEOUT_OFFSET"`     // offset to subtract from timeout in seconds
-	MaxConnLifetime time.Duration  `yaml:"max_connection_lifetime" env:"MAX_CONNECTION_LIFETIME"` // maximum amount of time a connection may be reused to any one target
-	MaxConns        int            `yaml:"max_connections" env:"MAX_CONNECTIONS"`                 // maximum number of open connections to any one target
-	MaxIdleConns    int            `yaml:"max_idle_connections" env:"MAX_IDLE_CONNECTIONS"`       // maximum number of idle connections to any one target
+	MinInterval             model.Duration `yaml:"min_interval" env:"MIN_INTERVAL"`                             // minimum interval between query executions, default is 0
+	ScrapeTimeout           model.Duration `yaml:"scrape_timeout" env:"SCRAPE_TIMEOUT"`                         // per-scrape timeout, global
+	TimeoutOffset           model.Duration `yaml:"scrape_timeout_offset" env:"SCRAPE_TIMEOUT_OFFSET"`           // offset to subtract from timeout in seconds
+	ScrapeErrorDropInterval model.Duration `yaml:"scrape_error_drop_interval" env:"SCRAPE_ERROR_DROP_INTERVAL"` // interval to drop scrape errors from the error counter, default is 0
+	MaxConnLifetime         time.Duration  `yaml:"max_connection_lifetime" env:"MAX_CONNECTION_LIFETIME"`       // maximum amount of time a connection may be reused to any one target
+
+	MaxConns     int `yaml:"max_connections" env:"MAX_CONNECTIONS"`           // maximum number of open connections to any one target
+	MaxIdleConns int `yaml:"max_idle_connections" env:"MAX_IDLE_CONNECTIONS"` // maximum number of idle connections to any one target
 
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]any `yaml:",inline" json:"-"`
@@ -26,6 +28,8 @@ func (g *GlobalConfig) UnmarshalYAML(unmarshal func(any) error) error {
 	g.MinInterval = model.Duration(0)
 	// Default to 10 seconds, since Prometheus has a 10 second scrape timeout default.
 	g.ScrapeTimeout = model.Duration(10 * time.Second)
+	// Default to 0 for scrape error drop interval.
+	g.ScrapeErrorDropInterval = model.Duration(0)
 	// Default to .5 seconds.
 	g.TimeoutOffset = model.Duration(500 * time.Millisecond)
 	g.MaxConns = 3

documentation/sql_exporter.yml

@@ -13,6 +13,8 @@ global:
   #
   # Must be strictly positive. The default is 500ms.
   scrape_timeout_offset: 500ms
+  # Interval between dropping scrape_errors_total metric: by default (0s) metrics are persistent.
+  scrape_error_drop_interval: 0s
   # Minimum interval between collector runs: by default (0s) collectors are executed on every scrape.
   min_interval: 0s
   # Maximum number of open connections to any one target. Metric queries will run concurrently on multiple connections,

exporter.go

@@ -33,6 +33,8 @@ type Exporter interface {
 	UpdateTarget([]Target)
 	// SetJobFilters sets the jobFilters field
 	SetJobFilters([]string)
+	// DropErrorMetrics resets the scrape_errors_total metric
+	DropErrorMetrics()
 }
 
 type exporter struct {
@@ -210,6 +212,12 @@ func (e *exporter) SetJobFilters(filters []string) {
 	e.jobFilters = filters
 }
 
+// DropErrorMetrics implements Exporter.
+func (e *exporter) DropErrorMetrics() {
+	scrapeErrorsMetric.Reset()
+	klog.Info("Dropped scrape_errors_total metric")
+}
+
 // registerScrapeErrorMetric registers the metrics for the exporter itself.
 func registerScrapeErrorMetric() *prometheus.CounterVec {
 	scrapeErrors := prometheus.NewCounterVec(prometheus.CounterOpts{

reload.go

@@ -47,7 +47,7 @@ func Reload(e Exporter, configFile *string) error {
 }
 
 func reloadTarget(e Exporter, nc, cc *cfg.Config) error {
-	klog.Warning("Recreating targets collectors...")
+	klog.Warning("Recreating target...")
 
 	// We want to preserve DSN from the previous config revision to avoid any connection changes
 	nc.Target.DSN = cc.Target.DSN
@@ -63,7 +63,7 @@ func reloadTarget(e Exporter, nc, cc *cfg.Config) error {
 
 	// Populate the target list
 	e.UpdateTarget([]Target{target})
-	klog.Warning("Collectors have been successfully reloaded for target")
+	klog.Warning("Collectors have been successfully updated for the target")
 	return nil
 }
 
@@ -98,6 +98,6 @@ func reloadJobs(e Exporter, nc, cc *cfg.Config) error {
 	}
 
 	e.UpdateTarget(targets)
-	klog.Warning("Query collectors have been successfully reloaded for jobs")
+	klog.Warning("Collectors have been successfully updated for the jobs")
 	return nil
 }