feat: use env replace config file params

Author: zhongming Fan

cmd/sql_exporter/main.go

@@ -37,7 +37,8 @@ var (
 	metricsPath   = flag.String("web.metrics-path", "/metrics", "Path under which to expose metrics")
 	enableReload  = flag.Bool("web.enable-reload", false, "Enable reload collector data handler")
 	webConfigFile = flag.String("web.config.file", "", "[EXPERIMENTAL] TLS/BasicAuth configuration file path")
-	configFile    = flag.String("config.file", "sql_exporter.yml", "SQL Exporter configuration file path")
+	configFile    = flag.String("config.file", "", "SQL Exporter configuration file path")
+	collectorFile = flag.String("collector.file", "", "SQL Exporter collector file path")
 	logFormatJSON = flag.Bool("log.json", false, "[DEPRECATED] Set log output format to JSON")
 	logFormat     = flag.String("log.format", "logfmt", "Set log output format")
 	logLevel      = flag.String("log.level", "info", "Set log level")
@@ -77,7 +78,7 @@ func main() {
 	}
 
 	klog.Warningf("Starting SQL exporter %s %s", version.Info(), version.BuildContext())
-	exporter, err := sql_exporter.NewExporter(*configFile)
+	exporter, err := sql_exporter.NewExporter(*configFile, *collectorFile)
 	if err != nil {
 		klog.Fatalf("Error creating exporter: %s", err)
 	}
@@ -86,7 +87,7 @@ func main() {
 	startScrapeErrorsDropTicker(exporter, exporter.Config().Globals.ScrapeErrorDropInterval)
 
 	// Start signal handler to reload collector and target data.
-	signalHandler(exporter, *configFile)
+	signalHandler(exporter, *configFile, *collectorFile)
 
 	// Setup and start webserver.
 	http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { http.Error(w, "OK", http.StatusOK) })
@@ -97,7 +98,7 @@ func main() {
 	http.Handle("/sql_exporter_metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{}))
 	// Expose refresh handler to reload collectors and targets
 	if *enableReload {
-		http.HandleFunc("/reload", reloadHandler(exporter, *configFile))
+		http.HandleFunc("/reload", reloadHandler(exporter, *configFile, *collectorFile))
 	}
 
 	server := &http.Server{Addr: *listenAddress, ReadHeaderTimeout: httpReadHeaderTimeout}
@@ -110,9 +111,9 @@ func main() {
 }
 
 // reloadHandler returns a handler that reloads collector and target data.
-func reloadHandler(e sql_exporter.Exporter, configFile string) func(http.ResponseWriter, *http.Request) {
+func reloadHandler(e sql_exporter.Exporter, configFile string, collectorFile string) func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
-		if err := sql_exporter.Reload(e, &configFile); err != nil {
+		if err := sql_exporter.Reload(e, &configFile, &collectorFile); err != nil {
 			klog.Error(err)
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
@@ -122,12 +123,12 @@ func reloadHandler(e sql_exporter.Exporter, configFile string) func(http.Respons
 }
 
 // signalHandler listens for SIGHUP signals and reloads the collector and target data.
-func signalHandler(e sql_exporter.Exporter, configFile string) {
+func signalHandler(e sql_exporter.Exporter, configFile string, collectorFile string) {
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, syscall.SIGHUP)
 	go func() {
 		for range c {
-			if err := sql_exporter.Reload(e, &configFile); err != nil {
+			if err := sql_exporter.Reload(e, &configFile, &collectorFile); err != nil {
 				klog.Error(err)
 			}
 		}

config/config.go

@@ -2,9 +2,12 @@ package config
 
 import (
 	"context"
+	"embed"
 	"fmt"
+	"github.com/prometheus/common/model"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/sethvargo/go-envconfig"
 	"gopkg.in/yaml.v3"
@@ -19,27 +22,43 @@ const MaxInt32 int = 1<<31 - 1
 const (
 	EnvPrefix string = "SQLEXPORTER_"
 
-	EnvConfigFile string = EnvPrefix + "CONFIG"
-	EnvDebug      string = EnvPrefix + "DEBUG"
+	EnvConfigFile              string = EnvPrefix + "CONFIG"
+	EnvDebug                   string = EnvPrefix + "DEBUG"
+	EmbedSqlExporterConfigFile string = "sql_exporter_embed.yml"
 )
 
 var (
 	EnablePing           bool
 	IgnoreMissingVals    bool
 	DsnOverride          string
 	TargetLabel          string
-	kingbaseDatabaseMode = os.Getenv("KINGBASE_DATABASE_MODE")
+	kingbaseDatabaseMode      = os.Getenv("KINGBASE_DATABASE_MODE")
+	useEmbedConfig       bool = true // Use embedded config by default
+	//go:embed sql_exporter_embed.yml
+	sqlExporterConfig embed.FS
 )
 
 // Load attempts to parse the given config file and return a Config object.
-func Load(configFile string) (*Config, error) {
-	klog.Infof("Loading configuration from %s", configFile)
-	buf, err := os.ReadFile(configFile)
+func Load(configFile string, collectorFile string) (*Config, error) {
+	source := configFile
+	var readFn func(string) ([]byte, error)
+	if source == "" {
+		useEmbedConfig = true
+		source = EmbedSqlExporterConfigFile
+		readFn = sqlExporterConfig.ReadFile
+		klog.Infof("Loading configuration from embed file: %s", source)
+	} else {
+		useEmbedConfig = false
+		readFn = os.ReadFile
+		klog.Infof("Loading configuration from: %s", source)
+	}
+
+	buf, err := readFn(source)
 	if err != nil {
 		return nil, err
 	}
 
-	c := Config{configFile: configFile}
+	c := Config{configFile: configFile, collectorFile: collectorFile}
 	err = yaml.Unmarshal(buf, &c)
 	if err != nil {
 		return nil, err
@@ -64,8 +83,9 @@ type Config struct {
 	Jobs           []*JobConfig       `yaml:"jobs,omitempty"`
 	Collectors     []*CollectorConfig `yaml:"collectors,omitempty"`
 
-	configFile string
-
+	configFile     string
+	collectorFile  string
+	useEmbedConfig bool
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]any `yaml:",inline" json:"-"`
 }
@@ -76,6 +96,12 @@ func (c *Config) UnmarshalYAML(unmarshal func(any) error) error {
 	if err := c.unmarshalConfig(unmarshal); err != nil {
 		return err
 	}
+
+	// Apply environment overrides.
+	if useEmbedConfig {
+		c.applyEnvOverrides(c.collectorFile)
+	}
+
 	// Populate global defaults.
 	if err := c.populateGlobalDefaults(); err != nil {
 		return err
@@ -177,7 +203,12 @@ func (c *Config) YAML() ([]byte, error) {
 
 // loadCollectorFiles resolves all collector file globs to files and loads the collectors they define.
 func (c *Config) loadCollectorFiles() error {
-	baseDir := filepath.Dir(c.configFile)
+	var baseDir string
+	if useEmbedConfig {
+		baseDir = "."
+	} else {
+		baseDir = filepath.Dir(c.configFile)
+	}
 
 	// pg模式下写死采集sql文件
 	if kingbaseDatabaseMode == "pg" {
@@ -219,3 +250,18 @@ func (c *Config) loadCollectorFiles() error {
 
 	return nil
 }
+
+func (c *Config) applyEnvOverrides(collectorFile string) {
+	// sql采集指标文件
+	c.CollectorFiles = []string{collectorFile}
+	// sql采集名称
+	c.Target.CollectorRefs = []string{os.Getenv("COLLECTOR_REFS")}
+
+	// 应用配置
+	SetDurationFromEnv("SCRAPE_TIMEOUT_OFFSET", "500ms", func(d model.Duration) { c.Globals.TimeoutOffset = d })
+	SetDurationFromEnv("MIN_INTERVAL", "0s", func(d model.Duration) { c.Globals.MinInterval = d })
+	SetIntFromEnv("MAX_CONNECTIONS", "3", func(i int) { c.Globals.MaxConns = i })
+	SetIntFromEnv("MAX_IDLE_CONNECTIONS", "3", func(i int) { c.Globals.MaxIdleConns = i })
+	SetTimeDurationFromEnv("MAX_CONNECTION_LIFETIME", "5m", func(d time.Duration) { c.Globals.MaxConnLifetime = d })
+	SetDurationFromEnv("SCRAPE_TIMEOUT", "10s", func(d model.Duration) { c.Globals.ScrapeTimeout = d })
+}

config/sql_exporter_embed.yml

@@ -0,0 +1,30 @@
+# Global defaults.
+global:
+  # Subtracted from Prometheus' scrape_timeout to give us some headroom and prevent Prometheus from timing out first.
+  scrape_timeout_offset: 500ms
+  # Minimum interval between collector runs: by default (0s) collectors are executed on every scrape.
+  min_interval: 0s
+  # Maximum number of open connections to any one target. Metric queries will run concurrently on multiple connections,
+  # as will concurrent scrapes.
+  max_connections: 3
+  # Maximum number of idle connections to any one target. Unless you use very long collection intervals, this should
+  # always be the same as max_connections.
+  max_idle_connections: 3
+  # Maximum number of maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse.
+  # If 0, connections are not closed due to a connection's age.
+  max_connection_lifetime: 5m
+
+# The target to monitor and the collectors to execute on it.
+target:
+  # Data source name always has a URI schema that matches the driver name. In some cases (e.g. MySQL)
+  # the schema gets dropped or replaced to match the driver expected DSN format.
+  data_source_name: 'dm://sysdba:sysdba001@127.0.0.1:5236'
+
+  # Collectors (referenced by name) to execute on the target.
+  # Glob patterns are supported (see <https://pkg.go.dev/path/filepath#Match> for syntax).
+  collectors: [demo]
+
+# Collector files specifies a list of globs. One collector definition is read from each matching file.
+# Glob patterns are supported (see <https://pkg.go.dev/path/filepath#Match> for syntax).
+collector_files: 
+  - "demo.collector.yml"

config/util.go

@@ -2,8 +2,12 @@ package config
 
 import (
 	"fmt"
+	"github.com/prometheus/common/model"
+	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
+	"time"
 
 	"k8s.io/klog/v2"
 )
@@ -69,3 +73,41 @@ func checkOverflow(m map[string]any, ctx string) error {
 	}
 	return nil
 }
+
+// GetEnvWithDefault 从环境变量获取值，如果为空则使用默认值
+func GetEnvWithDefault(key, defaultVal string) string {
+	if val := os.Getenv(key); val != "" {
+		return val
+	}
+	return defaultVal
+}
+
+// SetDurationFromEnv 从环境变量设置 model.Duration 类型值
+func SetDurationFromEnv(key, defaultVal string, setter func(model.Duration)) {
+	val := GetEnvWithDefault(key, defaultVal)
+	if parsed, err := model.ParseDuration(val); err == nil {
+		setter(parsed)
+	} else {
+		klog.Warningf("Invalid %s: %v", key, err)
+	}
+}
+
+// SetIntFromEnv 从环境变量设置整数类型值
+func SetIntFromEnv(key, defaultVal string, setter func(int)) {
+	val := GetEnvWithDefault(key, defaultVal)
+	if parsed, err := strconv.Atoi(val); err == nil {
+		setter(parsed)
+	} else {
+		klog.Warningf("Invalid %s: %v", key, err)
+	}
+}
+
+// SetTimeDurationFromEnv 从环境变量设置 time.Duration 类型值
+func SetTimeDurationFromEnv(key, defaultVal string, setter func(time.Duration)) {
+	val := GetEnvWithDefault(key, defaultVal)
+	if parsed, err := time.ParseDuration(val); err == nil {
+		setter(parsed)
+	} else {
+		klog.Warningf("Invalid %s: %v", key, err)
+	}
+}

exporter.go

@@ -73,8 +73,8 @@ type exporter struct {
 }
 
 // NewExporter returns a new Exporter with the provided config.
-func NewExporter(configFile string) (Exporter, error) {
-	c, err := config.Load(configFile)
+func NewExporter(configFile string, collectorFile string) (Exporter, error) {
+	c, err := config.Load(configFile, collectorFile)
 	if err != nil {
 		return nil, err
 	}

reload.go

@@ -8,10 +8,10 @@ import (
 )
 
 // Reload function is used to reload the exporter configuration without restarting the exporter
-func Reload(e Exporter, configFile *string) error {
+func Reload(e Exporter, configFile *string, collectorFile *string) error {
 	klog.Warning("Reloading collectors has started...")
 	klog.Warning("Connections will not be changed upon the restart of the exporter")
-	configNext, err := cfg.Load(*configFile)
+	configNext, err := cfg.Load(*configFile, *collectorFile)
 	if err != nil {
 		klog.Errorf("Error reading config file - %v", err)
 		return err