Skip to content

Commit d8e4c6f

Browse files
burningalchemistburningalchemist
authored
refactor: Reload targets and collectors, support SIGHUP (burningalchemist#548)
* refactor: Reload targets and collectors, support SIGHUP * refactor(reload): function names * refactor: set enable-ping default value if it's nil * docs: update function docs
1 parent 56513fa commit d8e4c6f

File tree

4 files changed

+133
-80
lines changed

4 files changed

+133
-80
lines changed

‎cmd/sql_exporter/main.go‎

Lines changed: 27 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ import (
55
"fmt"
66
"net/http"
77
"os"
8+
"os/signal"
89
"runtime"
10+
"syscall"
911
"time"
1012

1113
"github.com/burningalchemist/sql_exporter"
@@ -98,94 +100,43 @@ func main() {
98100
http.Handle(*metricsPath, promhttp.InstrumentMetricHandler(prometheus.DefaultRegisterer, ExporterHandlerFor(exporter)))
99101
// Expose exporter metrics separately, for debugging purposes.
100102
http.Handle("/sql_exporter_metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{}))
101-
102-
// Expose refresh handler to reload query collections
103+
// Expose refresh handler to reload collectors and targets
103104
if *enableReload {
104-
http.HandleFunc("/reload", reloadCollectors(exporter))
105+
http.HandleFunc("/reload", reloadHandler(exporter))
105106
}
106107

108+
// Handle SIGHUP for reloading the configuration
109+
go func() {
110+
c := make(chan os.Signal, 1)
111+
signal.Notify(c, syscall.SIGHUP)
112+
for {
113+
<-c
114+
err := sql_exporter.Reload(exporter, configFile)
115+
if err != nil {
116+
klog.Error(err)
117+
continue
118+
}
119+
}
120+
}()
121+
122+
// Start the web server
107123
server := &http.Server{Addr: *listenAddress, ReadHeaderTimeout: httpReadHeaderTimeout}
108-
if err := web.ListenAndServe(server, &web.FlagConfig{WebListenAddresses: &([]string{*listenAddress}),
109-
WebConfigFile: webConfigFile, WebSystemdSocket: OfBool(false)}, logger); err != nil {
124+
if err := web.ListenAndServe(server, &web.FlagConfig{
125+
WebListenAddresses: &([]string{*listenAddress}),
126+
WebConfigFile: webConfigFile, WebSystemdSocket: OfBool(false),
127+
}, logger); err != nil {
110128
klog.Fatal(err)
111129
}
112130
}
113131

114-
func reloadCollectors(e sql_exporter.Exporter) func(http.ResponseWriter, *http.Request) {
132+
// reloadHandler returns a handler that reloads collectors and targets
133+
func reloadHandler(e sql_exporter.Exporter) func(http.ResponseWriter, *http.Request) {
115134
return func(w http.ResponseWriter, r *http.Request) {
116-
klog.Warning("Reloading collectors has started...")
117-
klog.Warning("Connections will not be changed upon the restart of the exporter")
118-
exporterNewConfig, err := cfg.Load(*configFile)
135+
err := sql_exporter.Reload(e, configFile)
119136
if err != nil {
120-
klog.Errorf("Error reading config file - %v", err)
121137
http.Error(w, err.Error(), http.StatusInternalServerError)
122138
return
123139
}
124-
currentConfig := e.Config()
125-
klog.Infof("Total collector size change: %v -> %v", len(currentConfig.Collectors),
126-
len(exporterNewConfig.Collectors))
127-
128-
if len(currentConfig.Collectors) > 0 {
129-
currentConfig.Collectors = currentConfig.Collectors[:0]
130-
}
131-
currentConfig.Collectors = exporterNewConfig.Collectors
132-
133-
// Reload Collectors for a single target if there is one
134-
if currentConfig.Target != nil {
135-
klog.Warning("Reloading target collectors...")
136-
// FIXME: Should be t.Collectors() instead of config.Collectors
137-
target, err := sql_exporter.NewTarget("", currentConfig.Target.Name, "", string(currentConfig.Target.DSN),
138-
exporterNewConfig.Target.Collectors(), nil, currentConfig.Globals, currentConfig.Target.EnablePing)
139-
if err != nil {
140-
klog.Errorf("Error recreating a target - %v", err)
141-
http.Error(w, err.Error(), http.StatusInternalServerError)
142-
return
143-
}
144-
e.UpdateTarget([]sql_exporter.Target{target})
145-
klog.Warning("Collectors have been successfully reloaded for target")
146-
w.WriteHeader(http.StatusOK)
147-
return
148-
}
149-
150-
// Reload Collectors for Jobs if there are any
151-
if len(currentConfig.Jobs) > 0 {
152-
klog.Warning("Recreating jobs...")
153-
154-
// We want to preserve `static_configs`` from the previous config revision to avoid any connection changes
155-
for _, currentJob := range currentConfig.Jobs {
156-
for _, newJob := range exporterNewConfig.Jobs {
157-
if newJob.Name == currentJob.Name {
158-
newJob.StaticConfigs = currentJob.StaticConfigs
159-
}
160-
}
161-
}
162-
currentConfig.Jobs = exporterNewConfig.Jobs
163-
164-
var updateErr error
165-
targets := make([]sql_exporter.Target, 0, len(currentConfig.Jobs))
166-
167-
for _, jobConfigItem := range currentConfig.Jobs {
168-
job, err := sql_exporter.NewJob(jobConfigItem, currentConfig.Globals)
169-
if err != nil {
170-
updateErr = err
171-
break
172-
}
173-
targets = append(targets, job.Targets()...)
174-
klog.Infof("Recreated Job: %s", jobConfigItem.Name)
175-
}
176-
177-
if updateErr != nil {
178-
klog.Errorf("Error recreating jobs - %v", err)
179-
http.Error(w, err.Error(), http.StatusInternalServerError)
180-
return
181-
}
182-
183-
e.UpdateTarget(targets)
184-
klog.Warning("Query collectors have been successfully reloaded for jobs")
185-
w.WriteHeader(http.StatusOK)
186-
return
187-
}
188-
klog.Warning("No target or jobs have been found - nothing to reload")
189-
http.Error(w, "", http.StatusInternalServerError)
140+
w.WriteHeader(http.StatusOK)
190141
}
191142
}

‎exporter.go‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,9 +60,6 @@ func NewExporter(configFile string) (Exporter, error) {
6060

6161
var targets []Target
6262
if c.Target != nil {
63-
if c.Target.EnablePing == nil {
64-
c.Target.EnablePing = &config.EnablePing
65-
}
6663
target, err := NewTarget("", c.Target.Name, "", string(c.Target.DSN), c.Target.Collectors(), nil, c.Globals, c.Target.EnablePing)
6764
if err != nil {
6865
return nil, err

‎reload.go‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
package sql_exporter
2+
3+
import (
4+
"errors"
5+
6+
cfg "github.com/burningalchemist/sql_exporter/config"
7+
"k8s.io/klog/v2"
8+
)
9+
10+
// Reload function is used to reload the exporter configuration without restarting the exporter
11+
func Reload(e Exporter, configFile *string) error {
12+
klog.Warning("Reloading collectors has started...")
13+
klog.Warning("Connections will not be changed upon the restart of the exporter")
14+
configNext, err := cfg.Load(*configFile)
15+
if err != nil {
16+
klog.Errorf("Error reading config file - %v", err)
17+
return err
18+
}
19+
20+
configCurrent := e.Config()
21+
22+
// Clear current collectors and replace with new ones
23+
if len(configCurrent.Collectors) > 0 {
24+
configCurrent.Collectors = configCurrent.Collectors[:0]
25+
}
26+
configCurrent.Collectors = configNext.Collectors
27+
klog.Infof("Total collector size change: %v -> %v", len(configCurrent.Collectors),
28+
len(configNext.Collectors))
29+
30+
// Reload targets
31+
switch {
32+
case configCurrent.Target != nil && configNext.Target != nil:
33+
if err = reloadTarget(e, configNext, configCurrent); err != nil {
34+
return err
35+
}
36+
case len(configCurrent.Jobs) > 0 && len(configNext.Jobs) > 0:
37+
if err = reloadJobs(e, configNext, configCurrent); err != nil {
38+
return err
39+
}
40+
case configCurrent.Target != nil && len(configNext.Jobs) > 0:
41+
case len(configCurrent.Jobs) > 0 && configNext.Target != nil:
42+
return errors.New("changing scrape mode is not allowed. Please restart the exporter")
43+
default:
44+
klog.Warning("No target or jobs have been found - nothing to reload")
45+
}
46+
return nil
47+
}
48+
49+
func reloadTarget(e Exporter, nc, cc *cfg.Config) error {
50+
klog.Warning("Recreating targets collectors...")
51+
52+
// We want to preserve DSN from the previous config revision to avoid any connection changes
53+
nc.Target.DSN = cc.Target.DSN
54+
// Apply the new target configuration
55+
cc.Target = nc.Target
56+
// Recreate the target object
57+
target, err := NewTarget("", cc.Target.Name, "", string(cc.Target.DSN),
58+
cc.Target.Collectors(), nil, cc.Globals, cc.Target.EnablePing)
59+
if err != nil {
60+
klog.Errorf("Error recreating a target - %v", err)
61+
return err
62+
}
63+
64+
// Populate the target list
65+
e.UpdateTarget([]Target{target})
66+
klog.Warning("Collectors have been successfully reloaded for target")
67+
return nil
68+
}
69+
70+
func reloadJobs(e Exporter, nc, cc *cfg.Config) error {
71+
klog.Warning("Recreating jobs...")
72+
73+
// We want to preserve `static_configs`` from the previous config revision to avoid any connection changes
74+
for _, currentJob := range cc.Jobs {
75+
for _, newJob := range nc.Jobs {
76+
if newJob.Name == currentJob.Name {
77+
newJob.StaticConfigs = currentJob.StaticConfigs
78+
}
79+
}
80+
}
81+
cc.Jobs = nc.Jobs
82+
var updateErr error
83+
targets := make([]Target, 0, len(cc.Jobs))
84+
85+
for _, jobConfigItem := range cc.Jobs {
86+
job, err := NewJob(jobConfigItem, cc.Globals)
87+
if err != nil {
88+
updateErr = err
89+
break
90+
}
91+
targets = append(targets, job.Targets()...)
92+
klog.Infof("Recreated Job: %s", jobConfigItem.Name)
93+
}
94+
95+
if updateErr != nil {
96+
klog.Errorf("Error recreating jobs - %v", updateErr)
97+
return updateErr
98+
}
99+
100+
e.UpdateTarget(targets)
101+
klog.Warning("Query collectors have been successfully reloaded for jobs")
102+
return nil
103+
}

‎target.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,14 +57,16 @@ func NewTarget(
5757
logContext, tname, jg, dsn string, ccs []*config.CollectorConfig, constLabels prometheus.Labels, gc *config.GlobalConfig, ep *bool) (
5858
Target, errors.WithContext,
5959
) {
60-
6160
if tname != "" {
6261
logContext = TrimMissingCtx(fmt.Sprintf(`%s,target=%s`, logContext, tname))
6362
if constLabels == nil {
6463
constLabels = prometheus.Labels{config.TargetLabel: tname}
6564
}
6665
}
6766

67+
if ep == nil {
68+
ep = &config.EnablePing
69+
}
6870
klog.Infof("[%s] Target ping enabled: %v", logContext, *ep)
6971

7072
// Sort const labels by name to ensure consistent ordering.

0 commit comments

Comments
 (0)