From be145ce1017ebadea8c92c7774c65f5737ff66d0 Mon Sep 17 00:00:00 2001 From: Frederik Mogensen Date: Wed, 1 Apr 2020 14:27:33 +0200 Subject: [PATCH 1/2] Add /metrics endpoint --- go.mod | 1 + go.sum | 7 +++++++ main.go | 3 +++ 3 files changed, 11 insertions(+) diff --git a/go.mod b/go.mod index 8c3b669..0ab1a7e 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/gobuffalo/envy v1.8.1 github.com/miekg/dns v1.1.13 github.com/mitchellh/mapstructure v1.1.2 + github.com/prometheus/client_golang v0.9.3 github.com/sirupsen/logrus v1.2.0 github.com/spf13/viper v1.4.0 github.com/stretchr/testify v1.4.0 diff --git a/go.sum b/go.sum index df4adcc..e09cc35 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,7 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -40,6 +41,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -66,6 +68,7 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.1.13 h1:x7DQtkU0cedzeS8TD36tT/w1Hm4rDtfCaYYAHE7TTBI= github.com/miekg/dns v1.1.13/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -80,12 +83,16 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= diff --git a/main.go b/main.go index c1ef1fb..0e3e1e0 100644 --- a/main.go +++ b/main.go @@ -11,6 +11,7 @@ import ( rice "github.com/GeertJohan/go.rice" "github.com/gobuffalo/envy" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" ) @@ -43,6 +44,8 @@ func main() { _, _ = w.Write(data) }) + http.Handle("/metrics", promhttp.Handler()) + go manager.ProbeLoop(time.Duration(int64(interval)) * time.Minute) log.Fatal(http.ListenAndServe(":"+strconv.Itoa(*port), nil)) } From 3764518b15e9a421a9532c42ac4d94423db04566 Mon Sep 17 00:00:00 2001 From: Frederik Mogensen Date: Mon, 21 Jun 2021 10:44:22 +0200 Subject: [PATCH 2/2] Batching the probes to spread the workload and more exact time readings --- .gitignore | 2 ++ Dockerfile | 2 +- Makefile | 39 ++++++++++++++++++++++++++++++++ manager.go | 10 +++++++++ probe/http.go | 61 +++++++++++++++++++++++++++++++++++++++++--------- probe/probe.go | 4 ++++ 6 files changed, 107 insertions(+), 11 deletions(-) create mode 100644 Makefile diff --git a/.gitignore b/.gitignore index d8c94ba..d263e9e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ /board /board.yaml +bin/* +tmp/* diff --git a/Dockerfile b/Dockerfile index 7eceb84..d5e7bb8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:latest AS build-env +FROM golang:1.16-alpine AS build-env # Dependencies WORKDIR /build diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..84f52cc --- /dev/null +++ b/Makefile @@ -0,0 +1,39 @@ +BINDIR ?= $(CURDIR)/bin +TMPDIR ?= $(CURDIR)/tmp +ARCH ?= amd64 + +help: ## display this help + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n\nTargets:\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf " \033[36m%-20s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST) + +.PHONY: help build image all clean dev + +test: ## test board + go test ./... + +dev: ## live reload development + gin --path . --appPort 8080 --all --immediate --bin tmp/board run + +build: ## build board + mkdir -p $(BINDIR) + CGO_ENABLED=0 go build -o ./bin/board + +verify: test build ## tests and builds board + +image: ## build docker image + docker build -t lesterpig/board:latest . + +clean: ## clean up created files + rm -rf \ + $(BINDIR) \ + $(TMPDIR) + +all: clean test build image ## runs test, build and image + +test-coverage: ## Generate test coverage report + mkdir -p $(TMPDIR) + go test ./... --coverprofile $(TMPDIR)/outfile + go tool cover -html=$(TMPDIR)/outfile + +lint: ## Generate static analysis report + goreportcard-cli -v + golint ./... diff --git a/manager.go b/manager.go index 819f2d6..16926b4 100644 --- a/manager.go +++ b/manager.go @@ -1,6 +1,7 @@ package main import ( + "math" "time" "github.com/Lesterpig/board/probe" @@ -38,8 +39,17 @@ func (manager *Manager) ProbeAll() { manager.LastUpdate = time.Now() + i := 0.0 + for category, services := range manager.Services { for _, service := range services { + // Batching the probes to spread the workload + if math.Mod(i, 8) == 0 { + log.Debug("Waiting") + time.Sleep(2 * time.Second) + i = 0 + } + i++ go func(category string, service *Service) { prevStatus := service.Status service.Status, service.Message = service.Prober.Probe() diff --git a/probe/http.go b/probe/http.go index 24a5258..8b14802 100644 --- a/probe/http.go +++ b/probe/http.go @@ -2,8 +2,10 @@ package probe import ( "crypto/tls" + "fmt" "io/ioutil" "net/http" + "net/http/httptrace" "regexp" "strconv" "time" @@ -38,9 +40,11 @@ func (h *HTTP) Init(c Config) error { } /* #nosec G402 */ - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: !opts.VerifyCertificate}, - } + tr := http.DefaultTransport.(*http.Transport).Clone() + tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + // Disable keep alive to get more consistent measurements + tr.DisableKeepAlives = true + tr.ResponseHeaderTimeout = c.Fatal h.client = &http.Client{ Timeout: c.Fatal, @@ -55,17 +59,39 @@ func (h *HTTP) Init(c Config) error { // If the operation succeeds, the message will be the duration of the HTTP request in ms. // Otherwise, an error message is returned. func (h *HTTP) Probe() (status Status, message string) { - start := time.Now() - res, err := h.client.Get(h.Target) - duration := time.Since(start) + + req, _ := http.NewRequest("GET", h.Target, nil) + + var server, start time.Time + var initDuration, serverDuration, totalDuration time.Duration + + trace := &httptrace.ClientTrace{ + + TLSHandshakeDone: func(cs tls.ConnectionState, err error) { + server = time.Now() + initDuration = time.Since(start) + }, + } + + req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace)) + // Starting timer + start = time.Now() + // Set server time before request. This may be reset in TLSHandshakeDone if a TLSHandshake is preformed + server = start + + res, err := h.client.Transport.RoundTrip(req) + totalDuration = time.Since(start) + serverDuration = time.Since(server) + + if res != nil { + defer res.Body.Close() // MUST CLOSED THIS + } if err != nil { return StatusError, defaultConnectErrorMsg } - defer func() { _ = res.Body.Close() }() - - if res.StatusCode != 200 { + if !(res.StatusCode >= 200 && res.StatusCode <= 399) { return StatusError, strconv.Itoa(res.StatusCode) } @@ -74,5 +100,20 @@ func (h *HTTP) Probe() (status Status, message string) { return StatusError, "Unexpected result" } - return EvaluateDuration(duration, h.Warning) + return AdvancedEvaluateDuration(initDuration, serverDuration, totalDuration, h.Warning) +} + +// AdvancedEvaluateDuration is a shortcut for warning duration checks. +// It returns a message containing the duration, and a OK or a WARNING status +// depending on the provided warning duration. +func AdvancedEvaluateDuration(initDuration, serverDuration, duration time.Duration, warning time.Duration) (status Status, message string) { + if duration >= warning { + status = StatusWarning + } else { + status = StatusOK + } + + message = fmt.Sprintf("%d ms / %d ms", initDuration.Nanoseconds()/1000000, serverDuration.Nanoseconds()/1000000) + + return } diff --git a/probe/probe.go b/probe/probe.go index 98cfe7b..8b53c09 100644 --- a/probe/probe.go +++ b/probe/probe.go @@ -4,8 +4,12 @@ package probe import ( "fmt" "time" + + "github.com/sirupsen/logrus" ) +var log = logrus.StandardLogger() + // Config holds probe configuration, submitted through Init methods. type Config struct { Target string