From 5995530f08ae706e6f993d5c170c99ba171d72b8 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Thu, 26 Jun 2025 15:26:13 +0300 Subject: [PATCH 001/126] init commit, wrote first version metrics --- internal/MemStorage/storage.go | 106 +++++++++++++++++++++++++++++++++ internal/metrics/counter.go | 24 ++++++++ internal/metrics/gauge.go | 23 +++++++ internal/metrics/metrics.go | 22 +++++++ 4 files changed, 175 insertions(+) create mode 100644 internal/MemStorage/storage.go create mode 100644 internal/metrics/counter.go create mode 100644 internal/metrics/gauge.go create mode 100644 internal/metrics/metrics.go diff --git a/internal/MemStorage/storage.go b/internal/MemStorage/storage.go new file mode 100644 index 0000000..8c565d2 --- /dev/null +++ b/internal/MemStorage/storage.go @@ -0,0 +1,106 @@ +package MemStorage + +import ( + "sync" + + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" +) + +type Collector interface { + GetMetric(mtr metrics.Metric) (interface{}, error) + UpdateMetric(mtr metrics.Metric) error +} + +type MemStorage struct { + mutex sync.RWMutex + gauges map[string]float64 + counters map[string]int64 + //metrics map[string]metrics.Metric +} + +func NewMemStorage() *MemStorage { + return &MemStorage{ + gauges: make(map[string]float64), + counters: make(map[string]int64), + } +} + +func (ms *MemStorage) UpdateMetric(metric metrics.Metric) error { + switch metric.Type() { + case metrics.CounterType: + { + val, ok := metric.Value().(int64) + if !ok { + return metrics.ErrInvalidValueType + } + + ms.UpdateCounter(metric, val) + return nil + } + case metrics.GaugeType: + { + val, ok := metric.Value().(float64) + if !ok { + return metrics.ErrInvalidValueType + } + + ms.UpdateGauge(metric, val) + return nil + } + default: + return metrics.ErrInvalidMetricsType + } +} + +func (ms *MemStorage) UpdateGauge(metric metrics.Metric, value float64) { + ms.mutex.Lock() + defer ms.mutex.Unlock() + ms.gauges[metric.Name()] = value +} + +func (ms *MemStorage) UpdateCounter(metric metrics.Metric, value int64) { + ms.mutex.Lock() + defer ms.mutex.Unlock() + ms.counters[metric.Name()] += value +} + +func (ms *MemStorage) GetMetric(metric metrics.Metric) (interface{}, error) { + switch metric.Type() { + case metrics.CounterType: + { + val, ok := ms.GetCounter(metric.Name()) + if !ok { + return nil, metrics.ErrMetricsNotFound + } + + return val, nil + } + case metrics.GaugeType: + { + val, ok := ms.GetGauges(metric.Name()) + if !ok { + return nil, metrics.ErrMetricsNotFound + } + + return val, nil + } + default: + return nil, metrics.ErrInvalidMetricsType + } +} + +func (ms *MemStorage) GetGauges(name string) (float64, bool) { + ms.mutex.RLock() + defer ms.mutex.RUnlock() + + val, ok := ms.gauges[name] + return val, ok +} + +func (ms *MemStorage) GetCounter(name string) (int64, bool) { + ms.mutex.RLock() + defer ms.mutex.RUnlock() + + val, ok := ms.counters[name] + return val, ok +} diff --git a/internal/metrics/counter.go b/internal/metrics/counter.go new file mode 100644 index 0000000..7f8f6cc --- /dev/null +++ b/internal/metrics/counter.go @@ -0,0 +1,24 @@ +package metrics + +type counter struct { + name string + value int64 +} + +func NewCounter(name string) Metric { + return &counter{ + name: name, + } +} + +func (c *counter) Name() string { + return c.name +} + +func (c *counter) Type() string { + return CounterType +} + +func (c *counter) Value() interface{} { + return c.value +} diff --git a/internal/metrics/gauge.go b/internal/metrics/gauge.go new file mode 100644 index 0000000..812dd8f --- /dev/null +++ b/internal/metrics/gauge.go @@ -0,0 +1,23 @@ +package metrics + +type gauge struct { + name string + value float64 +} +func NewGauge(name string) Metric { + return &gauge{ + name: name, + } +} + +func (g *gauge) Name() string { + return g.name +} + +func (g *gauge) Type() string { + return GaugeType +} + +func (g *gauge) Value() interface{} { + return g.value +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..eae8bf8 --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,22 @@ +package metrics + +import ( + "errors" +) + +type Metric interface { + Value() interface{} + Name() string + Type() string +} + +const ( + CounterType = "counter" + GaugeType = "gauge" +) + +var ( + ErrInvalidMetricsType = errors.New("invalid metrics type") + ErrInvalidValueType = errors.New("invalid value type") + ErrMetricsNotFound = errors.New("unknown this metric") +) From 4fcd69847593318ad6670a1687de7a99f410321f Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Thu, 26 Jun 2025 19:21:16 +0300 Subject: [PATCH 002/126] first version --- cmd/server/main.go | 104 +++++++++++++++++++++++++++++++++++- internal/metrics/counter.go | 14 ++++- internal/metrics/gauge.go | 15 +++++- internal/metrics/metrics.go | 1 + 4 files changed, 129 insertions(+), 5 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index 38dd16d..26d0998 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -1,3 +1,105 @@ package main -func main() {} +import ( + "net/http" + "strconv" + "strings" + + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/MemStorage" + mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" +) + +func HandleCounter(res http.ResponseWriter, name, value string) mtr.Metric { + val, err := strconv.ParseInt(value, 10, 64) + if err != nil { + http.Error(res, "invalid value metric", http.StatusBadRequest) + return nil + } + + counter := mtr.NewCounter(name, val) + // if err := counter.SetValue(val); err != nil { + // http.Error(res, err.Error(), http.StatusBadRequest) + // return nil + // } + + res.WriteHeader(http.StatusOK) + return counter +} + +func HandleGauge(res http.ResponseWriter, name, value string) mtr.Metric { + val, err := strconv.ParseFloat(value, 64) + if err != nil { + http.Error(res, "invalid value metric", http.StatusBadRequest) + return nil + } + + gauge := mtr.NewGauge(name, val) + // if err := gauge.SetValue(val); err != nil { + // http.Error(res, err.Error(), http.StatusBadRequest) + // return nil + // } + + res.WriteHeader(http.StatusOK) + return gauge +} + +func HandleUnknownMetric(res http.ResponseWriter) { + http.Error(res, "unknown type metric!", http.StatusBadRequest) +} + +func mainHandle(storage ms.Collector) http.HandlerFunc { + return func(res http.ResponseWriter, req *http.Request) { + + if req.Method != http.MethodPost { + http.Error(res, "use only POST request", http.StatusMethodNotAllowed) + return + } + + if req.Header.Get("Content-Type") != "text/plain" { + http.Error(res, "use only text/plain", http.StatusBadRequest) + return + } + + parts := strings.Split(req.URL.Path, "/") + + if len(parts) != 4 && parts[0] != "update" { + http.Error(res, "invalid request", http.StatusBadRequest) + return + } + + metricType, name, value := parts[1], parts[2], parts[3] + + if name == "" { + http.Error(res, "metric name is missing", http.StatusFound) + return + } + + var metric mtr.Metric + switch metricType { + case mtr.GaugeType: + metric = HandleGauge(res, name, value) + case mtr.CounterType: + metric = HandleCounter(res, name, value) + default: + HandleUnknownMetric(res) + } + + if err := storage.UpdateMetric(metric); err != nil { + http.Error(res, err.Error(), http.StatusBadRequest) + return + } + + res.WriteHeader(http.StatusOK) + } +} + +func main() { + storage := ms.NewMemStorage() + + mux := http.NewServeMux() + mux.HandleFunc("/update", mainHandle(storage)) + + if err := http.ListenAndServe(`:8080`, mux); err != nil { + panic(err) + } +} diff --git a/internal/metrics/counter.go b/internal/metrics/counter.go index 7f8f6cc..6d4677f 100644 --- a/internal/metrics/counter.go +++ b/internal/metrics/counter.go @@ -5,9 +5,10 @@ type counter struct { value int64 } -func NewCounter(name string) Metric { +func NewCounter(name string, value int64) Metric { return &counter{ - name: name, + name: name, + value: value, } } @@ -22,3 +23,12 @@ func (c *counter) Type() string { func (c *counter) Value() interface{} { return c.value } + +func (c *counter) SetValue(v interface{}) error { + val, ok := v.(int64) + if !ok { + return ErrInvalidValueType + } + c.value = val + return nil +} diff --git a/internal/metrics/gauge.go b/internal/metrics/gauge.go index 812dd8f..38ff84d 100644 --- a/internal/metrics/gauge.go +++ b/internal/metrics/gauge.go @@ -4,9 +4,11 @@ type gauge struct { name string value float64 } -func NewGauge(name string) Metric { + +func NewGauge(name string, value float64) Metric { return &gauge{ - name: name, + name: name, + value: value, } } @@ -21,3 +23,12 @@ func (g *gauge) Type() string { func (g *gauge) Value() interface{} { return g.value } + +func (g *gauge) SetValue(v interface{}) error { + val, ok := v.(float64) + if !ok { + return ErrInvalidValueType + } + g.value = val + return nil +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index eae8bf8..882f575 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -5,6 +5,7 @@ import ( ) type Metric interface { + //SetValue(val interface{}) error Value() interface{} Name() string Type() string From 68a44fe40e848c9492542a9a573bb32b01b2181c Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Thu, 26 Jun 2025 19:21:53 +0300 Subject: [PATCH 003/126] add go.mod --- go.mod | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 go.mod diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..83ef410 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/rAch-kaplin/mipt-golang-course/MetricsService + +go 1.24.1 From 1c11cbfdc3a91dac4286040ba6138b0cf50ddc1f Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Thu, 26 Jun 2025 19:32:25 +0300 Subject: [PATCH 004/126] change go version for CI --- .github/workflows/metricstest.yml | 2 +- .github/workflows/statictest.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/metricstest.yml b/.github/workflows/metricstest.yml index 7392b92..462c49c 100644 --- a/.github/workflows/metricstest.yml +++ b/.github/workflows/metricstest.yml @@ -18,7 +18,7 @@ jobs: metricstest: runs-on: ubuntu-latest - container: golang:1.22 + container: golang:1.24.1 needs: branchtest services: diff --git a/.github/workflows/statictest.yml b/.github/workflows/statictest.yml index 61725e4..56f344e 100644 --- a/.github/workflows/statictest.yml +++ b/.github/workflows/statictest.yml @@ -9,7 +9,7 @@ on: jobs: statictest: runs-on: ubuntu-latest - container: golang:1.22 + container: golang:1.24.1 steps: - name: Checkout code uses: actions/checkout@v2 From 4642ae63210f58b85f11c6fb8f7afe28d4b47768 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Thu, 26 Jun 2025 19:43:52 +0300 Subject: [PATCH 005/126] wrote func removeEmptyStrings() --- cmd/server/main.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index 26d0998..e1a6e17 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -47,6 +47,17 @@ func HandleUnknownMetric(res http.ResponseWriter) { http.Error(res, "unknown type metric!", http.StatusBadRequest) } +func removeEmptyStrings(url []string) []string { + result := make([]string, 0, len(url)) + for _, str := range url { + if str != "" { + result = append(result, str) + } + } + + return result +} + func mainHandle(storage ms.Collector) http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { @@ -61,6 +72,7 @@ func mainHandle(storage ms.Collector) http.HandlerFunc { } parts := strings.Split(req.URL.Path, "/") + parts = removeEmptyStrings(parts) if len(parts) != 4 && parts[0] != "update" { http.Error(res, "invalid request", http.StatusBadRequest) @@ -97,7 +109,7 @@ func main() { storage := ms.NewMemStorage() mux := http.NewServeMux() - mux.HandleFunc("/update", mainHandle(storage)) + mux.HandleFunc("/update/", mainHandle(storage)) if err := http.ListenAndServe(`:8080`, mux); err != nil { panic(err) From d832015cbc28586ebcd8048c5550f07d3c5174bc Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Fri, 27 Jun 2025 03:29:06 +0300 Subject: [PATCH 006/126] fixed bugs, first version with successful tests --- cmd/server/main.go | 50 ++++++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index e1a6e17..fe55f21 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -9,38 +9,24 @@ import ( mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" ) -func HandleCounter(res http.ResponseWriter, name, value string) mtr.Metric { +func HandleCounter(res http.ResponseWriter, name, value string) (mtr.Metric, error) { val, err := strconv.ParseInt(value, 10, 64) if err != nil { - http.Error(res, "invalid value metric", http.StatusBadRequest) - return nil + //http.Error(res, "invalid value metric", http.StatusBadRequest) + return nil, err } - counter := mtr.NewCounter(name, val) - // if err := counter.SetValue(val); err != nil { - // http.Error(res, err.Error(), http.StatusBadRequest) - // return nil - // } - - res.WriteHeader(http.StatusOK) - return counter + return mtr.NewCounter(name, val), nil } -func HandleGauge(res http.ResponseWriter, name, value string) mtr.Metric { +func HandleGauge(res http.ResponseWriter, name, value string) (mtr.Metric, error) { val, err := strconv.ParseFloat(value, 64) if err != nil { - http.Error(res, "invalid value metric", http.StatusBadRequest) - return nil + //http.Error(res, "invalid value metric", http.StatusBadRequest) + return nil, err } - gauge := mtr.NewGauge(name, val) - // if err := gauge.SetValue(val); err != nil { - // http.Error(res, err.Error(), http.StatusBadRequest) - // return nil - // } - - res.WriteHeader(http.StatusOK) - return gauge + return mtr.NewGauge(name, val), nil } func HandleUnknownMetric(res http.ResponseWriter) { @@ -71,29 +57,37 @@ func mainHandle(storage ms.Collector) http.HandlerFunc { return } - parts := strings.Split(req.URL.Path, "/") + parts := strings.Split(strings.Trim(req.URL.Path, "/"), "/") parts = removeEmptyStrings(parts) - if len(parts) != 4 && parts[0] != "update" { - http.Error(res, "invalid request", http.StatusBadRequest) + if len(parts) != 4 || parts[0] != "update" { + http.Error(res, "invalid request", http.StatusNotFound) return } metricType, name, value := parts[1], parts[2], parts[3] if name == "" { - http.Error(res, "metric name is missing", http.StatusFound) + http.Error(res, "metric name is missing", http.StatusNotFound) return } var metric mtr.Metric + var err error + switch metricType { case mtr.GaugeType: - metric = HandleGauge(res, name, value) + metric, err = HandleGauge(res, name, value) case mtr.CounterType: - metric = HandleCounter(res, name, value) + metric, err = HandleCounter(res, name, value) default: HandleUnknownMetric(res) + return + } + + if err != nil { + http.Error(res, err.Error(), http.StatusBadRequest) + return } if err := storage.UpdateMetric(metric); err != nil { From ff4357f22f041ca5523edd534c294e22122b8ce3 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Fri, 27 Jun 2025 03:44:42 +0300 Subject: [PATCH 007/126] for static testing I downgraded the version of go --- .github/workflows/statictest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/statictest.yml b/.github/workflows/statictest.yml index 56f344e..0643b0b 100644 --- a/.github/workflows/statictest.yml +++ b/.github/workflows/statictest.yml @@ -9,7 +9,7 @@ on: jobs: statictest: runs-on: ubuntu-latest - container: golang:1.24.1 + container: golang:1.21 steps: - name: Checkout code uses: actions/checkout@v2 From d9e3fffe9c998426b2b6c82007c5ba627fe5a6e8 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Mon, 30 Jun 2025 02:49:39 +0300 Subject: [PATCH 008/126] add my logger --- .github/workflows/statictest.yml | 2 +- .gitignore | 2 + cmd/server/main.go | 10 ++++ logger/logger.go | 97 ++++++++++++++++++++++++++++++++ 4 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 logger/logger.go diff --git a/.github/workflows/statictest.yml b/.github/workflows/statictest.yml index 0643b0b..56f344e 100644 --- a/.github/workflows/statictest.yml +++ b/.github/workflows/statictest.yml @@ -9,7 +9,7 @@ on: jobs: statictest: runs-on: ubuntu-latest - container: golang:1.21 + container: golang:1.24.1 steps: - name: Checkout code uses: actions/checkout@v2 diff --git a/.gitignore b/.gitignore index 7125517..8c996bb 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ go.work.sum # IDEs directories .idea .vscode +logFile.log +cmd/server/server diff --git a/cmd/server/main.go b/cmd/server/main.go index fe55f21..3997ca6 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -7,6 +7,7 @@ import ( ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/MemStorage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" ) func HandleCounter(res http.ResponseWriter, name, value string) (mtr.Metric, error) { @@ -47,6 +48,8 @@ func removeEmptyStrings(url []string) []string { func mainHandle(storage ms.Collector) http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { + log.Debug("Incoming request: %s %s", req.Method, req.URL.Path) + if req.Method != http.MethodPost { http.Error(res, "use only POST request", http.StatusMethodNotAllowed) return @@ -66,6 +69,7 @@ func mainHandle(storage ms.Collector) http.HandlerFunc { } metricType, name, value := parts[1], parts[2], parts[3] + log.Debug("Parsed metric: type=%s, name=%s, value=%s", metricType, name, value) if name == "" { http.Error(res, "metric name is missing", http.StatusNotFound) @@ -95,11 +99,16 @@ func mainHandle(storage ms.Collector) http.HandlerFunc { return } + log.Info("Metric updated successfully: %s %s = %s", metricType, name, value) res.WriteHeader(http.StatusOK) } } func main() { + log.Init(log.DebugLevel, "logFile.log") + defer log.Destroy() + + log.Debug("START>") storage := ms.NewMemStorage() mux := http.NewServeMux() @@ -108,4 +117,5 @@ func main() { if err := http.ListenAndServe(`:8080`, mux); err != nil { panic(err) } + log.Debug("END<") } diff --git a/logger/logger.go b/logger/logger.go new file mode 100644 index 0000000..d3c2e05 --- /dev/null +++ b/logger/logger.go @@ -0,0 +1,97 @@ +package logger + +import ( + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "sync" + "time" +) + +type LogLevel int + +const ( + DebugLevel LogLevel = iota + InfoLevel + ErrorLevel +) + +var levelMap = map[LogLevel]string{ + DebugLevel: "[DEBUG]", + InfoLevel: "[INFO]", + ErrorLevel: "[ERROR]", +} + +type logger struct { + level LogLevel + out io.Writer +} + +var ( + instance *logger + once sync.Once +) + +func getLogger() *logger { + once.Do(func() { + instance = &logger{ + level: DebugLevel, + out: os.Stdout, + } + }) + return instance +} + +func Init(level LogLevel, logFileName string) error { + log := getLogger() + log.level = level + + if logFileName != "" { + file, err := os.Create(logFileName) + if err != nil { + return fmt.Errorf("Failed to open log file %s: %w", logFileName, err) + } + log.out = file + } else { + log.out = os.Stdout + } + + return nil +} + +func Destroy() { + log := getLogger() + if file, ok := log.out.(*os.File); ok { + file.Close() + } +} + +func (log *logger) log(level LogLevel, format string, args ...interface{}) { + if log.level > level { + return + } + + _, file, line, ok := runtime.Caller(2) + if !ok { + file, line = "---", 0 + } + time := time.Now().Format("2006-01-02 15:04:05") + levelStr := levelMap[level] + msg := fmt.Sprintf(format, args...) + + fmt.Fprintf(log.out, "[%s]%s[%s:%d]: %s \n", time, levelStr, filepath.Base(file), line, msg) +} + +func Debug(format string, args ...interface{}) { + getLogger().log(DebugLevel, format, args...) +} + +func Info(format string, args ...interface{}) { + getLogger().log(InfoLevel, format, args...) +} + +func Error(format string, args ...interface{}) { + getLogger().log(ErrorLevel, format, args...) +} From ac4f43a9c8706198b7dc13ff4831532f744699f6 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Wed, 2 Jul 2025 01:00:09 +0300 Subject: [PATCH 009/126] add agent v.1, without tests --- cmd/agent/main.go | 259 ++++++++++++++++++++++++++++++++- cmd/server/main.go | 4 +- go.mod | 2 + go.sum | 2 + internal/MemStorage/storage.go | 35 +++-- 5 files changed, 290 insertions(+), 12 deletions(-) create mode 100644 go.sum diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 38dd16d..c73631a 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -1,3 +1,260 @@ package main -func main() {} +import ( + "fmt" + "io" + "math/rand" + "net/http" + "runtime" + "time" + + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/MemStorage" + mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" +) + +const ( + pollInterval = 2 * time.Second + reportInterval = 10 * time.Second + + serverAddress = "http://localhost:8080" +) + +type MemRuntimeStat struct { + Name string + Type string + Get func(m *runtime.MemStats) interface{} +} + +var MemRuntimeStats []MemRuntimeStat = []MemRuntimeStat{ + { + Name: "Alloc", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.Alloc }, + }, + { + Name: "BuckHashSys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.BuckHashSys }, + }, + { + Name: "Frees", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.Frees }, + }, + { + Name: "GCCPUFraction", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.GCCPUFraction }, + }, + { + Name: "HeapAlloc", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.HeapAlloc }, + }, + { + Name: "HeapIdle", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.HeapIdle }, + }, + { + Name: "HeapInuse", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.HeapInuse }, + }, + { + Name: "HeapObjects", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.HeapObjects }, + }, + { + Name: "HeapReleased", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.HeapReleased }, + }, + { + Name: "HeapSys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.HeapSys }, + }, + { + Name: "LastGC", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.LastGC }, + }, + { + Name: "Lookups", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.Lookups }, + }, + { + Name: "MCacheInuse", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.MCacheInuse }, + }, + { + Name: "MCacheSys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.MCacheSys }, + }, + { + Name: "MSpanInuse", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.MSpanInuse }, + }, + { + Name: "MSpanSys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.MSpanSys }, + }, + { + Name: "Mallocs", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.Mallocs }, + }, + { + Name: "NextGC", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.NextGC }, + }, + { + Name: "NumForcedGC", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.NumForcedGC }, + }, + { + Name: "NumGC", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.NumGC }, + }, + { + Name: "OtherSys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.OtherSys }, + }, + { + Name: "PauseTotalNs", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.PauseTotalNs }, + }, + { + Name: "StackInuse", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.StackInuse }, + }, + { + Name: "StackSys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.StackSys }, + }, + { + Name: "Sys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.Sys }, + }, + { + Name: "TotalAlloc", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.TotalAlloc }, + }, +} + +func UpdateAllMetrics(storage *ms.MemStorage) { + var memStats runtime.MemStats + runtime.ReadMemStats(&memStats) + + for _, stat := range MemRuntimeStats { + value := stat.Get(&memStats) + var metric mtr.Metric + + switch v := value.(type) { + case uint64: + if stat.Type == mtr.GaugeType { + metric = mtr.NewGauge(stat.Name, float64(v)) + } + case float64: + if stat.Type == mtr.GaugeType { + metric = mtr.NewGauge(stat.Name, v) + } + default: + log.Error("ERROR: Unknown type for metric %s: %T", stat.Name, value) + continue + } + + if err := storage.UpdateMetric(metric); err != nil { + log.Error("ERROR!") + } + } + + storage.UpdateMetric(mtr.NewCounter("PollCount", 1)) + storage.UpdateMetric(mtr.NewGauge("RandomValue", rand.Float64())) +} + +func sendAllMetrics(client *http.Client, storage *ms.MemStorage) { + gauges, counters := storage.GetAllMetrics() + + for name, value := range gauges { + sendMetric(client, mtr.GaugeType, name, value) + } + + for name, value := range counters { + sendMetric(client, mtr.CounterType, name, value) + } +} + +func sendMetric(client *http.Client, mType string, mName string, mValue interface{}) { + url := fmt.Sprintf("%s/update/%s/%s/%v", serverAddress, mType, mName, mValue) + + req, err := http.NewRequest(http.MethodPost, url, http.NoBody) + if err != nil { + log.Error("Error creating a request for %s: %v\n", mName, err) + return + } + req.Header.Set("Content-Type", "text-plain") + + resp, err := client.Do(req) + if err != nil { + log.Error("Error sending metric: %s: %v\n", mName, err) + return + } + defer resp.Body.Close() + + _, err = io.Copy(io.Discard, resp.Body) + if err != nil { + log.Error("Failed to read response body from %s: %v", url, err) + } +} + +func collectionLoop(storage *ms.MemStorage, interval time.Duration) { + log.Debug("collectionLoop ...") + for { + UpdateAllMetrics(storage) + time.Sleep(interval) + } +} + +func reportLoop(client *http.Client, storage *ms.MemStorage, interval time.Duration) { + log.Debug("reportLoop ...") + for { + time.Sleep(interval) + sendAllMetrics(client, storage) + } +} + +func main() { + log.Init(log.DebugLevel, "logFile.log") + defer log.Destroy() + + log.Debug("START AGENT>") + storage := ms.NewMemStorage() + + client := &http.Client{ + Timeout: 5 * time.Second, + } + + go collectionLoop(storage, pollInterval) + go reportLoop(client, storage, reportInterval) + + select {} + log.Debug("END AGENT<") +} diff --git a/cmd/server/main.go b/cmd/server/main.go index 3997ca6..97db9ac 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -108,7 +108,7 @@ func main() { log.Init(log.DebugLevel, "logFile.log") defer log.Destroy() - log.Debug("START>") + log.Debug("START SERVER>") storage := ms.NewMemStorage() mux := http.NewServeMux() @@ -117,5 +117,5 @@ func main() { if err := http.ListenAndServe(`:8080`, mux); err != nil { panic(err) } - log.Debug("END<") + log.Debug("END SERVER<") } diff --git a/go.mod b/go.mod index 83ef410..c61b255 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module github.com/rAch-kaplin/mipt-golang-course/MetricsService go 1.24.1 + +require github.com/stretchr/testify v1.10.0 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..7bfdabe --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= diff --git a/internal/MemStorage/storage.go b/internal/MemStorage/storage.go index 8c565d2..dbeb93e 100644 --- a/internal/MemStorage/storage.go +++ b/internal/MemStorage/storage.go @@ -11,17 +11,17 @@ type Collector interface { UpdateMetric(mtr metrics.Metric) error } + type MemStorage struct { mutex sync.RWMutex - gauges map[string]float64 - counters map[string]int64 - //metrics map[string]metrics.Metric + Gauges map[string]float64 + Counters map[string]int64 } func NewMemStorage() *MemStorage { return &MemStorage{ - gauges: make(map[string]float64), - counters: make(map[string]int64), + Gauges: make(map[string]float64), + Counters: make(map[string]int64), } } @@ -55,13 +55,13 @@ func (ms *MemStorage) UpdateMetric(metric metrics.Metric) error { func (ms *MemStorage) UpdateGauge(metric metrics.Metric, value float64) { ms.mutex.Lock() defer ms.mutex.Unlock() - ms.gauges[metric.Name()] = value + ms.Gauges[metric.Name()] = value } func (ms *MemStorage) UpdateCounter(metric metrics.Metric, value int64) { ms.mutex.Lock() defer ms.mutex.Unlock() - ms.counters[metric.Name()] += value + ms.Counters[metric.Name()] += value } func (ms *MemStorage) GetMetric(metric metrics.Metric) (interface{}, error) { @@ -93,7 +93,7 @@ func (ms *MemStorage) GetGauges(name string) (float64, bool) { ms.mutex.RLock() defer ms.mutex.RUnlock() - val, ok := ms.gauges[name] + val, ok := ms.Gauges[name] return val, ok } @@ -101,6 +101,23 @@ func (ms *MemStorage) GetCounter(name string) (int64, bool) { ms.mutex.RLock() defer ms.mutex.RUnlock() - val, ok := ms.counters[name] + val, ok := ms.Counters[name] return val, ok } + +func (ms *MemStorage) GetAllMetrics() (map[string]float64, map[string]int64) { + ms.mutex.RLock() + defer ms.mutex.RUnlock() + + gaugesCopy := make(map[string]float64, len(ms.Gauges)) + for name, value := range ms.Gauges { + gaugesCopy[name] = value + } + + countersCopy := make(map[string]int64, len(ms.Counters)) + for name, value := range ms.Counters { + countersCopy[name] = value + } + + return gaugesCopy, countersCopy +} From 47e938bb3af3b527b5bca1771c279b6812c5f011 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Wed, 2 Jul 2025 01:24:26 +0300 Subject: [PATCH 010/126] divided into section --- cmd/agent/main.go | 233 +--------------------------- cmd/server/main.go | 100 +----------- internal/handlers/agent/agent.go | 239 +++++++++++++++++++++++++++++ internal/handlers/server/server.go | 105 +++++++++++++ 4 files changed, 349 insertions(+), 328 deletions(-) create mode 100644 internal/handlers/agent/agent.go create mode 100644 internal/handlers/server/server.go diff --git a/cmd/agent/main.go b/cmd/agent/main.go index c73631a..8dc2016 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -1,246 +1,19 @@ package main import ( - "fmt" - "io" - "math/rand" "net/http" - "runtime" "time" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/MemStorage" - mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/agent" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" ) const ( pollInterval = 2 * time.Second reportInterval = 10 * time.Second - - serverAddress = "http://localhost:8080" ) -type MemRuntimeStat struct { - Name string - Type string - Get func(m *runtime.MemStats) interface{} -} - -var MemRuntimeStats []MemRuntimeStat = []MemRuntimeStat{ - { - Name: "Alloc", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.Alloc }, - }, - { - Name: "BuckHashSys", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.BuckHashSys }, - }, - { - Name: "Frees", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.Frees }, - }, - { - Name: "GCCPUFraction", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.GCCPUFraction }, - }, - { - Name: "HeapAlloc", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.HeapAlloc }, - }, - { - Name: "HeapIdle", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.HeapIdle }, - }, - { - Name: "HeapInuse", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.HeapInuse }, - }, - { - Name: "HeapObjects", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.HeapObjects }, - }, - { - Name: "HeapReleased", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.HeapReleased }, - }, - { - Name: "HeapSys", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.HeapSys }, - }, - { - Name: "LastGC", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.LastGC }, - }, - { - Name: "Lookups", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.Lookups }, - }, - { - Name: "MCacheInuse", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.MCacheInuse }, - }, - { - Name: "MCacheSys", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.MCacheSys }, - }, - { - Name: "MSpanInuse", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.MSpanInuse }, - }, - { - Name: "MSpanSys", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.MSpanSys }, - }, - { - Name: "Mallocs", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.Mallocs }, - }, - { - Name: "NextGC", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.NextGC }, - }, - { - Name: "NumForcedGC", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.NumForcedGC }, - }, - { - Name: "NumGC", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.NumGC }, - }, - { - Name: "OtherSys", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.OtherSys }, - }, - { - Name: "PauseTotalNs", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.PauseTotalNs }, - }, - { - Name: "StackInuse", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.StackInuse }, - }, - { - Name: "StackSys", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.StackSys }, - }, - { - Name: "Sys", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.Sys }, - }, - { - Name: "TotalAlloc", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.TotalAlloc }, - }, -} - -func UpdateAllMetrics(storage *ms.MemStorage) { - var memStats runtime.MemStats - runtime.ReadMemStats(&memStats) - - for _, stat := range MemRuntimeStats { - value := stat.Get(&memStats) - var metric mtr.Metric - - switch v := value.(type) { - case uint64: - if stat.Type == mtr.GaugeType { - metric = mtr.NewGauge(stat.Name, float64(v)) - } - case float64: - if stat.Type == mtr.GaugeType { - metric = mtr.NewGauge(stat.Name, v) - } - default: - log.Error("ERROR: Unknown type for metric %s: %T", stat.Name, value) - continue - } - - if err := storage.UpdateMetric(metric); err != nil { - log.Error("ERROR!") - } - } - - storage.UpdateMetric(mtr.NewCounter("PollCount", 1)) - storage.UpdateMetric(mtr.NewGauge("RandomValue", rand.Float64())) -} - -func sendAllMetrics(client *http.Client, storage *ms.MemStorage) { - gauges, counters := storage.GetAllMetrics() - - for name, value := range gauges { - sendMetric(client, mtr.GaugeType, name, value) - } - - for name, value := range counters { - sendMetric(client, mtr.CounterType, name, value) - } -} - -func sendMetric(client *http.Client, mType string, mName string, mValue interface{}) { - url := fmt.Sprintf("%s/update/%s/%s/%v", serverAddress, mType, mName, mValue) - - req, err := http.NewRequest(http.MethodPost, url, http.NoBody) - if err != nil { - log.Error("Error creating a request for %s: %v\n", mName, err) - return - } - req.Header.Set("Content-Type", "text-plain") - - resp, err := client.Do(req) - if err != nil { - log.Error("Error sending metric: %s: %v\n", mName, err) - return - } - defer resp.Body.Close() - - _, err = io.Copy(io.Discard, resp.Body) - if err != nil { - log.Error("Failed to read response body from %s: %v", url, err) - } -} - -func collectionLoop(storage *ms.MemStorage, interval time.Duration) { - log.Debug("collectionLoop ...") - for { - UpdateAllMetrics(storage) - time.Sleep(interval) - } -} - -func reportLoop(client *http.Client, storage *ms.MemStorage, interval time.Duration) { - log.Debug("reportLoop ...") - for { - time.Sleep(interval) - sendAllMetrics(client, storage) - } -} - func main() { log.Init(log.DebugLevel, "logFile.log") defer log.Destroy() @@ -252,8 +25,8 @@ func main() { Timeout: 5 * time.Second, } - go collectionLoop(storage, pollInterval) - go reportLoop(client, storage, reportInterval) + go agent.CollectionLoop(storage, pollInterval) + go agent.ReportLoop(client, storage, reportInterval) select {} log.Debug("END AGENT<") diff --git a/cmd/server/main.go b/cmd/server/main.go index 97db9ac..ed06ae1 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -2,108 +2,12 @@ package main import ( "net/http" - "strconv" - "strings" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/MemStorage" - mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" ) -func HandleCounter(res http.ResponseWriter, name, value string) (mtr.Metric, error) { - val, err := strconv.ParseInt(value, 10, 64) - if err != nil { - //http.Error(res, "invalid value metric", http.StatusBadRequest) - return nil, err - } - - return mtr.NewCounter(name, val), nil -} - -func HandleGauge(res http.ResponseWriter, name, value string) (mtr.Metric, error) { - val, err := strconv.ParseFloat(value, 64) - if err != nil { - //http.Error(res, "invalid value metric", http.StatusBadRequest) - return nil, err - } - - return mtr.NewGauge(name, val), nil -} - -func HandleUnknownMetric(res http.ResponseWriter) { - http.Error(res, "unknown type metric!", http.StatusBadRequest) -} - -func removeEmptyStrings(url []string) []string { - result := make([]string, 0, len(url)) - for _, str := range url { - if str != "" { - result = append(result, str) - } - } - - return result -} - -func mainHandle(storage ms.Collector) http.HandlerFunc { - return func(res http.ResponseWriter, req *http.Request) { - - log.Debug("Incoming request: %s %s", req.Method, req.URL.Path) - - if req.Method != http.MethodPost { - http.Error(res, "use only POST request", http.StatusMethodNotAllowed) - return - } - - if req.Header.Get("Content-Type") != "text/plain" { - http.Error(res, "use only text/plain", http.StatusBadRequest) - return - } - - parts := strings.Split(strings.Trim(req.URL.Path, "/"), "/") - parts = removeEmptyStrings(parts) - - if len(parts) != 4 || parts[0] != "update" { - http.Error(res, "invalid request", http.StatusNotFound) - return - } - - metricType, name, value := parts[1], parts[2], parts[3] - log.Debug("Parsed metric: type=%s, name=%s, value=%s", metricType, name, value) - - if name == "" { - http.Error(res, "metric name is missing", http.StatusNotFound) - return - } - - var metric mtr.Metric - var err error - - switch metricType { - case mtr.GaugeType: - metric, err = HandleGauge(res, name, value) - case mtr.CounterType: - metric, err = HandleCounter(res, name, value) - default: - HandleUnknownMetric(res) - return - } - - if err != nil { - http.Error(res, err.Error(), http.StatusBadRequest) - return - } - - if err := storage.UpdateMetric(metric); err != nil { - http.Error(res, err.Error(), http.StatusBadRequest) - return - } - - log.Info("Metric updated successfully: %s %s = %s", metricType, name, value) - res.WriteHeader(http.StatusOK) - } -} - func main() { log.Init(log.DebugLevel, "logFile.log") defer log.Destroy() @@ -112,7 +16,7 @@ func main() { storage := ms.NewMemStorage() mux := http.NewServeMux() - mux.HandleFunc("/update/", mainHandle(storage)) + mux.HandleFunc("/update/", server.MainHandle(storage)) if err := http.ListenAndServe(`:8080`, mux); err != nil { panic(err) diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go new file mode 100644 index 0000000..b39fdfc --- /dev/null +++ b/internal/handlers/agent/agent.go @@ -0,0 +1,239 @@ +package agent + +import ( + "fmt" + "io" + "math/rand" + "net/http" + "runtime" + "time" + + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/MemStorage" + mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" +) + +const ( + serverAddress = "http://localhost:8080" +) + +type MemRuntimeStat struct { + Name string + Type string + Get func(m *runtime.MemStats) interface{} +} + +var MemRuntimeStats []MemRuntimeStat = []MemRuntimeStat{ + { + Name: "Alloc", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.Alloc }, + }, + { + Name: "BuckHashSys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.BuckHashSys }, + }, + { + Name: "Frees", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.Frees }, + }, + { + Name: "GCCPUFraction", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.GCCPUFraction }, + }, + { + Name: "HeapAlloc", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.HeapAlloc }, + }, + { + Name: "HeapIdle", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.HeapIdle }, + }, + { + Name: "HeapInuse", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.HeapInuse }, + }, + { + Name: "HeapObjects", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.HeapObjects }, + }, + { + Name: "HeapReleased", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.HeapReleased }, + }, + { + Name: "HeapSys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.HeapSys }, + }, + { + Name: "LastGC", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.LastGC }, + }, + { + Name: "Lookups", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.Lookups }, + }, + { + Name: "MCacheInuse", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.MCacheInuse }, + }, + { + Name: "MCacheSys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.MCacheSys }, + }, + { + Name: "MSpanInuse", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.MSpanInuse }, + }, + { + Name: "MSpanSys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.MSpanSys }, + }, + { + Name: "Mallocs", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.Mallocs }, + }, + { + Name: "NextGC", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.NextGC }, + }, + { + Name: "NumForcedGC", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.NumForcedGC }, + }, + { + Name: "NumGC", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.NumGC }, + }, + { + Name: "OtherSys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.OtherSys }, + }, + { + Name: "PauseTotalNs", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.PauseTotalNs }, + }, + { + Name: "StackInuse", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.StackInuse }, + }, + { + Name: "StackSys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.StackSys }, + }, + { + Name: "Sys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.Sys }, + }, + { + Name: "TotalAlloc", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) interface{} { return m.TotalAlloc }, + }, +} + +func UpdateAllMetrics(storage *ms.MemStorage) { + var memStats runtime.MemStats + runtime.ReadMemStats(&memStats) + + for _, stat := range MemRuntimeStats { + value := stat.Get(&memStats) + var metric mtr.Metric + + switch v := value.(type) { + case uint64: + if stat.Type == mtr.GaugeType { + metric = mtr.NewGauge(stat.Name, float64(v)) + } + case float64: + if stat.Type == mtr.GaugeType { + metric = mtr.NewGauge(stat.Name, v) + } + default: + log.Error("ERROR: Unknown type for metric %s: %T", stat.Name, value) + continue + } + + if err := storage.UpdateMetric(metric); err != nil { + log.Error("ERROR!") + } + } + + storage.UpdateMetric(mtr.NewCounter("PollCount", 1)) + storage.UpdateMetric(mtr.NewGauge("RandomValue", rand.Float64())) +} + +func sendAllMetrics(client *http.Client, storage *ms.MemStorage) { + gauges, counters := storage.GetAllMetrics() + + for name, value := range gauges { + sendMetric(client, mtr.GaugeType, name, value) + } + + for name, value := range counters { + sendMetric(client, mtr.CounterType, name, value) + } +} + +func sendMetric(client *http.Client, mType string, mName string, mValue interface{}) { + url := fmt.Sprintf("%s/update/%s/%s/%v", serverAddress, mType, mName, mValue) + + req, err := http.NewRequest(http.MethodPost, url, http.NoBody) + if err != nil { + log.Error("Error creating a request for %s: %v\n", mName, err) + return + } + req.Header.Set("Content-Type", "text-plain") + + resp, err := client.Do(req) + if err != nil { + log.Error("Error sending metric: %s: %v\n", mName, err) + return + } + defer resp.Body.Close() + + _, err = io.Copy(io.Discard, resp.Body) + if err != nil { + log.Error("Failed to read response body from %s: %v", url, err) + } +} + +func CollectionLoop(storage *ms.MemStorage, interval time.Duration) { + log.Debug("collectionLoop ...") + for { + UpdateAllMetrics(storage) + time.Sleep(interval) + } +} + +func ReportLoop(client *http.Client, storage *ms.MemStorage, interval time.Duration) { + log.Debug("reportLoop ...") + for { + time.Sleep(interval) + sendAllMetrics(client, storage) + } +} diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go new file mode 100644 index 0000000..2f45a84 --- /dev/null +++ b/internal/handlers/server/server.go @@ -0,0 +1,105 @@ +package server + +import ( + "net/http" + "strconv" + "strings" + + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/MemStorage" + mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" +) + +func HandleCounter(res http.ResponseWriter, name, value string) (mtr.Metric, error) { + val, err := strconv.ParseInt(value, 10, 64) + if err != nil { + http.Error(res, "invalid value metric", http.StatusBadRequest) + return nil, err + } + + return mtr.NewCounter(name, val), nil +} + +func HandleGauge(res http.ResponseWriter, name, value string) (mtr.Metric, error) { + val, err := strconv.ParseFloat(value, 64) + if err != nil { + http.Error(res, "invalid value metric", http.StatusBadRequest) + return nil, err + } + + return mtr.NewGauge(name, val), nil +} + +func HandleUnknownMetric(res http.ResponseWriter) { + http.Error(res, "unknown type metric!", http.StatusBadRequest) +} + +func removeEmptyStrings(url []string) []string { + result := make([]string, 0, len(url)) + for _, str := range url { + if str != "" { + result = append(result, str) + } + } + + return result +} + +func MainHandle(storage ms.Collector) http.HandlerFunc { + return func(res http.ResponseWriter, req *http.Request) { + + log.Debug("Incoming request: %s %s", req.Method, req.URL.Path) + + if req.Method != http.MethodPost { + http.Error(res, "use only POST request", http.StatusMethodNotAllowed) + return + } + + if req.Header.Get("Content-Type") != "text/plain" { + http.Error(res, "use only text/plain", http.StatusBadRequest) + return + } + + parts := strings.Split(strings.Trim(req.URL.Path, "/"), "/") + parts = removeEmptyStrings(parts) + + if len(parts) != 4 || parts[0] != "update" { + http.Error(res, "invalid request", http.StatusNotFound) + return + } + + metricType, name, value := parts[1], parts[2], parts[3] + log.Debug("Parsed metric: type=%s, name=%s, value=%s", metricType, name, value) + + if name == "" { + http.Error(res, "metric name is missing", http.StatusNotFound) + return + } + + var metric mtr.Metric + var err error + + switch metricType { + case mtr.GaugeType: + metric, err = HandleGauge(res, name, value) + case mtr.CounterType: + metric, err = HandleCounter(res, name, value) + default: + HandleUnknownMetric(res) + return + } + + if err != nil { + http.Error(res, err.Error(), http.StatusBadRequest) + return + } + + if err := storage.UpdateMetric(metric); err != nil { + http.Error(res, err.Error(), http.StatusBadRequest) + return + } + + log.Info("Metric updated successfully: %s %s = %s", metricType, name, value) + res.WriteHeader(http.StatusOK) + } +} From 7de3fec1bcbced18ebdbe22fe868d9b986ac94cb Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Wed, 2 Jul 2025 02:44:21 +0300 Subject: [PATCH 011/126] add tests for server --- internal/handlers/server/server_test.go | 67 +++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 internal/handlers/server/server_test.go diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go new file mode 100644 index 0000000..2e56dc8 --- /dev/null +++ b/internal/handlers/server/server_test.go @@ -0,0 +1,67 @@ +package server + +import ( + "testing" + "net/http/httptest" + "net/http" + + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/MemStorage" +) + +func TestMainHandle(t *testing.T) { + storage := ms.NewMemStorage() + handler := MainHandle(storage) + + tests := []struct { + name string + url string + method string + wantStatus int + }{ + { + name: "Gauge update (valid)", + url: "/update/gauge/test_metric/97.25", + method: http.MethodPost, + wantStatus: http.StatusOK, + }, + { + name: "Counter update (valid)", + url: "/update/counter/test_counter/90", + method: http.MethodPost, + wantStatus: http.StatusOK, + }, + { + name: "Invalid method", + url: "/update/gauge/test_metric/97.45", + method: http.MethodGet, + wantStatus: http.StatusMethodNotAllowed, + }, + { + name: "Invalid metric type", + url: "/update/invalid/test_metric/80", + method: http.MethodPost, + wantStatus: http.StatusBadRequest, + }, + { + name: "Missing value", + url: "/update/gauge/test_metric/", + method: http.MethodPost, + wantStatus: http.StatusNotFound, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req := httptest.NewRequest(test.method, test.url, nil) + req.Header.Set("Content-Type", "text/plain") + w := httptest.NewRecorder() + handler(w, req) + + res := w.Result() + if res.StatusCode != test.wantStatus { + t.Errorf("expected status %d, got %d", test.wantStatus, res.StatusCode) + } + }) + } + +} From 60d1be87e80b871f27b99f39a6052b352128b75b Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Wed, 2 Jul 2025 19:20:42 +0300 Subject: [PATCH 012/126] change go version --- .github/workflows/statictest.yml | 2 +- go.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/statictest.yml b/.github/workflows/statictest.yml index 56f344e..61725e4 100644 --- a/.github/workflows/statictest.yml +++ b/.github/workflows/statictest.yml @@ -9,7 +9,7 @@ on: jobs: statictest: runs-on: ubuntu-latest - container: golang:1.24.1 + container: golang:1.22 steps: - name: Checkout code uses: actions/checkout@v2 diff --git a/go.mod b/go.mod index c61b255..d6668ea 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,5 @@ module github.com/rAch-kaplin/mipt-golang-course/MetricsService -go 1.24.1 +go 1.22.0 require github.com/stretchr/testify v1.10.0 // indirect From 983a56d372c7dbfbb73cda5581d46f97e2b3c327 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Wed, 2 Jul 2025 19:55:53 +0300 Subject: [PATCH 013/126] fixed errors for statictest --- cmd/agent/main.go | 4 +- cmd/server/main.go | 2 +- internal/handlers/agent/agent.go | 2 +- internal/handlers/server/server.go | 2 +- internal/handlers/server/server_test.go | 3 +- internal/memStorage/storage.go | 123 ++++++++++++++++++++++++ internal/metrics/metrics.go | 1 - logger/logger.go | 2 +- 8 files changed, 131 insertions(+), 8 deletions(-) create mode 100644 internal/memStorage/storage.go diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 8dc2016..504bfb5 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -4,7 +4,7 @@ import ( "net/http" "time" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/MemStorage" + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memStorage" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/agent" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" ) @@ -28,6 +28,6 @@ func main() { go agent.CollectionLoop(storage, pollInterval) go agent.ReportLoop(client, storage, reportInterval) - select {} log.Debug("END AGENT<") + select {} } diff --git a/cmd/server/main.go b/cmd/server/main.go index ed06ae1..e07d9d9 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -3,7 +3,7 @@ package main import ( "net/http" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/MemStorage" + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memStorage" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" ) diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index b39fdfc..f033ca8 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -8,7 +8,7 @@ import ( "runtime" "time" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/MemStorage" + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memStorage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" ) diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index 2f45a84..ce7e8c5 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -5,7 +5,7 @@ import ( "strconv" "strings" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/MemStorage" + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memStorage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" ) diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index 2e56dc8..58833f8 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -5,7 +5,7 @@ import ( "net/http/httptest" "net/http" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/MemStorage" + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memStorage" ) func TestMainHandle(t *testing.T) { @@ -53,6 +53,7 @@ func TestMainHandle(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { req := httptest.NewRequest(test.method, test.url, nil) + defer req.Body.Close() req.Header.Set("Content-Type", "text/plain") w := httptest.NewRecorder() handler(w, req) diff --git a/internal/memStorage/storage.go b/internal/memStorage/storage.go new file mode 100644 index 0000000..dbeb93e --- /dev/null +++ b/internal/memStorage/storage.go @@ -0,0 +1,123 @@ +package MemStorage + +import ( + "sync" + + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" +) + +type Collector interface { + GetMetric(mtr metrics.Metric) (interface{}, error) + UpdateMetric(mtr metrics.Metric) error +} + + +type MemStorage struct { + mutex sync.RWMutex + Gauges map[string]float64 + Counters map[string]int64 +} + +func NewMemStorage() *MemStorage { + return &MemStorage{ + Gauges: make(map[string]float64), + Counters: make(map[string]int64), + } +} + +func (ms *MemStorage) UpdateMetric(metric metrics.Metric) error { + switch metric.Type() { + case metrics.CounterType: + { + val, ok := metric.Value().(int64) + if !ok { + return metrics.ErrInvalidValueType + } + + ms.UpdateCounter(metric, val) + return nil + } + case metrics.GaugeType: + { + val, ok := metric.Value().(float64) + if !ok { + return metrics.ErrInvalidValueType + } + + ms.UpdateGauge(metric, val) + return nil + } + default: + return metrics.ErrInvalidMetricsType + } +} + +func (ms *MemStorage) UpdateGauge(metric metrics.Metric, value float64) { + ms.mutex.Lock() + defer ms.mutex.Unlock() + ms.Gauges[metric.Name()] = value +} + +func (ms *MemStorage) UpdateCounter(metric metrics.Metric, value int64) { + ms.mutex.Lock() + defer ms.mutex.Unlock() + ms.Counters[metric.Name()] += value +} + +func (ms *MemStorage) GetMetric(metric metrics.Metric) (interface{}, error) { + switch metric.Type() { + case metrics.CounterType: + { + val, ok := ms.GetCounter(metric.Name()) + if !ok { + return nil, metrics.ErrMetricsNotFound + } + + return val, nil + } + case metrics.GaugeType: + { + val, ok := ms.GetGauges(metric.Name()) + if !ok { + return nil, metrics.ErrMetricsNotFound + } + + return val, nil + } + default: + return nil, metrics.ErrInvalidMetricsType + } +} + +func (ms *MemStorage) GetGauges(name string) (float64, bool) { + ms.mutex.RLock() + defer ms.mutex.RUnlock() + + val, ok := ms.Gauges[name] + return val, ok +} + +func (ms *MemStorage) GetCounter(name string) (int64, bool) { + ms.mutex.RLock() + defer ms.mutex.RUnlock() + + val, ok := ms.Counters[name] + return val, ok +} + +func (ms *MemStorage) GetAllMetrics() (map[string]float64, map[string]int64) { + ms.mutex.RLock() + defer ms.mutex.RUnlock() + + gaugesCopy := make(map[string]float64, len(ms.Gauges)) + for name, value := range ms.Gauges { + gaugesCopy[name] = value + } + + countersCopy := make(map[string]int64, len(ms.Counters)) + for name, value := range ms.Counters { + countersCopy[name] = value + } + + return gaugesCopy, countersCopy +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 882f575..eae8bf8 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -5,7 +5,6 @@ import ( ) type Metric interface { - //SetValue(val interface{}) error Value() interface{} Name() string Type() string diff --git a/logger/logger.go b/logger/logger.go index d3c2e05..69ab32c 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -51,7 +51,7 @@ func Init(level LogLevel, logFileName string) error { if logFileName != "" { file, err := os.Create(logFileName) if err != nil { - return fmt.Errorf("Failed to open log file %s: %w", logFileName, err) + return fmt.Errorf("failed to open log file %s: %w", logFileName, err) } log.out = file } else { From 7e8e3546033a97d1764ff045e4fe9b2532e7d8b8 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Wed, 2 Jul 2025 19:59:41 +0300 Subject: [PATCH 014/126] renamed package memStorage --- internal/memStorage/storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/memStorage/storage.go b/internal/memStorage/storage.go index dbeb93e..bfcd7b0 100644 --- a/internal/memStorage/storage.go +++ b/internal/memStorage/storage.go @@ -1,4 +1,4 @@ -package MemStorage +package memStorage import ( "sync" From 2b448d2d11a473e0c206a66910b6a57d86649608 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Wed, 2 Jul 2025 20:02:46 +0300 Subject: [PATCH 015/126] deleted MemStorage --- internal/MemStorage/storage.go | 123 --------------------------------- 1 file changed, 123 deletions(-) delete mode 100644 internal/MemStorage/storage.go diff --git a/internal/MemStorage/storage.go b/internal/MemStorage/storage.go deleted file mode 100644 index dbeb93e..0000000 --- a/internal/MemStorage/storage.go +++ /dev/null @@ -1,123 +0,0 @@ -package MemStorage - -import ( - "sync" - - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" -) - -type Collector interface { - GetMetric(mtr metrics.Metric) (interface{}, error) - UpdateMetric(mtr metrics.Metric) error -} - - -type MemStorage struct { - mutex sync.RWMutex - Gauges map[string]float64 - Counters map[string]int64 -} - -func NewMemStorage() *MemStorage { - return &MemStorage{ - Gauges: make(map[string]float64), - Counters: make(map[string]int64), - } -} - -func (ms *MemStorage) UpdateMetric(metric metrics.Metric) error { - switch metric.Type() { - case metrics.CounterType: - { - val, ok := metric.Value().(int64) - if !ok { - return metrics.ErrInvalidValueType - } - - ms.UpdateCounter(metric, val) - return nil - } - case metrics.GaugeType: - { - val, ok := metric.Value().(float64) - if !ok { - return metrics.ErrInvalidValueType - } - - ms.UpdateGauge(metric, val) - return nil - } - default: - return metrics.ErrInvalidMetricsType - } -} - -func (ms *MemStorage) UpdateGauge(metric metrics.Metric, value float64) { - ms.mutex.Lock() - defer ms.mutex.Unlock() - ms.Gauges[metric.Name()] = value -} - -func (ms *MemStorage) UpdateCounter(metric metrics.Metric, value int64) { - ms.mutex.Lock() - defer ms.mutex.Unlock() - ms.Counters[metric.Name()] += value -} - -func (ms *MemStorage) GetMetric(metric metrics.Metric) (interface{}, error) { - switch metric.Type() { - case metrics.CounterType: - { - val, ok := ms.GetCounter(metric.Name()) - if !ok { - return nil, metrics.ErrMetricsNotFound - } - - return val, nil - } - case metrics.GaugeType: - { - val, ok := ms.GetGauges(metric.Name()) - if !ok { - return nil, metrics.ErrMetricsNotFound - } - - return val, nil - } - default: - return nil, metrics.ErrInvalidMetricsType - } -} - -func (ms *MemStorage) GetGauges(name string) (float64, bool) { - ms.mutex.RLock() - defer ms.mutex.RUnlock() - - val, ok := ms.Gauges[name] - return val, ok -} - -func (ms *MemStorage) GetCounter(name string) (int64, bool) { - ms.mutex.RLock() - defer ms.mutex.RUnlock() - - val, ok := ms.Counters[name] - return val, ok -} - -func (ms *MemStorage) GetAllMetrics() (map[string]float64, map[string]int64) { - ms.mutex.RLock() - defer ms.mutex.RUnlock() - - gaugesCopy := make(map[string]float64, len(ms.Gauges)) - for name, value := range ms.Gauges { - gaugesCopy[name] = value - } - - countersCopy := make(map[string]int64, len(ms.Counters)) - for name, value := range ms.Counters { - countersCopy[name] = value - } - - return gaugesCopy, countersCopy -} From 9622d24147cb8903515bb913afa36941c35b079f Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Wed, 2 Jul 2025 20:10:24 +0300 Subject: [PATCH 016/126] again renamed memStorage --- internal/handlers/server/server_test.go | 2 +- internal/{memStorage => memstorage}/storage.go | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename internal/{memStorage => memstorage}/storage.go (100%) diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index 58833f8..d278bc4 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -53,12 +53,12 @@ func TestMainHandle(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { req := httptest.NewRequest(test.method, test.url, nil) - defer req.Body.Close() req.Header.Set("Content-Type", "text/plain") w := httptest.NewRecorder() handler(w, req) res := w.Result() + defer res.Body.Close() if res.StatusCode != test.wantStatus { t.Errorf("expected status %d, got %d", test.wantStatus, res.StatusCode) } diff --git a/internal/memStorage/storage.go b/internal/memstorage/storage.go similarity index 100% rename from internal/memStorage/storage.go rename to internal/memstorage/storage.go From e5ba7d92e400edcc1f8f761b35f0d5ea1eb069b4 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Wed, 2 Jul 2025 20:15:16 +0300 Subject: [PATCH 017/126] renamed package memstorage --- cmd/agent/main.go | 2 +- cmd/server/main.go | 2 +- internal/handlers/agent/agent.go | 2 +- internal/handlers/server/server.go | 2 +- internal/handlers/server/server_test.go | 2 +- internal/memstorage/storage.go | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 504bfb5..a8912ec 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -4,7 +4,7 @@ import ( "net/http" "time" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memStorage" + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/agent" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" ) diff --git a/cmd/server/main.go b/cmd/server/main.go index e07d9d9..f280ac7 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -3,7 +3,7 @@ package main import ( "net/http" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memStorage" + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" ) diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index f033ca8..44c4072 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -8,7 +8,7 @@ import ( "runtime" "time" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memStorage" + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" ) diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index ce7e8c5..f9510d4 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -5,7 +5,7 @@ import ( "strconv" "strings" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memStorage" + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" ) diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index d278bc4..1b354b5 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -5,7 +5,7 @@ import ( "net/http/httptest" "net/http" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memStorage" + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" ) func TestMainHandle(t *testing.T) { diff --git a/internal/memstorage/storage.go b/internal/memstorage/storage.go index bfcd7b0..eb618cc 100644 --- a/internal/memstorage/storage.go +++ b/internal/memstorage/storage.go @@ -1,4 +1,4 @@ -package memStorage +package memstorage import ( "sync" From 692f20d56af593145c8f2b18b8679045e3df2dc4 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Thu, 3 Jul 2025 01:54:14 +0300 Subject: [PATCH 018/126] first version iter3, with libs: chi, resty --- .gitignore | 1 + Makefile | 29 +++++ cmd/agent/main.go | 9 +- cmd/server/main.go | 14 ++- go.mod | 7 +- go.sum | 6 + internal/handlers/agent/agent.go | 33 +++--- internal/handlers/server/server.go | 145 +++++++++++++++++++----- internal/handlers/server/server_test.go | 144 ++++++++++++++++++----- internal/memstorage/storage.go | 46 +++----- 10 files changed, 319 insertions(+), 115 deletions(-) create mode 100644 Makefile diff --git a/.gitignore b/.gitignore index 8c996bb..995dbda 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ go.work.sum .vscode logFile.log cmd/server/server +cmd/agent/agent diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d20f0aa --- /dev/null +++ b/Makefile @@ -0,0 +1,29 @@ +PROJECT_NAME := MetricsService + +SERVER_SRC_DIR := ./cmd/server +AGENT_SRC_DIR := ./cmd/agent + +SERVER_BIN_NAME := server +AGENT_BIN_NAME := agent + +SERVER_FULL_PATH := $(SERVER_SRC_DIR)/$(SERVER_BIN_NAME) +AGENT_FULL_PATH := $(AGENT_SRC_DIR)/$(AGENT_BIN_NAME) + +.PHONY: all server agent build clean test + +all: build + +build: server agent + +server: + @go build -o $(SERVER_FULL_PATH) $(SERVER_SRC_DIR) + @echo "Built $(SERVER_FULL_PATH)" + +agent: + @go build -o $(AGENT_FULL_PATH) $(AGENT_SRC_DIR) + @echo "Built $(AGENT_FULL_PATH)" + +clean: + @rm -f $(SERVER_FULL_PATH) $(AGENT_FULL_PATH) + @echo "Cleaned." + diff --git a/cmd/agent/main.go b/cmd/agent/main.go index a8912ec..626824f 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -1,11 +1,12 @@ package main import ( - "net/http" "time" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" + "github.com/go-resty/resty/v2" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/agent" + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" ) @@ -21,9 +22,7 @@ func main() { log.Debug("START AGENT>") storage := ms.NewMemStorage() - client := &http.Client{ - Timeout: 5 * time.Second, - } + client := resty.New().SetTimeout(5 * time.Second) go agent.CollectionLoop(storage, pollInterval) go agent.ReportLoop(client, storage, reportInterval) diff --git a/cmd/server/main.go b/cmd/server/main.go index f280ac7..ecb87a4 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -3,6 +3,7 @@ package main import ( "net/http" + "github.com/go-chi/chi/v5" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" @@ -13,12 +14,19 @@ func main() { defer log.Destroy() log.Debug("START SERVER>") + storage := ms.NewMemStorage() + r := chi.NewRouter() - mux := http.NewServeMux() - mux.HandleFunc("/update/", server.MainHandle(storage)) + r.Route("/", func(r chi.Router) { + r.Get("/", server.GetAllMetrics(storage)) + r.Route("/", func(r chi.Router) { + r.Get("/value/{mType}/{mName}", server.GetMetric(storage)) + r.Post("/update/{mType}/{mName}/{mValue}", server.UpdateMetric(storage)) + }) + }) - if err := http.ListenAndServe(`:8080`, mux); err != nil { + if err := http.ListenAndServe(`:8080`, r); err != nil { panic(err) } log.Debug("END SERVER<") diff --git a/go.mod b/go.mod index d6668ea..b72ae39 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,9 @@ module github.com/rAch-kaplin/mipt-golang-course/MetricsService go 1.22.0 -require github.com/stretchr/testify v1.10.0 // indirect +require ( + github.com/go-chi/chi/v5 v5.2.2 // indirect + github.com/go-resty/resty/v2 v2.16.5 // indirect + github.com/stretchr/testify v1.10.0 // indirect + golang.org/x/net v0.33.0 // indirect +) diff --git a/go.sum b/go.sum index 7bfdabe..a6b5d35 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,8 @@ +github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618= +github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= +github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM= +github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 44c4072..d2013f0 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -2,12 +2,13 @@ package agent import ( "fmt" - "io" "math/rand" "net/http" "runtime" "time" + "github.com/go-resty/resty/v2" + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" @@ -173,6 +174,10 @@ func UpdateAllMetrics(storage *ms.MemStorage) { if stat.Type == mtr.GaugeType { metric = mtr.NewGauge(stat.Name, v) } + case uint32: + if stat.Type == mtr.GaugeType { + metric = mtr.NewGauge(stat.Name, float64(v)) + } default: log.Error("ERROR: Unknown type for metric %s: %T", stat.Name, value) continue @@ -187,7 +192,7 @@ func UpdateAllMetrics(storage *ms.MemStorage) { storage.UpdateMetric(mtr.NewGauge("RandomValue", rand.Float64())) } -func sendAllMetrics(client *http.Client, storage *ms.MemStorage) { +func sendAllMetrics(client *resty.Client, storage *ms.MemStorage) { gauges, counters := storage.GetAllMetrics() for name, value := range gauges { @@ -199,26 +204,22 @@ func sendAllMetrics(client *http.Client, storage *ms.MemStorage) { } } -func sendMetric(client *http.Client, mType string, mName string, mValue interface{}) { +func sendMetric(client *resty.Client, mType string, mName string, mValue interface{}) { url := fmt.Sprintf("%s/update/%s/%s/%v", serverAddress, mType, mName, mValue) - req, err := http.NewRequest(http.MethodPost, url, http.NoBody) - if err != nil { - log.Error("Error creating a request for %s: %v\n", mName, err) - return - } - req.Header.Set("Content-Type", "text-plain") + res, err := client.R(). + SetHeader("Content-Type", "text/plain"). + Post(url) - resp, err := client.Do(req) if err != nil { - log.Error("Error sending metric: %s: %v\n", mName, err) + log.Error("Error creating a request for %s: %v", mName, err) return } - defer resp.Body.Close() - _, err = io.Copy(io.Discard, resp.Body) - if err != nil { - log.Error("Failed to read response body from %s: %v", url, err) + if res.StatusCode() != http.StatusOK { + log.Error("Server returned non-OK status for %s/%s: %d %s", mType, mName, res.StatusCode(), res.String()) + } else { + log.Debug("Metric %s/%s sent successfully. Status: %d", mType, mName, res.StatusCode()) } } @@ -230,7 +231,7 @@ func CollectionLoop(storage *ms.MemStorage, interval time.Duration) { } } -func ReportLoop(client *http.Client, storage *ms.MemStorage, interval time.Duration) { +func ReportLoop(client *resty.Client, storage *ms.MemStorage, interval time.Duration) { log.Debug("reportLoop ...") for { time.Sleep(interval) diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index f9510d4..2b62e1f 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -1,16 +1,25 @@ package server import ( + "fmt" + "html/template" "net/http" "strconv" - "strings" + "github.com/go-chi/chi/v5" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" ) -func HandleCounter(res http.ResponseWriter, name, value string) (mtr.Metric, error) { +type MetricTable struct { + Name string + Type string + Value string +} + +// FIXME other function name +func NewCounter(res http.ResponseWriter, name, value string) (mtr.Metric, error) { val, err := strconv.ParseInt(value, 10, 64) if err != nil { http.Error(res, "invalid value metric", http.StatusBadRequest) @@ -20,7 +29,8 @@ func HandleCounter(res http.ResponseWriter, name, value string) (mtr.Metric, err return mtr.NewCounter(name, val), nil } -func HandleGauge(res http.ResponseWriter, name, value string) (mtr.Metric, error) { +// FIXME other function name +func NewGauge(res http.ResponseWriter, name, value string) (mtr.Metric, error) { val, err := strconv.ParseFloat(value, 64) if err != nil { http.Error(res, "invalid value metric", http.StatusBadRequest) @@ -34,56 +44,131 @@ func HandleUnknownMetric(res http.ResponseWriter) { http.Error(res, "unknown type metric!", http.StatusBadRequest) } -func removeEmptyStrings(url []string) []string { - result := make([]string, 0, len(url)) - for _, str := range url { - if str != "" { - result = append(result, str) +func GetMetric(storage ms.Collector) http.HandlerFunc { + return func(res http.ResponseWriter, req *http.Request) { + log.Debug("Incoming GET request: %s %s", req.Method, req.URL.Path) + + mType := chi.URLParam(req, "mType") + mName := chi.URLParam(req, "mName") + log.Debug("Incoming request for metric: Type=%s, Name=%s", mType, mName) + + value, found := storage.GetMetric(mType, mName) + if !found { + http.Error(res, fmt.Sprintf("Metric %s was not found", mName), http.StatusNotFound) + return + } + + var valueStr string + switch v := value.(type) { + case float64: + valueStr = strconv.FormatFloat(v, 'f', -1, 64) + case int64: + valueStr = strconv.FormatInt(v, 10) + default: + http.Error(res, "an unexpected type of metric", http.StatusInternalServerError) + return //FIXME fix this case } - } - return result + res.Header().Set("Content-Type", "text/plain") + res.WriteHeader(http.StatusOK) + res.Write([]byte(valueStr)) + + log.Debug("the metric has been send") + } } -func MainHandle(storage ms.Collector) http.HandlerFunc { +func GetAllMetrics(storage ms.Collector) http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { + gauges, counters := storage.GetAllMetrics() - log.Debug("Incoming request: %s %s", req.Method, req.URL.Path) + var metricsToTable []MetricTable - if req.Method != http.MethodPost { - http.Error(res, "use only POST request", http.StatusMethodNotAllowed) - return + for name, value := range gauges { + metricsToTable = append(metricsToTable, MetricTable{ + Name: name, + Type: mtr.GaugeType, + Value: strconv.FormatFloat(value, 'f', -1, 64), + }) + } + for name, value := range counters { + metricsToTable = append(metricsToTable, MetricTable{ + Name: name, + Type: mtr.CounterType, + Value: strconv.FormatInt(value, 10), + }) } - if req.Header.Get("Content-Type") != "text/plain" { - http.Error(res, "use only text/plain", http.StatusBadRequest) + const htmlTemplate = ` + + + + Metrics + + +

Metric

+ + + + + + + + + + {{range .}} + + + + + + {{end}} + +
Name of MetricTypeValue
{{.Name}}{{.Type}}{{.Value}}
+ + +` + + template, err := template.New("Metrics").Parse(htmlTemplate) + if err != nil { + log.Error("couldn't make it out HTML template: %v", err) + http.Error(res, "Internal server error, failed html-template", http.StatusInternalServerError) return } - parts := strings.Split(strings.Trim(req.URL.Path, "/"), "/") - parts = removeEmptyStrings(parts) + res.Header().Set("Content-Type", "text/html; charset=utf-8") - if len(parts) != 4 || parts[0] != "update" { - http.Error(res, "invalid request", http.StatusNotFound) - return + if err := template.Execute(res, metricsToTable); err != nil { + log.Error("failed complete template: %v", err) } - metricType, name, value := parts[1], parts[2], parts[3] - log.Debug("Parsed metric: type=%s, name=%s, value=%s", metricType, name, value) + log.Debug("the metrics has been send") + res.WriteHeader(http.StatusOK) + } +} + +func UpdateMetric(storage ms.Collector) http.HandlerFunc { + return func(res http.ResponseWriter, req *http.Request) { + log.Debug("Incoming POST request: %s %s", req.Method, req.URL.Path) + + mType := chi.URLParam(req, "mType") + mName := chi.URLParam(req, "mName") + mValue := chi.URLParam(req, "mValue") + log.Debug("Parsed metric: type=%s, name=%s, value=%s", mType, mName, mValue) - if name == "" { - http.Error(res, "metric name is missing", http.StatusNotFound) + if mName == "" { + log.Error("the metric name is not specified") + http.Error(res, "the metric name is not specified", http.StatusBadRequest) return } var metric mtr.Metric var err error - switch metricType { + switch mType { case mtr.GaugeType: - metric, err = HandleGauge(res, name, value) + metric, err = NewGauge(res, mName, mValue) case mtr.CounterType: - metric, err = HandleCounter(res, name, value) + metric, err = NewCounter(res, mName, mValue) default: HandleUnknownMetric(res) return @@ -99,7 +184,7 @@ func MainHandle(storage ms.Collector) http.HandlerFunc { return } - log.Info("Metric updated successfully: %s %s = %s", metricType, name, value) + log.Info("Metric updated successfully: %s %s = %s", mType, mName, mValue) res.WriteHeader(http.StatusOK) } } diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index 1b354b5..453a003 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -1,68 +1,156 @@ package server import ( - "testing" - "net/http/httptest" + "io" "net/http" + "net/http/httptest" + "testing" + "github.com/go-chi/chi/v5" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" + mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" ) -func TestMainHandle(t *testing.T) { +func setupTestRouter(storage ms.Collector) http.Handler { + r := chi.NewRouter() + + r.Route("/", func(r chi.Router) { + r.Get("/", GetAllMetrics(storage)) + r.Route("/", func(r chi.Router) { + r.Get("/value/{mType}/{mName}", GetMetric(storage)) + r.Post("/update/{mType}/{mName}/{mValue}", UpdateMetric(storage)) + }) + }) + return r +} + +func TestUpdateMetric(t *testing.T) { storage := ms.NewMemStorage() - handler := MainHandle(storage) + router := setupTestRouter(storage) tests := []struct { name string - url string method string + url string wantStatus int }{ { - name: "Gauge update (valid)", - url: "/update/gauge/test_metric/97.25", + name: "Valid Gauge Update", method: http.MethodPost, + url: "/update/gauge/testGauge/123.45", wantStatus: http.StatusOK, }, { - name: "Counter update (valid)", - url: "/update/counter/test_counter/90", + name: "Valid Counter Update", method: http.MethodPost, + url: "/update/counter/testCounter/100", wantStatus: http.StatusOK, }, { - name: "Invalid method", - url: "/update/gauge/test_metric/97.45", - method: http.MethodGet, - wantStatus: http.StatusMethodNotAllowed, + name: "Update Counter with another value", + method: http.MethodPost, + url: "/update/counter/testCounter/50", + wantStatus: http.StatusOK, }, { - name: "Invalid metric type", - url: "/update/invalid/test_metric/80", + name: "Invalid Gauge Value", method: http.MethodPost, + url: "/update/gauge/invalidGauge/abc", wantStatus: http.StatusBadRequest, }, { - name: "Missing value", - url: "/update/gauge/test_metric/", + name: "Invalid Counter Value", method: http.MethodPost, - wantStatus: http.StatusNotFound, + url: "/update/counter/invalidCounter/xyz", + wantStatus: http.StatusBadRequest, + }, + { + name: "Unknown Metric Type", + method: http.MethodPost, + url: "/update/unknown/testMetric/123", + wantStatus: http.StatusBadRequest, + }, + { + name: "Missing Metric Name", + method: http.MethodPost, + url: "/update/gauge//123.45", + wantStatus: http.StatusBadRequest, }, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - req := httptest.NewRequest(test.method, test.url, nil) - req.Header.Set("Content-Type", "text/plain") - w := httptest.NewRecorder() - handler(w, req) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(tt.method, tt.url, nil) + rr := httptest.NewRecorder() + + router.ServeHTTP(rr, req) - res := w.Result() - defer res.Body.Close() - if res.StatusCode != test.wantStatus { - t.Errorf("expected status %d, got %d", test.wantStatus, res.StatusCode) + if rr.Code != tt.wantStatus { + t.Errorf("Test %s: expected status %d, got %d", tt.name, tt.wantStatus, rr.Code) } }) } +} +func TestGetMetric(t *testing.T) { + storage := ms.NewMemStorage() + + storage.UpdateMetric(mtr.NewGauge("cpu_usage", 75.5)) + storage.UpdateMetric(mtr.NewCounter("requests_total", 100)) + router := setupTestRouter(storage) + + tests := []struct { + name string + url string + wantStatus int + wantBody string + }{ + { + name: "Get Existing Gauge", + url: "/value/gauge/cpu_usage", + wantStatus: http.StatusOK, + wantBody: "75.5", + }, + { + name: "Get Existing Counter", + url: "/value/counter/requests_total", + wantStatus: http.StatusOK, + wantBody: "100", + }, + { + name: "Get Non-Existing Metric", + url: "/value/gauge/non_existent", + wantStatus: http.StatusNotFound, + wantBody: "Metric non_existent was not found\n", + }, + { + name: "Get Unknown Metric Type", + url: "/value/invalid_type/some_metric", + wantStatus: http.StatusNotFound, + wantBody: "Metric some_metric was not found\n", + }, + { + name: "GET with missing name", + url: "/value/gauge/", + wantStatus: http.StatusNotFound, + wantBody: "404 page not found\n", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, tt.url, nil) + rr := httptest.NewRecorder() + + router.ServeHTTP(rr, req) + + if rr.Code != tt.wantStatus { + t.Errorf("Test %s: expected status %d, got %d", tt.name, tt.wantStatus, rr.Code) + } + body, _ := io.ReadAll(rr.Body) + if string(body) != tt.wantBody { + t.Errorf("Test %s: expected body %q, got %q", tt.name, tt.wantBody, string(body)) + } + }) + } } diff --git a/internal/memstorage/storage.go b/internal/memstorage/storage.go index eb618cc..2afb471 100644 --- a/internal/memstorage/storage.go +++ b/internal/memstorage/storage.go @@ -7,7 +7,8 @@ import ( ) type Collector interface { - GetMetric(mtr metrics.Metric) (interface{}, error) + GetMetric(mType, mName string) (interface{}, bool) + GetAllMetrics() (map[string]float64, map[string]int64) UpdateMetric(mtr metrics.Metric) error } @@ -64,45 +65,26 @@ func (ms *MemStorage) UpdateCounter(metric metrics.Metric, value int64) { ms.Counters[metric.Name()] += value } -func (ms *MemStorage) GetMetric(metric metrics.Metric) (interface{}, error) { - switch metric.Type() { - case metrics.CounterType: +func (ms *MemStorage) GetMetric(mType, mName string) (interface{}, bool) { + ms.mutex.RLock() + defer ms.mutex.RUnlock() + + switch mType { + case metrics.GaugeType: { - val, ok := ms.GetCounter(metric.Name()) - if !ok { - return nil, metrics.ErrMetricsNotFound + if val, ok := ms.Gauges[mName]; ok { + return val, true } - - return val, nil } - case metrics.GaugeType: + case metrics.CounterType: { - val, ok := ms.GetGauges(metric.Name()) - if !ok { - return nil, metrics.ErrMetricsNotFound + if val, ok := ms.Counters[mName]; ok { + return val, true } - - return val, nil } - default: - return nil, metrics.ErrInvalidMetricsType } -} - -func (ms *MemStorage) GetGauges(name string) (float64, bool) { - ms.mutex.RLock() - defer ms.mutex.RUnlock() - - val, ok := ms.Gauges[name] - return val, ok -} - -func (ms *MemStorage) GetCounter(name string) (int64, bool) { - ms.mutex.RLock() - defer ms.mutex.RUnlock() - val, ok := ms.Counters[name] - return val, ok + return nil, false } func (ms *MemStorage) GetAllMetrics() (map[string]float64, map[string]int64) { From e903bffc808e861115f5275fe8e7b71063a59e5b Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Thu, 3 Jul 2025 14:05:19 +0300 Subject: [PATCH 019/126] added a command line flag handler --- cmd/agent/main.go | 64 +++++++++++++++++++++++++++++++++++++++++++--- cmd/server/main.go | 21 +++++++++++++-- 2 files changed, 79 insertions(+), 6 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 626824f..d213152 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -1,6 +1,11 @@ package main import ( + "flag" + "fmt" + "os" + "strconv" + "strings" "time" "github.com/go-resty/resty/v2" @@ -11,10 +16,58 @@ import ( ) const ( - pollInterval = 2 * time.Second - reportInterval = 10 * time.Second + defaultEndpoint = "localhost:8080" + defaultPollInterval = 2 + defaultReportInterval = 10 ) +type options struct { + endPointAddr string + pollInterval uint + reportInterval uint +} + +func flagsInit(opts *options) { + flag.StringVar(&opts.endPointAddr, "a", defaultEndpoint, "endpoint HTTP-server addr") + flag.UintVar(&opts.pollInterval, "p", defaultPollInterval, "PollInterval value") + flag.UintVar(&opts.reportInterval, "r", defaultReportInterval, "ReportInterval value") + + flag.Parse() + + if opts.pollInterval == 0 || opts.reportInterval == 0 { + fmt.Println("Error: poll interval and report interval must be greater than 0") + flag.Usage() + os.Exit(1) + } + + if err := validateEndpoint(opts.endPointAddr); err != nil { + fmt.Printf("Error in endpoint address: %v\n", err) + flag.Usage() + os.Exit(1) + } +} + +func validateEndpoint(addr string) error { + parts := strings.Split(addr, ":") + + if len(parts) != 2 { + return fmt.Errorf("address must be in format 'host:port'") + } + + if parts[0] == "" { + return fmt.Errorf("host cannot be empty") + } + + port, err := strconv.Atoi(parts[1]) + if err != nil || port <= 0 { + fmt.Printf("Error: Port must be >0 number\n") + flag.Usage() + os.Exit(1) + } + + return nil +} + func main() { log.Init(log.DebugLevel, "logFile.log") defer log.Destroy() @@ -22,10 +75,13 @@ func main() { log.Debug("START AGENT>") storage := ms.NewMemStorage() + var opts options + flagsInit(&opts) + client := resty.New().SetTimeout(5 * time.Second) - go agent.CollectionLoop(storage, pollInterval) - go agent.ReportLoop(client, storage, reportInterval) + go agent.CollectionLoop(storage, time.Duration(opts.pollInterval)*time.Second) + go agent.ReportLoop(client, storage, time.Duration(opts.reportInterval)*time.Second) log.Debug("END AGENT<") select {} diff --git a/cmd/server/main.go b/cmd/server/main.go index ecb87a4..225fa2a 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -1,18 +1,35 @@ package main import ( + "flag" "net/http" "github.com/go-chi/chi/v5" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" ) +const ( + defaultEndpoint = "localhost:8080" +) + +type options struct { + endPointAddr string +} + +func flagsInit(opts *options) { + flag.StringVar(&opts.endPointAddr, "a", defaultEndpoint, "endpoint HTTP-server address") + flag.Parse() +} + func main() { log.Init(log.DebugLevel, "logFile.log") defer log.Destroy() + var opts options + flagsInit(&opts) + log.Debug("START SERVER>") storage := ms.NewMemStorage() @@ -26,7 +43,7 @@ func main() { }) }) - if err := http.ListenAndServe(`:8080`, r); err != nil { + if err := http.ListenAndServe(opts.endPointAddr, r); err != nil { panic(err) } log.Debug("END SERVER<") From 559c8114d8932bc45627b4925a5ac0a4286bd4e2 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Thu, 3 Jul 2025 14:22:57 +0300 Subject: [PATCH 020/126] fixed the independence of the server and agent --- cmd/agent/main.go | 6 ++++-- internal/handlers/agent/agent.go | 18 +++++++----------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index d213152..9ef6e6f 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -78,10 +78,12 @@ func main() { var opts options flagsInit(&opts) - client := resty.New().SetTimeout(5 * time.Second) + client := resty.New(). + SetTimeout(5 * time.Second). + SetBaseURL("http://" + opts.endPointAddr) go agent.CollectionLoop(storage, time.Duration(opts.pollInterval)*time.Second) - go agent.ReportLoop(client, storage, time.Duration(opts.reportInterval)*time.Second) + go agent.ReportLoop(client, opts.endPointAddr, storage, time.Duration(opts.reportInterval)*time.Second) log.Debug("END AGENT<") select {} diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index d2013f0..8d877e3 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -14,10 +14,6 @@ import ( log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" ) -const ( - serverAddress = "http://localhost:8080" -) - type MemRuntimeStat struct { Name string Type string @@ -192,20 +188,20 @@ func UpdateAllMetrics(storage *ms.MemStorage) { storage.UpdateMetric(mtr.NewGauge("RandomValue", rand.Float64())) } -func sendAllMetrics(client *resty.Client, storage *ms.MemStorage) { +func sendAllMetrics(client *resty.Client, endPointAddr string, storage *ms.MemStorage) { gauges, counters := storage.GetAllMetrics() for name, value := range gauges { - sendMetric(client, mtr.GaugeType, name, value) + sendMetric(client, endPointAddr, mtr.GaugeType, name, value) } for name, value := range counters { - sendMetric(client, mtr.CounterType, name, value) + sendMetric(client, endPointAddr, mtr.CounterType, name, value) } } -func sendMetric(client *resty.Client, mType string, mName string, mValue interface{}) { - url := fmt.Sprintf("%s/update/%s/%s/%v", serverAddress, mType, mName, mValue) +func sendMetric(client *resty.Client, endPointAddr string, mType string, mName string, mValue interface{}) { + url := fmt.Sprintf("%s/update/%s/%s/%v", endPointAddr, mType, mName, mValue) res, err := client.R(). SetHeader("Content-Type", "text/plain"). @@ -231,10 +227,10 @@ func CollectionLoop(storage *ms.MemStorage, interval time.Duration) { } } -func ReportLoop(client *resty.Client, storage *ms.MemStorage, interval time.Duration) { +func ReportLoop(client *resty.Client, endPointAddr string, storage *ms.MemStorage, interval time.Duration) { log.Debug("reportLoop ...") for { time.Sleep(interval) - sendAllMetrics(client, storage) + sendAllMetrics(client, endPointAddr, storage) } } From 008c208c182c543ad69aaa4bb96b7bb5d716db85 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Thu, 3 Jul 2025 14:40:47 +0300 Subject: [PATCH 021/126] fixed test 2A --- .github/workflows/metricstest.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/metricstest.yml b/.github/workflows/metricstest.yml index 462c49c..d35961a 100644 --- a/.github/workflows/metricstest.yml +++ b/.github/workflows/metricstest.yml @@ -127,7 +127,8 @@ jobs: run: | metricstest -test.v -test.run=^TestIteration2[AB]*$ \ -source-path=. \ - -agent-binary-path=cmd/agent/agent + -agent-binary-path=cmd/agent/agent \ + -binary-path=cmd/server/server - name: "Code increment #3" if: | From 0647cfda5975d5a9104d5fbaea904ba4f5a161d3 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Thu, 3 Jul 2025 14:51:22 +0300 Subject: [PATCH 022/126] fixed POST request url --- cmd/agent/main.go | 2 +- internal/handlers/agent/agent.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 9ef6e6f..fd00d2f 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -83,7 +83,7 @@ func main() { SetBaseURL("http://" + opts.endPointAddr) go agent.CollectionLoop(storage, time.Duration(opts.pollInterval)*time.Second) - go agent.ReportLoop(client, opts.endPointAddr, storage, time.Duration(opts.reportInterval)*time.Second) + go agent.ReportLoop(client, storage, time.Duration(opts.reportInterval)*time.Second) log.Debug("END AGENT<") select {} diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 8d877e3..6bf7a06 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -188,20 +188,20 @@ func UpdateAllMetrics(storage *ms.MemStorage) { storage.UpdateMetric(mtr.NewGauge("RandomValue", rand.Float64())) } -func sendAllMetrics(client *resty.Client, endPointAddr string, storage *ms.MemStorage) { +func sendAllMetrics(client *resty.Client, storage *ms.MemStorage) { gauges, counters := storage.GetAllMetrics() for name, value := range gauges { - sendMetric(client, endPointAddr, mtr.GaugeType, name, value) + sendMetric(client, mtr.GaugeType, name, value) } for name, value := range counters { - sendMetric(client, endPointAddr, mtr.CounterType, name, value) + sendMetric(client, mtr.CounterType, name, value) } } -func sendMetric(client *resty.Client, endPointAddr string, mType string, mName string, mValue interface{}) { - url := fmt.Sprintf("%s/update/%s/%s/%v", endPointAddr, mType, mName, mValue) +func sendMetric(client *resty.Client, mType string, mName string, mValue interface{}) { + url := fmt.Sprintf("update/%s/%s/%v", mType, mName, mValue) res, err := client.R(). SetHeader("Content-Type", "text/plain"). @@ -227,10 +227,10 @@ func CollectionLoop(storage *ms.MemStorage, interval time.Duration) { } } -func ReportLoop(client *resty.Client, endPointAddr string, storage *ms.MemStorage, interval time.Duration) { +func ReportLoop(client *resty.Client, storage *ms.MemStorage, interval time.Duration) { log.Debug("reportLoop ...") for { time.Sleep(interval) - sendAllMetrics(client, endPointAddr, storage) + sendAllMetrics(client, storage) } } From c1076df02562a2e2602e24e640a0f6fe8c5b7cd9 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Thu, 3 Jul 2025 19:35:21 +0300 Subject: [PATCH 023/126] first version env --- cmd/agent/main.go | 51 ++++++++++++++++++++++++++++++++------- cmd/server/main.go | 60 +++++++++++++++++++++++++++++++++++++++++++--- go.mod | 1 + go.sum | 2 ++ 4 files changed, 102 insertions(+), 12 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index fd00d2f..0a24667 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/caarlos0/env/v6" "github.com/go-resty/resty/v2" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/agent" @@ -23,19 +24,48 @@ const ( type options struct { endPointAddr string - pollInterval uint - reportInterval uint + pollInterval int + reportInterval int } -func flagsInit(opts *options) { +type envConfig struct { + endPointAddr string `env:"ADDRESS"` + pollInterval int `env:"POLL_INTERVAL"` + reportInterval int `env:"REPORT_INTERVAL"` +} + +func envAndFlagsInit() *options { + var cfg envConfig + err := env.Parse(&cfg) + if err != nil { + fmt.Println("environment variables parsing error") + os.Exit(1) + } + + opts := &options{ + endPointAddr: defaultEndpoint, + pollInterval: defaultPollInterval, + reportInterval: defaultReportInterval, + } + + if cfg.endPointAddr != "" { + opts.endPointAddr = cfg.endPointAddr + } + if cfg.pollInterval > 0 { + opts.pollInterval = cfg.pollInterval + } + if cfg.reportInterval > 0 { + opts.reportInterval = cfg.reportInterval + } + flag.StringVar(&opts.endPointAddr, "a", defaultEndpoint, "endpoint HTTP-server addr") - flag.UintVar(&opts.pollInterval, "p", defaultPollInterval, "PollInterval value") - flag.UintVar(&opts.reportInterval, "r", defaultReportInterval, "ReportInterval value") + flag.IntVar(&opts.pollInterval, "p", defaultPollInterval, "PollInterval value") + flag.IntVar(&opts.reportInterval, "r", defaultReportInterval, "ReportInterval value") flag.Parse() - if opts.pollInterval == 0 || opts.reportInterval == 0 { - fmt.Println("Error: poll interval and report interval must be greater than 0") + if opts.pollInterval <= 0 || opts.reportInterval <= 0 { + fmt.Println("Error: poll interval and report interval must be > 0") flag.Usage() os.Exit(1) } @@ -45,6 +75,8 @@ func flagsInit(opts *options) { flag.Usage() os.Exit(1) } + + return opts } func validateEndpoint(addr string) error { @@ -75,8 +107,9 @@ func main() { log.Debug("START AGENT>") storage := ms.NewMemStorage() - var opts options - flagsInit(&opts) + opts := envAndFlagsInit() + log.Debug("Configuration: endPointAddr=%s, pollInterval=%ds, reportInterval=%ds", + opts.endPointAddr, opts.pollInterval, opts.reportInterval) client := resty.New(). SetTimeout(5 * time.Second). diff --git a/cmd/server/main.go b/cmd/server/main.go index 225fa2a..5d8a29d 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -2,9 +2,15 @@ package main import ( "flag" + "fmt" "net/http" + "os" + "strconv" + "strings" + "github.com/caarlos0/env/v6" "github.com/go-chi/chi/v5" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" @@ -18,17 +24,65 @@ type options struct { endPointAddr string } -func flagsInit(opts *options) { +type envConfig struct { + endPointAddr string `env:"ADDRESS"` +} + +func envAndFlagsInit() *options { + var cfg envConfig + err := env.Parse(&cfg) + if err != nil { + fmt.Println("environment variable parsing error") + os.Exit(1) + } + + opts := &options{ + endPointAddr: defaultEndpoint, + } + + if cfg.endPointAddr != "" { + opts.endPointAddr = cfg.endPointAddr + } + flag.StringVar(&opts.endPointAddr, "a", defaultEndpoint, "endpoint HTTP-server address") flag.Parse() + + if err := validateEndpoint(opts.endPointAddr); err != nil { + fmt.Printf("Error in endpoint address: %v\n", err) + flag.Usage() + os.Exit(1) + } + + return opts +} + +func validateEndpoint(addr string) error { + parts := strings.Split(addr, ":") + + if len(parts) != 2 { + return fmt.Errorf("address must be in format 'host:port'") + } + + if parts[0] == "" { + return fmt.Errorf("host cannot be empty") + } + + port, err := strconv.Atoi(parts[1]) + if err != nil || port <= 0 { + fmt.Printf("Error: Port must be >0 number\n") + flag.Usage() + os.Exit(1) + } + + return nil } func main() { log.Init(log.DebugLevel, "logFile.log") defer log.Destroy() - var opts options - flagsInit(&opts) + opts := envAndFlagsInit() + log.Debug("Server configuration: Address=%s", opts.endPointAddr) log.Debug("START SERVER>") diff --git a/go.mod b/go.mod index b72ae39..bfcbf75 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/rAch-kaplin/mipt-golang-course/MetricsService go 1.22.0 require ( + github.com/caarlos0/env/v6 v6.10.1 // indirect github.com/go-chi/chi/v5 v5.2.2 // indirect github.com/go-resty/resty/v2 v2.16.5 // indirect github.com/stretchr/testify v1.10.0 // indirect diff --git a/go.sum b/go.sum index a6b5d35..f91a1b1 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/caarlos0/env/v6 v6.10.1 h1:t1mPSxNpei6M5yAeu1qtRdPAK29Nbcf/n3G7x+b3/II= +github.com/caarlos0/env/v6 v6.10.1/go.mod h1:hvp/ryKXKipEkcuYjs9mI4bBCg+UI0Yhgm5Zu0ddvwc= github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618= github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM= From 0a4e3477fd94baaa44e5551f3d94ce0b9e276624 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Thu, 3 Jul 2025 20:00:16 +0300 Subject: [PATCH 024/126] fixed defaults values --- cmd/agent/main.go | 6 +++--- cmd/server/main.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 0a24667..4fa861d 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -58,9 +58,9 @@ func envAndFlagsInit() *options { opts.reportInterval = cfg.reportInterval } - flag.StringVar(&opts.endPointAddr, "a", defaultEndpoint, "endpoint HTTP-server addr") - flag.IntVar(&opts.pollInterval, "p", defaultPollInterval, "PollInterval value") - flag.IntVar(&opts.reportInterval, "r", defaultReportInterval, "ReportInterval value") + flag.StringVar(&opts.endPointAddr, "a", opts.endPointAddr, "endpoint HTTP-server addr") + flag.IntVar(&opts.pollInterval, "p", opts.pollInterval, "PollInterval value") + flag.IntVar(&opts.reportInterval, "r", opts.reportInterval, "ReportInterval value") flag.Parse() diff --git a/cmd/server/main.go b/cmd/server/main.go index 5d8a29d..14f9585 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -44,7 +44,7 @@ func envAndFlagsInit() *options { opts.endPointAddr = cfg.endPointAddr } - flag.StringVar(&opts.endPointAddr, "a", defaultEndpoint, "endpoint HTTP-server address") + flag.StringVar(&opts.endPointAddr, "a", opts.endPointAddr, "endpoint HTTP-server address") flag.Parse() if err := validateEndpoint(opts.endPointAddr); err != nil { From b128e7f6f40dfb1ed27a0d1f1b767f106ed006bc Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Thu, 3 Jul 2025 21:50:33 +0300 Subject: [PATCH 025/126] mada variables exportable for env --- .gitignore | 2 ++ cmd/agent/main.go | 42 +++++++++++++++++++++--------------------- cmd/server/main.go | 27 ++++++++++++++++----------- go.mod | 10 +++++----- go.sum | 4 ++-- 5 files changed, 46 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index 995dbda..c938849 100644 --- a/.gitignore +++ b/.gitignore @@ -31,5 +31,7 @@ go.work.sum .idea .vscode logFile.log +logFileServer.log +logFileAgent.log cmd/server/server cmd/agent/agent diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 4fa861d..0e681f1 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -29,41 +29,41 @@ type options struct { } type envConfig struct { - endPointAddr string `env:"ADDRESS"` - pollInterval int `env:"POLL_INTERVAL"` - reportInterval int `env:"REPORT_INTERVAL"` + EndPointAddr string `env:"ADDRESS"` + PollInterval int `env:"POLL_INTERVAL"` + ReportInterval int `env:"REPORT_INTERVAL"` } func envAndFlagsInit() *options { - var cfg envConfig - err := env.Parse(&cfg) - if err != nil { - fmt.Println("environment variables parsing error") - os.Exit(1) - } - opts := &options{ endPointAddr: defaultEndpoint, pollInterval: defaultPollInterval, reportInterval: defaultReportInterval, } - if cfg.endPointAddr != "" { - opts.endPointAddr = cfg.endPointAddr - } - if cfg.pollInterval > 0 { - opts.pollInterval = cfg.pollInterval - } - if cfg.reportInterval > 0 { - opts.reportInterval = cfg.reportInterval - } - flag.StringVar(&opts.endPointAddr, "a", opts.endPointAddr, "endpoint HTTP-server addr") flag.IntVar(&opts.pollInterval, "p", opts.pollInterval, "PollInterval value") flag.IntVar(&opts.reportInterval, "r", opts.reportInterval, "ReportInterval value") flag.Parse() + var cfg envConfig + err := env.Parse(&cfg) + if err != nil { + fmt.Println("environment variables parsing error") + os.Exit(1) + } + + if cfg.EndPointAddr != "" { + opts.endPointAddr = cfg.EndPointAddr + } + if cfg.PollInterval > 0 { + opts.pollInterval = cfg.PollInterval + } + if cfg.ReportInterval > 0 { + opts.reportInterval = cfg.ReportInterval + } + if opts.pollInterval <= 0 || opts.reportInterval <= 0 { fmt.Println("Error: poll interval and report interval must be > 0") flag.Usage() @@ -101,7 +101,7 @@ func validateEndpoint(addr string) error { } func main() { - log.Init(log.DebugLevel, "logFile.log") + log.Init(log.DebugLevel, "logFileAgent.log") defer log.Destroy() log.Debug("START AGENT>") diff --git a/cmd/server/main.go b/cmd/server/main.go index 14f9585..e3fa4d1 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -25,10 +25,17 @@ type options struct { } type envConfig struct { - endPointAddr string `env:"ADDRESS"` + EndPointAddr string `env:"ADDRESS"` } func envAndFlagsInit() *options { + opts := &options{ + endPointAddr: defaultEndpoint, + } + + flag.StringVar(&opts.endPointAddr, "a", opts.endPointAddr, "endpoint HTTP-server address") + flag.Parse() + var cfg envConfig err := env.Parse(&cfg) if err != nil { @@ -36,17 +43,10 @@ func envAndFlagsInit() *options { os.Exit(1) } - opts := &options{ - endPointAddr: defaultEndpoint, - } - - if cfg.endPointAddr != "" { - opts.endPointAddr = cfg.endPointAddr + if cfg.EndPointAddr != "" { + opts.endPointAddr = cfg.EndPointAddr } - flag.StringVar(&opts.endPointAddr, "a", opts.endPointAddr, "endpoint HTTP-server address") - flag.Parse() - if err := validateEndpoint(opts.endPointAddr); err != nil { fmt.Printf("Error in endpoint address: %v\n", err) flag.Usage() @@ -78,13 +78,15 @@ func validateEndpoint(addr string) error { } func main() { - log.Init(log.DebugLevel, "logFile.log") + log.Init(log.DebugLevel, "logFileServer.log") defer log.Destroy() opts := envAndFlagsInit() log.Debug("Server configuration: Address=%s", opts.endPointAddr) log.Debug("START SERVER>") + fmt.Printf("Endpoint: [%s]\n", opts.endPointAddr) + storage := ms.NewMemStorage() r := chi.NewRouter() @@ -98,7 +100,10 @@ func main() { }) if err := http.ListenAndServe(opts.endPointAddr, r); err != nil { + log.Error("HTTP-server didn't start: %v", err) panic(err) } + log.Debug("ListenAndServe returned") + log.Debug("END SERVER<") } diff --git a/go.mod b/go.mod index bfcbf75..5b2025e 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module github.com/rAch-kaplin/mipt-golang-course/MetricsService go 1.22.0 require ( - github.com/caarlos0/env/v6 v6.10.1 // indirect - github.com/go-chi/chi/v5 v5.2.2 // indirect - github.com/go-resty/resty/v2 v2.16.5 // indirect - github.com/stretchr/testify v1.10.0 // indirect - golang.org/x/net v0.33.0 // indirect + github.com/caarlos0/env/v6 v6.10.1 + github.com/go-chi/chi/v5 v5.2.2 + github.com/go-resty/resty/v2 v2.16.5 ) + +require golang.org/x/net v0.33.0 // indirect diff --git a/go.sum b/go.sum index f91a1b1..86ebad3 100644 --- a/go.sum +++ b/go.sum @@ -4,7 +4,7 @@ github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618= github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM= github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= From 0a26babcd6b34a6a734747f7bc63791fc5329d3f Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Fri, 4 Jul 2025 21:36:19 +0300 Subject: [PATCH 026/126] made some changes after the review --- Makefile | 6 ++ cmd/agent/cfg_agent.go | 73 ++++++++++++++++++++++++ cmd/agent/main.go | 92 +----------------------------- cmd/server/cfg_server.go | 50 ++++++++++++++++ cmd/server/main.go | 69 +--------------------- go.mod | 7 ++- go.sum | 10 ++++ internal/handlers/agent/agent.go | 17 +++--- internal/handlers/server/server.go | 2 +- internal/memstorage/storage.go | 12 +--- {logger => pkg/logger}/logger.go | 0 11 files changed, 161 insertions(+), 177 deletions(-) create mode 100644 cmd/agent/cfg_agent.go create mode 100644 cmd/server/cfg_server.go rename {logger => pkg/logger}/logger.go (100%) diff --git a/Makefile b/Makefile index d20f0aa..c9bb312 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,12 @@ agent: @go build -o $(AGENT_FULL_PATH) $(AGENT_SRC_DIR) @echo "Built $(AGENT_FULL_PATH)" +test: + @go test ./... -v + +lint: + @golangci-lint run + clean: @rm -f $(SERVER_FULL_PATH) $(AGENT_FULL_PATH) @echo "Cleaned." diff --git a/cmd/agent/cfg_agent.go b/cmd/agent/cfg_agent.go new file mode 100644 index 0000000..c7021ff --- /dev/null +++ b/cmd/agent/cfg_agent.go @@ -0,0 +1,73 @@ +package main + +import ( + "flag" + "fmt" + "net" + "os" + + "github.com/caarlos0/env/v6" +) + +const ( + defaultEndpoint = "localhost:8080" + defaultPollInterval = 2 + defaultReportInterval = 10 +) + +type options struct { + endPointAddr string + pollInterval int + reportInterval int +} + +type envConfig struct { + EndPointAddr string `env:"ADDRESS"` + PollInterval int `env:"POLL_INTERVAL"` + ReportInterval int `env:"REPORT_INTERVAL"` +} + +func envAndFlagsInit() *options { + opts := &options{ + endPointAddr: defaultEndpoint, + pollInterval: defaultPollInterval, + reportInterval: defaultReportInterval, + } + + flag.StringVar(&opts.endPointAddr, "a", opts.endPointAddr, "endpoint HTTP-server addr") + flag.IntVar(&opts.pollInterval, "p", opts.pollInterval, "PollInterval value") + flag.IntVar(&opts.reportInterval, "r", opts.reportInterval, "ReportInterval value") + + flag.Parse() + + var cfg envConfig + err := env.Parse(&cfg) + if err != nil { + fmt.Println("environment variables parsing error") + os.Exit(1) + } + + if cfg.EndPointAddr != "" { + opts.endPointAddr = cfg.EndPointAddr + } + if cfg.PollInterval > 0 { + opts.pollInterval = cfg.PollInterval + } + if cfg.ReportInterval > 0 { + opts.reportInterval = cfg.ReportInterval + } + + if opts.pollInterval <= 0 || opts.reportInterval <= 0 { + fmt.Println("Error: poll interval and report interval must be > 0") + flag.Usage() + os.Exit(1) + } + + if _, _, err := net.SplitHostPort(opts.endPointAddr); err != nil { + fmt.Errorf("invalid address %s: %w", opts.endPointAddr, err) + flag.Usage() + os.Exit(1) + } + + return opts +} diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 0e681f1..52fb7e0 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -1,105 +1,15 @@ package main import ( - "flag" - "fmt" - "os" - "strconv" - "strings" "time" - "github.com/caarlos0/env/v6" "github.com/go-resty/resty/v2" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/agent" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" - log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" + log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) -const ( - defaultEndpoint = "localhost:8080" - defaultPollInterval = 2 - defaultReportInterval = 10 -) - -type options struct { - endPointAddr string - pollInterval int - reportInterval int -} - -type envConfig struct { - EndPointAddr string `env:"ADDRESS"` - PollInterval int `env:"POLL_INTERVAL"` - ReportInterval int `env:"REPORT_INTERVAL"` -} - -func envAndFlagsInit() *options { - opts := &options{ - endPointAddr: defaultEndpoint, - pollInterval: defaultPollInterval, - reportInterval: defaultReportInterval, - } - - flag.StringVar(&opts.endPointAddr, "a", opts.endPointAddr, "endpoint HTTP-server addr") - flag.IntVar(&opts.pollInterval, "p", opts.pollInterval, "PollInterval value") - flag.IntVar(&opts.reportInterval, "r", opts.reportInterval, "ReportInterval value") - - flag.Parse() - - var cfg envConfig - err := env.Parse(&cfg) - if err != nil { - fmt.Println("environment variables parsing error") - os.Exit(1) - } - - if cfg.EndPointAddr != "" { - opts.endPointAddr = cfg.EndPointAddr - } - if cfg.PollInterval > 0 { - opts.pollInterval = cfg.PollInterval - } - if cfg.ReportInterval > 0 { - opts.reportInterval = cfg.ReportInterval - } - - if opts.pollInterval <= 0 || opts.reportInterval <= 0 { - fmt.Println("Error: poll interval and report interval must be > 0") - flag.Usage() - os.Exit(1) - } - - if err := validateEndpoint(opts.endPointAddr); err != nil { - fmt.Printf("Error in endpoint address: %v\n", err) - flag.Usage() - os.Exit(1) - } - - return opts -} - -func validateEndpoint(addr string) error { - parts := strings.Split(addr, ":") - - if len(parts) != 2 { - return fmt.Errorf("address must be in format 'host:port'") - } - - if parts[0] == "" { - return fmt.Errorf("host cannot be empty") - } - - port, err := strconv.Atoi(parts[1]) - if err != nil || port <= 0 { - fmt.Printf("Error: Port must be >0 number\n") - flag.Usage() - os.Exit(1) - } - - return nil -} - func main() { log.Init(log.DebugLevel, "logFileAgent.log") defer log.Destroy() diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go new file mode 100644 index 0000000..91df44e --- /dev/null +++ b/cmd/server/cfg_server.go @@ -0,0 +1,50 @@ +package main + +import ( + "flag" + "fmt" + "net" + "os" + + "github.com/caarlos0/env/v6" +) + +const ( + defaultEndpoint = "localhost:8080" +) + +type options struct { + endPointAddr string +} + +type envConfig struct { + EndPointAddr string `env:"ADDRESS"` +} + +func envAndFlagsInit() *options { + opts := &options{ + endPointAddr: defaultEndpoint, + } + + flag.StringVar(&opts.endPointAddr, "a", opts.endPointAddr, "endpoint HTTP-server address") + flag.Parse() + + var cfg envConfig + err := env.Parse(&cfg) + if err != nil { + fmt.Println("environment variable parsing error") + os.Exit(1) + } + + if cfg.EndPointAddr != "" { + opts.endPointAddr = cfg.EndPointAddr + } + + if _, _, err := net.SplitHostPort(opts.endPointAddr); err != nil { + fmt.Errorf("invalid address %s: %w", opts.endPointAddr, err) + flag.Usage() + os.Exit(1) + } + + return opts +} diff --git a/cmd/server/main.go b/cmd/server/main.go index e3fa4d1..51b1719 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -1,82 +1,16 @@ package main import ( - "flag" "fmt" "net/http" - "os" - "strconv" - "strings" - "github.com/caarlos0/env/v6" "github.com/go-chi/chi/v5" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" - log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" + log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) -const ( - defaultEndpoint = "localhost:8080" -) - -type options struct { - endPointAddr string -} - -type envConfig struct { - EndPointAddr string `env:"ADDRESS"` -} - -func envAndFlagsInit() *options { - opts := &options{ - endPointAddr: defaultEndpoint, - } - - flag.StringVar(&opts.endPointAddr, "a", opts.endPointAddr, "endpoint HTTP-server address") - flag.Parse() - - var cfg envConfig - err := env.Parse(&cfg) - if err != nil { - fmt.Println("environment variable parsing error") - os.Exit(1) - } - - if cfg.EndPointAddr != "" { - opts.endPointAddr = cfg.EndPointAddr - } - - if err := validateEndpoint(opts.endPointAddr); err != nil { - fmt.Printf("Error in endpoint address: %v\n", err) - flag.Usage() - os.Exit(1) - } - - return opts -} - -func validateEndpoint(addr string) error { - parts := strings.Split(addr, ":") - - if len(parts) != 2 { - return fmt.Errorf("address must be in format 'host:port'") - } - - if parts[0] == "" { - return fmt.Errorf("host cannot be empty") - } - - port, err := strconv.Atoi(parts[1]) - if err != nil || port <= 0 { - fmt.Printf("Error: Port must be >0 number\n") - flag.Usage() - os.Exit(1) - } - - return nil -} - func main() { log.Init(log.DebugLevel, "logFileServer.log") defer log.Destroy() @@ -87,7 +21,6 @@ func main() { log.Debug("START SERVER>") fmt.Printf("Endpoint: [%s]\n", opts.endPointAddr) - storage := ms.NewMemStorage() r := chi.NewRouter() diff --git a/go.mod b/go.mod index 5b2025e..c903c69 100644 --- a/go.mod +++ b/go.mod @@ -8,4 +8,9 @@ require ( github.com/go-resty/resty/v2 v2.16.5 ) -require golang.org/x/net v0.33.0 // indirect +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/cobra v1.9.1 // indirect + github.com/spf13/pflag v1.0.6 // indirect + golang.org/x/net v0.33.0 // indirect +) diff --git a/go.sum b/go.sum index 86ebad3..8a67247 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,20 @@ github.com/caarlos0/env/v6 v6.10.1 h1:t1mPSxNpei6M5yAeu1qtRdPAK29Nbcf/n3G7x+b3/II= github.com/caarlos0/env/v6 v6.10.1/go.mod h1:hvp/ryKXKipEkcuYjs9mI4bBCg+UI0Yhgm5Zu0ddvwc= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618= github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM= github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 6bf7a06..e0efeb9 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -11,7 +11,7 @@ import ( ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" - log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" + log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) type MemRuntimeStat struct { @@ -171,9 +171,9 @@ func UpdateAllMetrics(storage *ms.MemStorage) { metric = mtr.NewGauge(stat.Name, v) } case uint32: - if stat.Type == mtr.GaugeType { - metric = mtr.NewGauge(stat.Name, float64(v)) - } + if stat.Type == mtr.GaugeType { + metric = mtr.NewGauge(stat.Name, float64(v)) + } default: log.Error("ERROR: Unknown type for metric %s: %T", stat.Name, value) continue @@ -201,11 +201,14 @@ func sendAllMetrics(client *resty.Client, storage *ms.MemStorage) { } func sendMetric(client *resty.Client, mType string, mName string, mValue interface{}) { - url := fmt.Sprintf("update/%s/%s/%v", mType, mName, mValue) - res, err := client.R(). SetHeader("Content-Type", "text/plain"). - Post(url) + SetPathParams(map[string]string{ + "mType": mType, + "mName": mName, + "mValue": fmt.Sprintf("%v", mValue), + }). + Post("update/{mType}/{mName}/{mValue}") if err != nil { log.Error("Error creating a request for %s: %v", mName, err) diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index 2b62e1f..eb92a10 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -9,7 +9,7 @@ import ( "github.com/go-chi/chi/v5" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" - log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/logger" + log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) type MetricTable struct { diff --git a/internal/memstorage/storage.go b/internal/memstorage/storage.go index 2afb471..12f006b 100644 --- a/internal/memstorage/storage.go +++ b/internal/memstorage/storage.go @@ -2,6 +2,7 @@ package memstorage import ( "sync" + "maps" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" ) @@ -91,15 +92,8 @@ func (ms *MemStorage) GetAllMetrics() (map[string]float64, map[string]int64) { ms.mutex.RLock() defer ms.mutex.RUnlock() - gaugesCopy := make(map[string]float64, len(ms.Gauges)) - for name, value := range ms.Gauges { - gaugesCopy[name] = value - } - - countersCopy := make(map[string]int64, len(ms.Counters)) - for name, value := range ms.Counters { - countersCopy[name] = value - } + gaugesCopy := maps.Clone(ms.Gauges) + countersCopy := maps.Clone(ms.Counters) return gaugesCopy, countersCopy } diff --git a/logger/logger.go b/pkg/logger/logger.go similarity index 100% rename from logger/logger.go rename to pkg/logger/logger.go From 80e853b4262aad4b5e4223ff3bf77850500bbf6c Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Fri, 4 Jul 2025 22:30:25 +0300 Subject: [PATCH 027/126] rewrote the flag library to cobra --- cmd/agent/cfg_agent.go | 78 ++++++++++++++++++++-------------------- cmd/agent/main.go | 6 +++- cmd/server/cfg_server.go | 55 ++++++++++++++-------------- cmd/server/main.go | 5 ++- 4 files changed, 77 insertions(+), 67 deletions(-) diff --git a/cmd/agent/cfg_agent.go b/cmd/agent/cfg_agent.go index c7021ff..dab53d3 100644 --- a/cmd/agent/cfg_agent.go +++ b/cmd/agent/cfg_agent.go @@ -1,12 +1,11 @@ package main import ( - "flag" "fmt" "net" - "os" "github.com/caarlos0/env/v6" + "github.com/spf13/cobra" ) const ( @@ -27,47 +26,48 @@ type envConfig struct { ReportInterval int `env:"REPORT_INTERVAL"` } -func envAndFlagsInit() *options { - opts := &options{ - endPointAddr: defaultEndpoint, - pollInterval: defaultPollInterval, - reportInterval: defaultReportInterval, - } +var opts = &options{} - flag.StringVar(&opts.endPointAddr, "a", opts.endPointAddr, "endpoint HTTP-server addr") - flag.IntVar(&opts.pollInterval, "p", opts.pollInterval, "PollInterval value") - flag.IntVar(&opts.reportInterval, "r", opts.reportInterval, "ReportInterval value") +var rootCmd = &cobra.Command{ + Use: "agent", + Short: "MetricService", + Long: "MetricService", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + var cfg envConfig + err := env.Parse(&cfg) + if err != nil { + return fmt.Errorf("poll and report intervals must be > 0") + } - flag.Parse() + if cfg.EndPointAddr != "" { + opts.endPointAddr = cfg.EndPointAddr + } + if cfg.PollInterval > 0 { + opts.pollInterval = cfg.PollInterval + } + if cfg.ReportInterval > 0 { + opts.reportInterval = cfg.ReportInterval + } - var cfg envConfig - err := env.Parse(&cfg) - if err != nil { - fmt.Println("environment variables parsing error") - os.Exit(1) - } + if opts.pollInterval <= 0 || opts.reportInterval <= 0 { + return fmt.Errorf("poll and report intervals must be > 0") + } - if cfg.EndPointAddr != "" { - opts.endPointAddr = cfg.EndPointAddr - } - if cfg.PollInterval > 0 { - opts.pollInterval = cfg.PollInterval - } - if cfg.ReportInterval > 0 { - opts.reportInterval = cfg.ReportInterval - } + if _, _, err := net.SplitHostPort(opts.endPointAddr); err != nil { + return fmt.Errorf("invalid address %s: %w", opts.endPointAddr, err) + } - if opts.pollInterval <= 0 || opts.reportInterval <= 0 { - fmt.Println("Error: poll interval and report interval must be > 0") - flag.Usage() - os.Exit(1) - } - - if _, _, err := net.SplitHostPort(opts.endPointAddr); err != nil { - fmt.Errorf("invalid address %s: %w", opts.endPointAddr, err) - flag.Usage() - os.Exit(1) - } + return nil + }, +} - return opts +func init() { + opts.endPointAddr = defaultEndpoint + opts.pollInterval = defaultPollInterval + opts.reportInterval = defaultReportInterval + + rootCmd.Flags().StringVarP(&opts.endPointAddr, "a", "a", opts.endPointAddr, "endpoint HTTP-server addr") + rootCmd.Flags().IntVarP(&opts.pollInterval, "p", "p", opts.pollInterval, "PollInterval value") + rootCmd.Flags().IntVarP(&opts.reportInterval, "r", "r", opts.reportInterval, "PollInterval value") } diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 52fb7e0..619b1e3 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -2,6 +2,7 @@ package main import ( "time" + "os" "github.com/go-resty/resty/v2" @@ -14,10 +15,13 @@ func main() { log.Init(log.DebugLevel, "logFileAgent.log") defer log.Destroy() + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } + log.Debug("START AGENT>") storage := ms.NewMemStorage() - opts := envAndFlagsInit() log.Debug("Configuration: endPointAddr=%s, pollInterval=%ds, reportInterval=%ds", opts.endPointAddr, opts.pollInterval, opts.reportInterval) diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 91df44e..019c7c7 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -1,12 +1,11 @@ package main import ( - "flag" "fmt" "net" - "os" "github.com/caarlos0/env/v6" + "github.com/spf13/cobra" ) const ( @@ -21,30 +20,34 @@ type envConfig struct { EndPointAddr string `env:"ADDRESS"` } -func envAndFlagsInit() *options { - opts := &options{ - endPointAddr: defaultEndpoint, - } - - flag.StringVar(&opts.endPointAddr, "a", opts.endPointAddr, "endpoint HTTP-server address") - flag.Parse() - - var cfg envConfig - err := env.Parse(&cfg) - if err != nil { - fmt.Println("environment variable parsing error") - os.Exit(1) - } - - if cfg.EndPointAddr != "" { - opts.endPointAddr = cfg.EndPointAddr - } +var opts = &options{} + +var rootCmd = &cobra.Command{ + Use: "server", + Short: "MetricService", + Long: "MetricService", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + var cfg envConfig + err := env.Parse(&cfg) + if err != nil { + return fmt.Errorf("poll and report intervals must be > 0") + } + + if cfg.EndPointAddr != "" { + opts.endPointAddr = cfg.EndPointAddr + } + + if _, _, err := net.SplitHostPort(opts.endPointAddr); err != nil { + return fmt.Errorf("invalid address %s: %w", opts.endPointAddr, err) + } + + return nil + }, +} - if _, _, err := net.SplitHostPort(opts.endPointAddr); err != nil { - fmt.Errorf("invalid address %s: %w", opts.endPointAddr, err) - flag.Usage() - os.Exit(1) - } +func init() { + opts.endPointAddr = defaultEndpoint - return opts + rootCmd.Flags().StringVarP(&opts.endPointAddr, "a", "a", opts.endPointAddr, "endpoint HTTP-server addr") } diff --git a/cmd/server/main.go b/cmd/server/main.go index 51b1719..cf876d9 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -3,6 +3,7 @@ package main import ( "fmt" "net/http" + "os" "github.com/go-chi/chi/v5" @@ -15,7 +16,9 @@ func main() { log.Init(log.DebugLevel, "logFileServer.log") defer log.Destroy() - opts := envAndFlagsInit() + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } log.Debug("Server configuration: Address=%s", opts.endPointAddr) log.Debug("START SERVER>") From d76298f0a6af14c54e62763e2e822e13de3c9125 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Fri, 4 Jul 2025 22:48:17 +0300 Subject: [PATCH 028/126] made NewRouter() for test and main --- cmd/server/main.go | 12 +----------- internal/handlers/server/server.go | 15 +++++++++++++++ internal/handlers/server/server_test.go | 18 ++---------------- 3 files changed, 18 insertions(+), 27 deletions(-) diff --git a/cmd/server/main.go b/cmd/server/main.go index cf876d9..7377aba 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -5,8 +5,6 @@ import ( "net/http" "os" - "github.com/go-chi/chi/v5" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" @@ -25,15 +23,7 @@ func main() { fmt.Printf("Endpoint: [%s]\n", opts.endPointAddr) storage := ms.NewMemStorage() - r := chi.NewRouter() - - r.Route("/", func(r chi.Router) { - r.Get("/", server.GetAllMetrics(storage)) - r.Route("/", func(r chi.Router) { - r.Get("/value/{mType}/{mName}", server.GetMetric(storage)) - r.Post("/update/{mType}/{mName}/{mValue}", server.UpdateMetric(storage)) - }) - }) + r := server.NewRouter(storage) if err := http.ListenAndServe(opts.endPointAddr, r); err != nil { log.Error("HTTP-server didn't start: %v", err) diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index eb92a10..0f75bff 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -7,6 +7,7 @@ import ( "strconv" "github.com/go-chi/chi/v5" + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" @@ -18,6 +19,20 @@ type MetricTable struct { Value string } +func NewRouter(storage ms.Collector) http.Handler { + r := chi.NewRouter() + + r.Route("/", func(r chi.Router) { + r.Get("/", GetAllMetrics(storage)) + r.Route("/", func(r chi.Router) { + r.Get("/value/{mType}/{mName}", GetMetric(storage)) + r.Post("/update/{mType}/{mName}/{mValue}", UpdateMetric(storage)) + }) + }) + + return r +} + // FIXME other function name func NewCounter(res http.ResponseWriter, name, value string) (mtr.Metric, error) { val, err := strconv.ParseInt(value, 10, 64) diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index 453a003..28f0cac 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -6,27 +6,13 @@ import ( "net/http/httptest" "testing" - "github.com/go-chi/chi/v5" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" ) -func setupTestRouter(storage ms.Collector) http.Handler { - r := chi.NewRouter() - - r.Route("/", func(r chi.Router) { - r.Get("/", GetAllMetrics(storage)) - r.Route("/", func(r chi.Router) { - r.Get("/value/{mType}/{mName}", GetMetric(storage)) - r.Post("/update/{mType}/{mName}/{mValue}", UpdateMetric(storage)) - }) - }) - return r -} - func TestUpdateMetric(t *testing.T) { storage := ms.NewMemStorage() - router := setupTestRouter(storage) + router := NewRouter(storage) tests := []struct { name string @@ -97,7 +83,7 @@ func TestGetMetric(t *testing.T) { storage.UpdateMetric(mtr.NewGauge("cpu_usage", 75.5)) storage.UpdateMetric(mtr.NewCounter("requests_total", 100)) - router := setupTestRouter(storage) + router := NewRouter(storage) tests := []struct { name string From d31986e4ac723617a738288c50357e83c5405c6a Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Fri, 4 Jul 2025 23:11:09 +0300 Subject: [PATCH 029/126] fixed lint errors --- cmd/agent/main.go | 7 +++++-- cmd/server/main.go | 4 +++- internal/handlers/agent/agent.go | 11 ++++++++--- internal/handlers/server/server.go | 7 ++++++- internal/handlers/server/server_test.go | 10 ++++++++-- pkg/logger/logger.go | 4 +++- 6 files changed, 33 insertions(+), 10 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 619b1e3..7df0809 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -1,8 +1,9 @@ package main import ( - "time" + "fmt" "os" + "time" "github.com/go-resty/resty/v2" @@ -12,7 +13,9 @@ import ( ) func main() { - log.Init(log.DebugLevel, "logFileAgent.log") + if err := log.Init(log.DebugLevel, "logFileAgent.log"); err != nil { + fmt.Errorf("Error initializing the log file: %v", err) + } defer log.Destroy() if err := rootCmd.Execute(); err != nil { diff --git a/cmd/server/main.go b/cmd/server/main.go index 7377aba..8699177 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -11,7 +11,9 @@ import ( ) func main() { - log.Init(log.DebugLevel, "logFileServer.log") + if err := log.Init(log.DebugLevel, "logFileServer.log"); err != nil { + fmt.Errorf("Error initializing the log file: %v", err) + } defer log.Destroy() if err := rootCmd.Execute(); err != nil { diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index e0efeb9..15852d5 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -180,12 +180,17 @@ func UpdateAllMetrics(storage *ms.MemStorage) { } if err := storage.UpdateMetric(metric); err != nil { - log.Error("ERROR!") + log.Error("Failed to update metric %s: %v", stat.Name, err) } } - storage.UpdateMetric(mtr.NewCounter("PollCount", 1)) - storage.UpdateMetric(mtr.NewGauge("RandomValue", rand.Float64())) + if err := storage.UpdateMetric(mtr.NewCounter("PollCount", 1)); err != nil { + log.Error("Failed to update PollCount metric: %v", err) + } + + if err := storage.UpdateMetric(mtr.NewGauge("RandomValue", rand.Float64())); err != nil { + log.Error("Failed to update RandomValue metric: %v", err) + } } func sendAllMetrics(client *resty.Client, storage *ms.MemStorage) { diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index 0f75bff..911f6d7 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -86,7 +86,12 @@ func GetMetric(storage ms.Collector) http.HandlerFunc { res.Header().Set("Content-Type", "text/plain") res.WriteHeader(http.StatusOK) - res.Write([]byte(valueStr)) + + _, err := res.Write([]byte(valueStr)) + if err != nil { + log.Error("Failed to write response: %v", err) + + } log.Debug("the metric has been send") } diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index 28f0cac..9ad2e0e 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -8,6 +8,7 @@ import ( ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) func TestUpdateMetric(t *testing.T) { @@ -81,8 +82,13 @@ func TestUpdateMetric(t *testing.T) { func TestGetMetric(t *testing.T) { storage := ms.NewMemStorage() - storage.UpdateMetric(mtr.NewGauge("cpu_usage", 75.5)) - storage.UpdateMetric(mtr.NewCounter("requests_total", 100)) + if err := storage.UpdateMetric(mtr.NewGauge("cpu_usage", 75.5)); err != nil { + log.Error("Failed to update metric cpu_usage: %v", err) + } + + if err := storage.UpdateMetric(mtr.NewCounter("requests_total", 100)); err != nil { + log.Error("Failed to update metric requests_total: %v", err) + } router := NewRouter(storage) tests := []struct { diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 69ab32c..dd314df 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -64,7 +64,9 @@ func Init(level LogLevel, logFileName string) error { func Destroy() { log := getLogger() if file, ok := log.out.(*os.File); ok { - file.Close() + if err := file.Close(); err != nil { + fmt.Errorf("Failed to close log file: %v", err) + } } } From 551f0a433f22b00f6388db6d0d3d6d7c8c4c9ed9 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Fri, 4 Jul 2025 23:15:21 +0300 Subject: [PATCH 030/126] fixed lint errors --- cmd/agent/main.go | 2 +- cmd/server/main.go | 2 +- pkg/logger/logger.go | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 7df0809..e1c9737 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -14,7 +14,7 @@ import ( func main() { if err := log.Init(log.DebugLevel, "logFileAgent.log"); err != nil { - fmt.Errorf("Error initializing the log file: %v", err) + fmt.Printf("Error initializing the log file: %v", err) } defer log.Destroy() diff --git a/cmd/server/main.go b/cmd/server/main.go index 8699177..7817662 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -12,7 +12,7 @@ import ( func main() { if err := log.Init(log.DebugLevel, "logFileServer.log"); err != nil { - fmt.Errorf("Error initializing the log file: %v", err) + fmt.Printf("Error initializing the log file: %v", err) } defer log.Destroy() diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index dd314df..5047025 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -65,7 +65,7 @@ func Destroy() { log := getLogger() if file, ok := log.out.(*os.File); ok { if err := file.Close(); err != nil { - fmt.Errorf("Failed to close log file: %v", err) + fmt.Printf("Failed to close log file: %v", err) } } } @@ -83,7 +83,10 @@ func (log *logger) log(level LogLevel, format string, args ...interface{}) { levelStr := levelMap[level] msg := fmt.Sprintf(format, args...) - fmt.Fprintf(log.out, "[%s]%s[%s:%d]: %s \n", time, levelStr, filepath.Base(file), line, msg) + _, err := fmt.Fprintf(log.out, "[%s]%s[%s:%d]: %s \n", time, levelStr, filepath.Base(file), line, msg) + if err != nil { + fmt.Printf("failed to write log message: %v\n", err) + } } func Debug(format string, args ...interface{}) { From de81e2f696243f204e3ea2168d866baf2780c698 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Sat, 5 Jul 2025 18:00:01 +0300 Subject: [PATCH 031/126] rewrote my logger to zerolog --- cmd/agent/cfg_agent.go | 35 +++++- cmd/agent/main.go | 27 ----- cmd/server/cfg_server.go | 32 ++++- cmd/server/main.go | 24 ---- go.mod | 8 +- go.sum | 18 +++ internal/handlers/agent/agent.go | 20 +-- internal/handlers/server/server.go | 24 ++-- internal/handlers/server/server_test.go | 6 +- .../{memstorage => mem-storage}/storage.go | 0 pkg/logger/logger.go | 114 ++++++------------ 11 files changed, 154 insertions(+), 154 deletions(-) rename internal/{memstorage => mem-storage}/storage.go (100%) diff --git a/cmd/agent/cfg_agent.go b/cmd/agent/cfg_agent.go index dab53d3..401f212 100644 --- a/cmd/agent/cfg_agent.go +++ b/cmd/agent/cfg_agent.go @@ -3,9 +3,16 @@ package main import ( "fmt" "net" + "os" + "time" "github.com/caarlos0/env/v6" + "github.com/go-resty/resty/v2" "github.com/spf13/cobra" + + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/agent" + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" + log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) const ( @@ -34,8 +41,15 @@ var rootCmd = &cobra.Command{ Long: "MetricService", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { + logFile, err := log.InitLogger("logFileAgent.log") + if err != nil { + fmt.Fprintf(os.Stderr, "Logger init error: %v\n", err) + os.Exit(1) + } + defer logFile.Close() + var cfg envConfig - err := env.Parse(&cfg) + err = env.Parse(&cfg) if err != nil { return fmt.Errorf("poll and report intervals must be > 0") } @@ -58,6 +72,8 @@ var rootCmd = &cobra.Command{ return fmt.Errorf("invalid address %s: %w", opts.endPointAddr, err) } + startAgent() + return nil }, } @@ -66,8 +82,23 @@ func init() { opts.endPointAddr = defaultEndpoint opts.pollInterval = defaultPollInterval opts.reportInterval = defaultReportInterval - + rootCmd.Flags().StringVarP(&opts.endPointAddr, "a", "a", opts.endPointAddr, "endpoint HTTP-server addr") rootCmd.Flags().IntVarP(&opts.pollInterval, "p", "p", opts.pollInterval, "PollInterval value") rootCmd.Flags().IntVarP(&opts.reportInterval, "r", "r", opts.reportInterval, "PollInterval value") } + +func startAgent() { + storage := ms.NewMemStorage() + + client := resty.New(). + SetTimeout(5 * time.Second). + SetBaseURL("http://" + opts.endPointAddr) + + log.Info().Msg("Starting collection and reporting loops") + + go agent.CollectionLoop(storage, time.Duration(opts.pollInterval)*time.Second) + go agent.ReportLoop(client, storage, time.Duration(opts.reportInterval)*time.Second) + + select {} +} diff --git a/cmd/agent/main.go b/cmd/agent/main.go index e1c9737..839d48d 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -1,40 +1,13 @@ package main import ( - "fmt" "os" - "time" - - "github.com/go-resty/resty/v2" - - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/agent" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" - log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) func main() { - if err := log.Init(log.DebugLevel, "logFileAgent.log"); err != nil { - fmt.Printf("Error initializing the log file: %v", err) - } - defer log.Destroy() if err := rootCmd.Execute(); err != nil { os.Exit(1) } - log.Debug("START AGENT>") - storage := ms.NewMemStorage() - - log.Debug("Configuration: endPointAddr=%s, pollInterval=%ds, reportInterval=%ds", - opts.endPointAddr, opts.pollInterval, opts.reportInterval) - - client := resty.New(). - SetTimeout(5 * time.Second). - SetBaseURL("http://" + opts.endPointAddr) - - go agent.CollectionLoop(storage, time.Duration(opts.pollInterval)*time.Second) - go agent.ReportLoop(client, storage, time.Duration(opts.reportInterval)*time.Second) - - log.Debug("END AGENT<") - select {} } diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 019c7c7..1f4811b 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -3,9 +3,15 @@ package main import ( "fmt" "net" + "net/http" + "os" "github.com/caarlos0/env/v6" "github.com/spf13/cobra" + + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" + log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) const ( @@ -28,8 +34,15 @@ var rootCmd = &cobra.Command{ Long: "MetricService", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { + logFile, err := log.InitLogger("logFileServer.log") + if err != nil { + fmt.Fprintf(os.Stderr, "Logger init error: %v\n", err) + os.Exit(1) + } + defer logFile.Close() + var cfg envConfig - err := env.Parse(&cfg) + err = env.Parse(&cfg) if err != nil { return fmt.Errorf("poll and report intervals must be > 0") } @@ -42,6 +55,8 @@ var rootCmd = &cobra.Command{ return fmt.Errorf("invalid address %s: %w", opts.endPointAddr, err) } + startServer() + return nil }, } @@ -51,3 +66,18 @@ func init() { rootCmd.Flags().StringVarP(&opts.endPointAddr, "a", "a", opts.endPointAddr, "endpoint HTTP-server addr") } + +func startServer() { + log.Info(). + Str("address", opts.endPointAddr). + Msg("Server configuration") + + storage := ms.NewMemStorage() + r := server.NewRouter(storage) + + if err := http.ListenAndServe(opts.endPointAddr, r); err != nil { + fmt.Fprintf(os.Stderr, "HTTP-server didn't start: %v", err) + panic(err) + } + +} diff --git a/cmd/server/main.go b/cmd/server/main.go index 7817662..839d48d 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -1,37 +1,13 @@ package main import ( - "fmt" - "net/http" "os" - - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" - log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) func main() { - if err := log.Init(log.DebugLevel, "logFileServer.log"); err != nil { - fmt.Printf("Error initializing the log file: %v", err) - } - defer log.Destroy() if err := rootCmd.Execute(); err != nil { os.Exit(1) } - log.Debug("Server configuration: Address=%s", opts.endPointAddr) - - log.Debug("START SERVER>") - fmt.Printf("Endpoint: [%s]\n", opts.endPointAddr) - - storage := ms.NewMemStorage() - r := server.NewRouter(storage) - - if err := http.ListenAndServe(opts.endPointAddr, r); err != nil { - log.Error("HTTP-server didn't start: %v", err) - panic(err) - } - log.Debug("ListenAndServe returned") - log.Debug("END SERVER<") } diff --git a/go.mod b/go.mod index c903c69..f75a3e6 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/rAch-kaplin/mipt-golang-course/MetricsService -go 1.22.0 +go 1.23.0 + +toolchain go1.24.3 require ( github.com/caarlos0/env/v6 v6.10.1 @@ -10,7 +12,11 @@ require ( require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/rs/zerolog v1.34.0 // indirect github.com/spf13/cobra v1.9.1 // indirect github.com/spf13/pflag v1.0.6 // indirect golang.org/x/net v0.33.0 // indirect + golang.org/x/sys v0.33.0 // indirect ) diff --git a/go.sum b/go.sum index 8a67247..6ed1198 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,25 @@ github.com/caarlos0/env/v6 v6.10.1 h1:t1mPSxNpei6M5yAeu1qtRdPAK29Nbcf/n3G7x+b3/II= github.com/caarlos0/env/v6 v6.10.1/go.mod h1:hvp/ryKXKipEkcuYjs9mI4bBCg+UI0Yhgm5Zu0ddvwc= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618= github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM= github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= +github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= @@ -14,6 +27,11 @@ github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 15852d5..6809c0f 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -9,7 +9,7 @@ import ( "github.com/go-resty/resty/v2" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -175,21 +175,21 @@ func UpdateAllMetrics(storage *ms.MemStorage) { metric = mtr.NewGauge(stat.Name, float64(v)) } default: - log.Error("ERROR: Unknown type for metric %s: %T", stat.Name, value) + log.Error().Msgf("Unknown type for metric %s: %T", stat.Name, value) continue } if err := storage.UpdateMetric(metric); err != nil { - log.Error("Failed to update metric %s: %v", stat.Name, err) + log.Error().Msgf("Failed to update metric %s: %v", stat.Name, err) } } if err := storage.UpdateMetric(mtr.NewCounter("PollCount", 1)); err != nil { - log.Error("Failed to update PollCount metric: %v", err) + log.Error().Msgf("Failed to update PollCount metric: %v", err) } if err := storage.UpdateMetric(mtr.NewGauge("RandomValue", rand.Float64())); err != nil { - log.Error("Failed to update RandomValue metric: %v", err) + log.Error().Msgf("Failed to update RandomValue metric: %v", err) } } @@ -216,19 +216,19 @@ func sendMetric(client *resty.Client, mType string, mName string, mValue interfa Post("update/{mType}/{mName}/{mValue}") if err != nil { - log.Error("Error creating a request for %s: %v", mName, err) + log.Error().Msgf("Error creating a request for %s: %v", mName, err) return } if res.StatusCode() != http.StatusOK { - log.Error("Server returned non-OK status for %s/%s: %d %s", mType, mName, res.StatusCode(), res.String()) + log.Error().Msgf("Server returned non-OK status for %s/%s: %d %s", mType, mName, res.StatusCode(), res.String()) } else { - log.Debug("Metric %s/%s sent successfully. Status: %d", mType, mName, res.StatusCode()) + log.Debug().Msgf("Metric %s/%s sent successfully. Status: %d", mType, mName, res.StatusCode()) } } func CollectionLoop(storage *ms.MemStorage, interval time.Duration) { - log.Debug("collectionLoop ...") + log.Debug().Msg("collectionLoop ...") for { UpdateAllMetrics(storage) time.Sleep(interval) @@ -236,7 +236,7 @@ func CollectionLoop(storage *ms.MemStorage, interval time.Duration) { } func ReportLoop(client *resty.Client, storage *ms.MemStorage, interval time.Duration) { - log.Debug("reportLoop ...") + log.Debug().Msg("reportLoop ...") for { time.Sleep(interval) sendAllMetrics(client, storage) diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index 911f6d7..8c7d26f 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -8,7 +8,7 @@ import ( "github.com/go-chi/chi/v5" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -61,11 +61,11 @@ func HandleUnknownMetric(res http.ResponseWriter) { func GetMetric(storage ms.Collector) http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { - log.Debug("Incoming GET request: %s %s", req.Method, req.URL.Path) + log.Debug().Msgf("Incoming GET request: %s %s", req.Method, req.URL.Path) mType := chi.URLParam(req, "mType") mName := chi.URLParam(req, "mName") - log.Debug("Incoming request for metric: Type=%s, Name=%s", mType, mName) + log.Debug().Msgf("Incoming request for metric: Type=%s, Name=%s", mType, mName) value, found := storage.GetMetric(mType, mName) if !found { @@ -89,11 +89,11 @@ func GetMetric(storage ms.Collector) http.HandlerFunc { _, err := res.Write([]byte(valueStr)) if err != nil { - log.Error("Failed to write response: %v", err) + log.Error().Msgf("Failed to write response: %v", err) } - log.Debug("the metric has been send") + log.Debug().Msg("the metric has been send") } } @@ -150,7 +150,7 @@ func GetAllMetrics(storage ms.Collector) http.HandlerFunc { template, err := template.New("Metrics").Parse(htmlTemplate) if err != nil { - log.Error("couldn't make it out HTML template: %v", err) + log.Error().Msgf("couldn't make it out HTML template: %v", err) http.Error(res, "Internal server error, failed html-template", http.StatusInternalServerError) return } @@ -158,25 +158,25 @@ func GetAllMetrics(storage ms.Collector) http.HandlerFunc { res.Header().Set("Content-Type", "text/html; charset=utf-8") if err := template.Execute(res, metricsToTable); err != nil { - log.Error("failed complete template: %v", err) + log.Error().Msgf("failed complete template: %v", err) } - log.Debug("the metrics has been send") + log.Debug().Msg("the metrics has been send") res.WriteHeader(http.StatusOK) } } func UpdateMetric(storage ms.Collector) http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { - log.Debug("Incoming POST request: %s %s", req.Method, req.URL.Path) + log.Debug().Msgf("Incoming POST request: %s %s", req.Method, req.URL.Path) mType := chi.URLParam(req, "mType") mName := chi.URLParam(req, "mName") mValue := chi.URLParam(req, "mValue") - log.Debug("Parsed metric: type=%s, name=%s, value=%s", mType, mName, mValue) + log.Debug().Msgf("Parsed metric: type=%s, name=%s, value=%s", mType, mName, mValue) if mName == "" { - log.Error("the metric name is not specified") + log.Error().Msgf("the metric name is not specified") http.Error(res, "the metric name is not specified", http.StatusBadRequest) return } @@ -204,7 +204,7 @@ func UpdateMetric(storage ms.Collector) http.HandlerFunc { return } - log.Info("Metric updated successfully: %s %s = %s", mType, mName, mValue) + log.Info().Msgf("Metric updated successfully: %s %s = %s", mType, mName, mValue) res.WriteHeader(http.StatusOK) } } diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index 9ad2e0e..09b9ab3 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -6,7 +6,7 @@ import ( "net/http/httptest" "testing" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/memstorage" + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -83,11 +83,11 @@ func TestGetMetric(t *testing.T) { storage := ms.NewMemStorage() if err := storage.UpdateMetric(mtr.NewGauge("cpu_usage", 75.5)); err != nil { - log.Error("Failed to update metric cpu_usage: %v", err) + log.Error().Msgf("Failed to update metric cpu_usage: %v", err) } if err := storage.UpdateMetric(mtr.NewCounter("requests_total", 100)); err != nil { - log.Error("Failed to update metric requests_total: %v", err) + log.Error().Msgf("Failed to update metric requests_total: %v", err) } router := NewRouter(storage) diff --git a/internal/memstorage/storage.go b/internal/mem-storage/storage.go similarity index 100% rename from internal/memstorage/storage.go rename to internal/mem-storage/storage.go diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 5047025..98fca6c 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -2,101 +2,67 @@ package logger import ( "fmt" - "io" "os" "path/filepath" - "runtime" - "sync" - "time" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" ) -type LogLevel int +var logger zerolog.Logger -const ( - DebugLevel LogLevel = iota - InfoLevel - ErrorLevel -) +func InitLogger(logFilePath string) (*os.File, error) { + file, err := os.OpenFile(logFilePath, os.O_CREATE|os.O_WRONLY, 0666) + if err != nil { + return nil, fmt.Errorf("failed to open log file %s: %w", logFilePath, err) + } -var levelMap = map[LogLevel]string{ - DebugLevel: "[DEBUG]", - InfoLevel: "[INFO]", - ErrorLevel: "[ERROR]", -} + zerolog.SetGlobalLevel(zerolog.DebugLevel) + zerolog.TimeFieldFormat = "2006-01-02 15:04:05" -type logger struct { - level LogLevel - out io.Writer -} + zerolog.CallerMarshalFunc = func(pc uintptr, file string, line int) string { + return fmt.Sprintf("%s:%d", filepath.Base(file), line) + } -var ( - instance *logger - once sync.Once -) + logger = zerolog.New(file). + With(). + Timestamp(). + CallerWithSkipFrameCount(2). + Logger() -func getLogger() *logger { - once.Do(func() { - instance = &logger{ - level: DebugLevel, - out: os.Stdout, - } - }) - return instance -} + log.Logger = logger -func Init(level LogLevel, logFileName string) error { - log := getLogger() - log.level = level - - if logFileName != "" { - file, err := os.Create(logFileName) - if err != nil { - return fmt.Errorf("failed to open log file %s: %w", logFileName, err) - } - log.out = file - } else { - log.out = os.Stdout - } + return file, nil +} - return nil +func Trace() *zerolog.Event { + return logger.Trace() } -func Destroy() { - log := getLogger() - if file, ok := log.out.(*os.File); ok { - if err := file.Close(); err != nil { - fmt.Printf("Failed to close log file: %v", err) - } - } +func Debug() *zerolog.Event { + return logger.Debug() } -func (log *logger) log(level LogLevel, format string, args ...interface{}) { - if log.level > level { - return - } +func Info() *zerolog.Event { + return logger.Info() +} - _, file, line, ok := runtime.Caller(2) - if !ok { - file, line = "---", 0 - } - time := time.Now().Format("2006-01-02 15:04:05") - levelStr := levelMap[level] - msg := fmt.Sprintf(format, args...) +func Warn() *zerolog.Event { + return logger.Warn() +} - _, err := fmt.Fprintf(log.out, "[%s]%s[%s:%d]: %s \n", time, levelStr, filepath.Base(file), line, msg) - if err != nil { - fmt.Printf("failed to write log message: %v\n", err) - } +func Error() *zerolog.Event { + return logger.Error() } -func Debug(format string, args ...interface{}) { - getLogger().log(DebugLevel, format, args...) +func Fatal() *zerolog.Event { + return logger.Fatal() } -func Info(format string, args ...interface{}) { - getLogger().log(InfoLevel, format, args...) +func Panic() *zerolog.Event { + return logger.Panic() } -func Error(format string, args ...interface{}) { - getLogger().log(ErrorLevel, format, args...) +func WithLevel(level zerolog.Level) *zerolog.Event { + return logger.WithLevel(level) } From facea5afb2a77460ea7b03bbaf24581c1492dd43 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Sat, 5 Jul 2025 22:03:47 +0300 Subject: [PATCH 032/126] switched to zerolog and wrote a logger for the server --- cmd/agent/cfg_agent.go | 7 ++++++- cmd/server/cfg_server.go | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/cmd/agent/cfg_agent.go b/cmd/agent/cfg_agent.go index 401f212..fb2e346 100644 --- a/cmd/agent/cfg_agent.go +++ b/cmd/agent/cfg_agent.go @@ -46,7 +46,12 @@ var rootCmd = &cobra.Command{ fmt.Fprintf(os.Stderr, "Logger init error: %v\n", err) os.Exit(1) } - defer logFile.Close() + + defer func() { + if err := logFile.Close(); err != nil { + log.Error().Err(err).Msg("Failed to close log file") + } + }() var cfg envConfig err = env.Parse(&cfg) diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 1f4811b..2c7f731 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -39,7 +39,12 @@ var rootCmd = &cobra.Command{ fmt.Fprintf(os.Stderr, "Logger init error: %v\n", err) os.Exit(1) } - defer logFile.Close() + + defer func() { + if err := logFile.Close(); err != nil { + log.Error().Err(err).Msg("Failed to close log file") + } + }() var cfg envConfig err = env.Parse(&cfg) From e6d5ea70cae17de831e4b73702715b186bf479a5 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Sun, 6 Jul 2025 18:20:17 +0300 Subject: [PATCH 033/126] some commit --- .gitignore | 3 + internal/handlers/agent/agent.go | 198 ++++++----------------------- pkg/runtime-stats/runtime-stats.go | 146 +++++++++++++++++++++ 3 files changed, 188 insertions(+), 159 deletions(-) create mode 100644 pkg/runtime-stats/runtime-stats.go diff --git a/.gitignore b/.gitignore index c938849..245363d 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,6 @@ logFileServer.log logFileAgent.log cmd/server/server cmd/agent/agent + + +log.txt diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 6809c0f..aae907a 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -12,175 +12,55 @@ import ( ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" + rt "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/runtime-stats" ) -type MemRuntimeStat struct { - Name string - Type string - Get func(m *runtime.MemStats) interface{} -} - -var MemRuntimeStats []MemRuntimeStat = []MemRuntimeStat{ - { - Name: "Alloc", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.Alloc }, - }, - { - Name: "BuckHashSys", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.BuckHashSys }, - }, - { - Name: "Frees", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.Frees }, - }, - { - Name: "GCCPUFraction", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.GCCPUFraction }, - }, - { - Name: "HeapAlloc", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.HeapAlloc }, - }, - { - Name: "HeapIdle", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.HeapIdle }, - }, - { - Name: "HeapInuse", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.HeapInuse }, - }, - { - Name: "HeapObjects", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.HeapObjects }, - }, - { - Name: "HeapReleased", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.HeapReleased }, - }, - { - Name: "HeapSys", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.HeapSys }, - }, - { - Name: "LastGC", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.LastGC }, - }, - { - Name: "Lookups", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.Lookups }, - }, - { - Name: "MCacheInuse", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.MCacheInuse }, - }, - { - Name: "MCacheSys", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.MCacheSys }, - }, - { - Name: "MSpanInuse", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.MSpanInuse }, - }, - { - Name: "MSpanSys", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.MSpanSys }, - }, - { - Name: "Mallocs", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.Mallocs }, - }, - { - Name: "NextGC", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.NextGC }, - }, - { - Name: "NumForcedGC", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.NumForcedGC }, - }, - { - Name: "NumGC", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.NumGC }, - }, - { - Name: "OtherSys", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.OtherSys }, - }, - { - Name: "PauseTotalNs", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.PauseTotalNs }, - }, - { - Name: "StackInuse", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.StackInuse }, - }, - { - Name: "StackSys", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.StackSys }, - }, - { - Name: "Sys", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.Sys }, - }, - { - Name: "TotalAlloc", - Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) interface{} { return m.TotalAlloc }, - }, +func toFloat64(val any) (float64, bool) { + switch v := val.(type) { + case float64: + return v, true + case uint64: + return float64(v), true + case uint32: + return float64(v), true + case int: + return float64(v), true + case int64: + return float64(v), true + default: + return 0, false + } } func UpdateAllMetrics(storage *ms.MemStorage) { var memStats runtime.MemStats runtime.ReadMemStats(&memStats) - for _, stat := range MemRuntimeStats { - value := stat.Get(&memStats) - var metric mtr.Metric - - switch v := value.(type) { - case uint64: - if stat.Type == mtr.GaugeType { - metric = mtr.NewGauge(stat.Name, float64(v)) - } - case float64: - if stat.Type == mtr.GaugeType { - metric = mtr.NewGauge(stat.Name, v) + for _, stat := range rt.MemRuntimeStats { + val := stat.Get(&memStats) + + switch stat.Type { + case mtr.GaugeType: + value, ok := toFloat64(val) + if !ok { + log.Error(). + Str("metric", stat.Name). + Str("type", fmt.Sprintf("%T", val)). + Msg("Failed to convert metric value to float64") + continue } - case uint32: - if stat.Type == mtr.GaugeType { - metric = mtr.NewGauge(stat.Name, float64(v)) - } - default: - log.Error().Msgf("Unknown type for metric %s: %T", stat.Name, value) - continue - } - if err := storage.UpdateMetric(metric); err != nil { - log.Error().Msgf("Failed to update metric %s: %v", stat.Name, err) + if err := storage.UpdateMetric(mtr.NewGauge(stat.Name, value)); err != nil { + log.Error(). + Err(err). + Str("metric", stat.Name). + Msg("Failed to update metric") + } + default: + log.Error(). + Str("metric", stat.Name). + Str("type", fmt.Sprintf("%T", val)). + Msg("Unsupported metric type") } } diff --git a/pkg/runtime-stats/runtime-stats.go b/pkg/runtime-stats/runtime-stats.go new file mode 100644 index 0000000..d8b0629 --- /dev/null +++ b/pkg/runtime-stats/runtime-stats.go @@ -0,0 +1,146 @@ +package runtimestats + +import ( + "runtime" + + mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" +) + +type MemRuntimeStat struct { + Name string + Type string + Get func(m *runtime.MemStats) any +} + +var MemRuntimeStats []MemRuntimeStat = []MemRuntimeStat{ + { + Name: "Alloc", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.Alloc }, + }, + { + Name: "BuckHashSys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.BuckHashSys }, + }, + { + Name: "Frees", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.Frees }, + }, + { + Name: "GCCPUFraction", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.GCCPUFraction }, + }, + { + Name: "HeapAlloc", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.HeapAlloc }, + }, + { + Name: "HeapIdle", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.HeapIdle }, + }, + { + Name: "HeapInuse", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.HeapInuse }, + }, + { + Name: "HeapObjects", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.HeapObjects }, + }, + { + Name: "HeapReleased", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.HeapReleased }, + }, + { + Name: "HeapSys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.HeapSys }, + }, + { + Name: "LastGC", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.LastGC }, + }, + { + Name: "Lookups", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.Lookups }, + }, + { + Name: "MCacheInuse", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.MCacheInuse }, + }, + { + Name: "MCacheSys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.MCacheSys }, + }, + { + Name: "MSpanInuse", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.MSpanInuse }, + }, + { + Name: "MSpanSys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.MSpanSys }, + }, + { + Name: "Mallocs", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.Mallocs }, + }, + { + Name: "NextGC", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.NextGC }, + }, + { + Name: "NumForcedGC", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.NumForcedGC }, + }, + { + Name: "NumGC", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.NumGC }, + }, + { + Name: "OtherSys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.OtherSys }, + }, + { + Name: "PauseTotalNs", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.PauseTotalNs }, + }, + { + Name: "StackInuse", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.StackInuse }, + }, + { + Name: "StackSys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.StackSys }, + }, + { + Name: "Sys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.Sys }, + }, + { + Name: "TotalAlloc", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return m.TotalAlloc }, + }, +} From 076fe0d3ac5d8d8ff6208bd177f851b3da14d311 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Sun, 6 Jul 2025 13:49:42 +0300 Subject: [PATCH 034/126] update go-version go.mod --- go.mod | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index f75a3e6..7b7d3b1 100644 --- a/go.mod +++ b/go.mod @@ -1,21 +1,19 @@ module github.com/rAch-kaplin/mipt-golang-course/MetricsService -go 1.23.0 - -toolchain go1.24.3 +go 1.24.3 require ( github.com/caarlos0/env/v6 v6.10.1 github.com/go-chi/chi/v5 v5.2.2 github.com/go-resty/resty/v2 v2.16.5 + github.com/rs/zerolog v1.34.0 + github.com/spf13/cobra v1.9.1 ) require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/rs/zerolog v1.34.0 // indirect - github.com/spf13/cobra v1.9.1 // indirect github.com/spf13/pflag v1.0.6 // indirect golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.33.0 // indirect From f40ac6909574de336223729184c2bc5108619204 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Sun, 6 Jul 2025 14:36:28 +0300 Subject: [PATCH 035/126] update statictest for my rep --- .github/workflows/statictest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/statictest.yml b/.github/workflows/statictest.yml index 61725e4..a93f184 100644 --- a/.github/workflows/statictest.yml +++ b/.github/workflows/statictest.yml @@ -17,7 +17,7 @@ jobs: - name: Download statictest binary uses: robinraju/release-downloader@v1.8 with: - repository: mipt-golang-course/go-autotests + repository: rAch-kaplin/go-autotests.git latest: true fileName: statictest out-file-path: .tools From 4410247ad76bead92f180f6f7d582e3013327beb Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Sun, 6 Jul 2025 14:46:09 +0300 Subject: [PATCH 036/126] delete .git for rep req --- .github/workflows/statictest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/statictest.yml b/.github/workflows/statictest.yml index a93f184..4861b07 100644 --- a/.github/workflows/statictest.yml +++ b/.github/workflows/statictest.yml @@ -17,7 +17,7 @@ jobs: - name: Download statictest binary uses: robinraju/release-downloader@v1.8 with: - repository: rAch-kaplin/go-autotests.git + repository: rAch-kaplin/go-autotests latest: true fileName: statictest out-file-path: .tools From 52d9f4c6c93ed117040f74abd67bb6db76149d3e Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Sun, 6 Jul 2025 14:55:13 +0300 Subject: [PATCH 037/126] update rep in metrictest --- .github/workflows/metricstest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/metricstest.yml b/.github/workflows/metricstest.yml index d35961a..a96ddad 100644 --- a/.github/workflows/metricstest.yml +++ b/.github/workflows/metricstest.yml @@ -40,7 +40,7 @@ jobs: - name: Download autotests binaries uses: robinraju/release-downloader@v1.8 with: - repository: mipt-golang-course/go-autotests + repository: rAch-kaplin/go-autotests latest: true fileName: "*" out-file-path: .tools From 6f4ca86547be1bdf8d2c696c78e0b7a4ad9355bf Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Sun, 6 Jul 2025 15:16:19 +0300 Subject: [PATCH 038/126] update to ver -> 1.24.3 --- .github/workflows/metricstest.yml | 2 +- .github/workflows/statictest.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/metricstest.yml b/.github/workflows/metricstest.yml index a96ddad..c6afc84 100644 --- a/.github/workflows/metricstest.yml +++ b/.github/workflows/metricstest.yml @@ -18,7 +18,7 @@ jobs: metricstest: runs-on: ubuntu-latest - container: golang:1.24.1 + container: golang:1.24.3 needs: branchtest services: diff --git a/.github/workflows/statictest.yml b/.github/workflows/statictest.yml index 4861b07..63bd7a1 100644 --- a/.github/workflows/statictest.yml +++ b/.github/workflows/statictest.yml @@ -9,7 +9,7 @@ on: jobs: statictest: runs-on: ubuntu-latest - container: golang:1.22 + container: golang:1.24.3 steps: - name: Checkout code uses: actions/checkout@v2 From 06c1745c6e324781f24cb214ca31cde0c2f26f25 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Mon, 7 Jul 2025 00:22:54 +0300 Subject: [PATCH 039/126] refactor storage --- internal/handlers/agent/agent.go | 35 ++++++++---- internal/handlers/server/server.go | 51 ++++++++++------- internal/mem-storage/storage.go | 91 +++++++++++------------------- internal/metrics/counter.go | 19 ++++++- internal/metrics/gauge.go | 19 ++++++- internal/metrics/metrics.go | 3 +- 6 files changed, 126 insertions(+), 92 deletions(-) diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index aae907a..6288aaf 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -74,17 +74,34 @@ func UpdateAllMetrics(storage *ms.MemStorage) { } func sendAllMetrics(client *resty.Client, storage *ms.MemStorage) { - gauges, counters := storage.GetAllMetrics() - - for name, value := range gauges { - sendMetric(client, mtr.GaugeType, name, value) - } - - for name, value := range counters { - sendMetric(client, mtr.CounterType, name, value) + allMetrics := storage.GetAllMetrics() + + for mType, innerMap := range allMetrics { + for mName, metric := range innerMap { + switch mType { + case mtr.GaugeType: + val, ok := metric.Value().(float64) + if !ok { + log.Error().Str("metric_name", mName).Str("metric_type", mType). + Msg("Invalid metric value type") + continue + } + sendMetric(client, mtr.GaugeType, mName, val) + + case mtr.CounterType: + val, ok := metric.Value().(int64) + if !ok { + log.Error().Str("metric_name", mName).Str("metric_type", mType). + Msg("Invalid metric value type") + continue + } + sendMetric(client, mtr.CounterType, mName, val) + } + } } } + func sendMetric(client *resty.Client, mType string, mName string, mValue interface{}) { res, err := client.R(). SetHeader("Content-Type", "text/plain"). @@ -102,8 +119,6 @@ func sendMetric(client *resty.Client, mType string, mName string, mValue interfa if res.StatusCode() != http.StatusOK { log.Error().Msgf("Server returned non-OK status for %s/%s: %d %s", mType, mName, res.StatusCode(), res.String()) - } else { - log.Debug().Msgf("Metric %s/%s sent successfully. Status: %d", mType, mName, res.StatusCode()) } } diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index 8c7d26f..1a9a842 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -55,10 +55,6 @@ func NewGauge(res http.ResponseWriter, name, value string) (mtr.Metric, error) { return mtr.NewGauge(name, val), nil } -func HandleUnknownMetric(res http.ResponseWriter) { - http.Error(res, "unknown type metric!", http.StatusBadRequest) -} - func GetMetric(storage ms.Collector) http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { log.Debug().Msgf("Incoming GET request: %s %s", req.Method, req.URL.Path) @@ -99,23 +95,40 @@ func GetMetric(storage ms.Collector) http.HandlerFunc { func GetAllMetrics(storage ms.Collector) http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { - gauges, counters := storage.GetAllMetrics() + allMetrics := storage.GetAllMetrics() var metricsToTable []MetricTable - for name, value := range gauges { - metricsToTable = append(metricsToTable, MetricTable{ - Name: name, - Type: mtr.GaugeType, - Value: strconv.FormatFloat(value, 'f', -1, 64), - }) - } - for name, value := range counters { - metricsToTable = append(metricsToTable, MetricTable{ - Name: name, - Type: mtr.CounterType, - Value: strconv.FormatInt(value, 10), - }) + for mType, innerMap := range allMetrics { + for mName, metric := range innerMap { + var valStr string + + switch metric.Type() { + case mtr.GaugeType: + val, ok := metric.Value().(float64) + if !ok { + log.Error().Str("metric_name", mName).Str("metric_type", mType). + Msg("Invalid metric value type") + continue + } + valStr = strconv.FormatFloat(val, 'f', -1, 64) + + case mtr.CounterType: + val, ok := metric.Value().(int64) + if !ok { + log.Error().Str("metric_name", mName).Str("metric_type", mType). + Msg("Invalid metric value type") + continue + } + valStr = strconv.FormatInt(val, 10) + } + + metricsToTable = append(metricsToTable, MetricTable{ + Name: mName, + Type: mType, + Value: valStr, + }) + } } const htmlTemplate = ` @@ -190,7 +203,7 @@ func UpdateMetric(storage ms.Collector) http.HandlerFunc { case mtr.CounterType: metric, err = NewCounter(res, mName, mValue) default: - HandleUnknownMetric(res) + http.Error(res, "unknown type metric!", http.StatusBadRequest) return } diff --git a/internal/mem-storage/storage.go b/internal/mem-storage/storage.go index 12f006b..54f7140 100644 --- a/internal/mem-storage/storage.go +++ b/internal/mem-storage/storage.go @@ -8,92 +8,67 @@ import ( ) type Collector interface { - GetMetric(mType, mName string) (interface{}, bool) - GetAllMetrics() (map[string]float64, map[string]int64) + GetMetric(mType, mName string) (any, bool) + GetAllMetrics() map[string]map[string]metrics.Metric UpdateMetric(mtr metrics.Metric) error } - type MemStorage struct { mutex sync.RWMutex - Gauges map[string]float64 - Counters map[string]int64 + storage map[string]map[string]metrics.Metric } func NewMemStorage() *MemStorage { return &MemStorage{ - Gauges: make(map[string]float64), - Counters: make(map[string]int64), + storage: make(map[string]map[string]metrics.Metric), } } func (ms *MemStorage) UpdateMetric(metric metrics.Metric) error { - switch metric.Type() { - case metrics.CounterType: - { - val, ok := metric.Value().(int64) - if !ok { - return metrics.ErrInvalidValueType - } - - ms.UpdateCounter(metric, val) - return nil - } - case metrics.GaugeType: - { - val, ok := metric.Value().(float64) - if !ok { - return metrics.ErrInvalidValueType - } - - ms.UpdateGauge(metric, val) - return nil - } - default: - return metrics.ErrInvalidMetricsType - } -} - -func (ms *MemStorage) UpdateGauge(metric metrics.Metric, value float64) { ms.mutex.Lock() defer ms.mutex.Unlock() - ms.Gauges[metric.Name()] = value -} -func (ms *MemStorage) UpdateCounter(metric metrics.Metric, value int64) { - ms.mutex.Lock() - defer ms.mutex.Unlock() - ms.Counters[metric.Name()] += value + mType := metric.Type() + mName := metric.Name() + + if _, ok := ms.storage[mType]; !ok { + ms.storage[mType] = make(map[string]metrics.Metric) + } + + if oldMetric, ok := ms.storage[mType][mName]; ok { + err := oldMetric.Update(metric) + if err != nil { + return err + } + } else { + ms.storage[mType][mName] = metric + } + + return nil } -func (ms *MemStorage) GetMetric(mType, mName string) (interface{}, bool) { +func (ms *MemStorage) GetMetric(mType, mName string) (any, bool) { ms.mutex.RLock() defer ms.mutex.RUnlock() - switch mType { - case metrics.GaugeType: - { - if val, ok := ms.Gauges[mName]; ok { - return val, true - } - } - case metrics.CounterType: - { - if val, ok := ms.Counters[mName]; ok { - return val, true - } - } + if metric, ok := ms.storage[mType][mName]; ok { + return metric.Value(), true } return nil, false } -func (ms *MemStorage) GetAllMetrics() (map[string]float64, map[string]int64) { +func (ms *MemStorage) GetAllMetrics() map[string]map[string]metrics.Metric { ms.mutex.RLock() defer ms.mutex.RUnlock() - gaugesCopy := maps.Clone(ms.Gauges) - countersCopy := maps.Clone(ms.Counters) + result := make(map[string]map[string]metrics.Metric, len(ms.storage)) - return gaugesCopy, countersCopy + for mType, innerMap := range ms.storage { + innerCopy := maps.Clone(innerMap) + result[mType] = innerCopy + } + + return result } + diff --git a/internal/metrics/counter.go b/internal/metrics/counter.go index 6d4677f..cc38dcf 100644 --- a/internal/metrics/counter.go +++ b/internal/metrics/counter.go @@ -20,11 +20,26 @@ func (c *counter) Type() string { return CounterType } -func (c *counter) Value() interface{} { +func (c *counter) Value() any { return c.value } -func (c *counter) SetValue(v interface{}) error { +func (c *counter) Update(mtr Metric) error { + if mtr.Type() != c.Type() { + return ErrInvalidMetricsType + } + + mtrValue, ok := mtr.Value().(int64) + if !ok { + return ErrInvalidValueType + } + + c.value += mtrValue + + return nil +} + +func (c *counter) SetValue(v any) error { val, ok := v.(int64) if !ok { return ErrInvalidValueType diff --git a/internal/metrics/gauge.go b/internal/metrics/gauge.go index 38ff84d..734d765 100644 --- a/internal/metrics/gauge.go +++ b/internal/metrics/gauge.go @@ -20,11 +20,26 @@ func (g *gauge) Type() string { return GaugeType } -func (g *gauge) Value() interface{} { +func (g *gauge) Value() any { return g.value } -func (g *gauge) SetValue(v interface{}) error { +func (g *gauge) Update(mtr Metric) error { + if mtr.Type() != g.Type() { + return ErrInvalidMetricsType + } + + mtrValue, ok := mtr.Value().(float64) + if !ok { + return ErrInvalidValueType + } + + g.value = mtrValue + + return nil +} + +func (g *gauge) SetValue(v any) error { val, ok := v.(float64) if !ok { return ErrInvalidValueType diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index eae8bf8..b58f416 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -5,9 +5,10 @@ import ( ) type Metric interface { - Value() interface{} + Value() any Name() string Type() string + Update(mtr Metric) error } const ( From db279a7cf163a20c39ce9d70438d844f6c32c3aa Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Mon, 7 Jul 2025 00:28:33 +0300 Subject: [PATCH 040/126] empty --- internal/handlers/agent/agent.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 6288aaf..683522a 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -101,7 +101,6 @@ func sendAllMetrics(client *resty.Client, storage *ms.MemStorage) { } } - func sendMetric(client *resty.Client, mType string, mName string, mValue interface{}) { res, err := client.R(). SetHeader("Content-Type", "text/plain"). From 0c0f028f518a64c2d16753d2836f6bc102be8c72 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Mon, 7 Jul 2025 15:30:54 +0300 Subject: [PATCH 041/126] rewrote the storage logic --- internal/handlers/agent/agent.go | 49 +++------------ internal/handlers/server/server.go | 82 +++++++++++++------------ internal/handlers/server/server_test.go | 5 +- internal/mem-storage/storage.go | 52 ++++++++++------ internal/metrics/counter.go | 6 +- internal/metrics/gauge.go | 6 +- internal/metrics/metrics.go | 2 +- 7 files changed, 94 insertions(+), 108 deletions(-) diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 683522a..69369be 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -15,23 +15,6 @@ import ( rt "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/runtime-stats" ) -func toFloat64(val any) (float64, bool) { - switch v := val.(type) { - case float64: - return v, true - case uint64: - return float64(v), true - case uint32: - return float64(v), true - case int: - return float64(v), true - case int64: - return float64(v), true - default: - return 0, false - } -} - func UpdateAllMetrics(storage *ms.MemStorage) { var memStats runtime.MemStats runtime.ReadMemStats(&memStats) @@ -39,36 +22,20 @@ func UpdateAllMetrics(storage *ms.MemStorage) { for _, stat := range rt.MemRuntimeStats { val := stat.Get(&memStats) - switch stat.Type { - case mtr.GaugeType: - value, ok := toFloat64(val) - if !ok { - log.Error(). - Str("metric", stat.Name). - Str("type", fmt.Sprintf("%T", val)). - Msg("Failed to convert metric value to float64") - continue - } - - if err := storage.UpdateMetric(mtr.NewGauge(stat.Name, value)); err != nil { - log.Error(). - Err(err). - Str("metric", stat.Name). - Msg("Failed to update metric") - } - default: - log.Error(). - Str("metric", stat.Name). - Str("type", fmt.Sprintf("%T", val)). - Msg("Unsupported metric type") + if err := storage.UpdateMetric(stat.Type, stat.Name, val); err != nil { + log.Error(). + Err(err). + Str("metric", stat.Name). + Str("type", fmt.Sprintf("%T", val)). + Msg("Failed to update runtime metric") } } - if err := storage.UpdateMetric(mtr.NewCounter("PollCount", 1)); err != nil { + if err := storage.UpdateMetric(mtr.CounterType, "PollCount", 1); err != nil { log.Error().Msgf("Failed to update PollCount metric: %v", err) } - if err := storage.UpdateMetric(mtr.NewGauge("RandomValue", rand.Float64())); err != nil { + if err := storage.UpdateMetric(mtr.GaugeType, "RandomValue", rand.Float64()); err != nil { log.Error().Msgf("Failed to update RandomValue metric: %v", err) } } diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index 1a9a842..81318da 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -33,26 +33,23 @@ func NewRouter(storage ms.Collector) http.Handler { return r } -// FIXME other function name -func NewCounter(res http.ResponseWriter, name, value string) (mtr.Metric, error) { - val, err := strconv.ParseInt(value, 10, 64) - if err != nil { - http.Error(res, "invalid value metric", http.StatusBadRequest) - return nil, err - } - - return mtr.NewCounter(name, val), nil -} - -// FIXME other function name -func NewGauge(res http.ResponseWriter, name, value string) (mtr.Metric, error) { - val, err := strconv.ParseFloat(value, 64) - if err != nil { - http.Error(res, "invalid value metric", http.StatusBadRequest) - return nil, err +func ConvertByType(mType, mValue string) (any, error) { + switch mType { + case mtr.GaugeType: + if val, err := strconv.ParseFloat(mValue, 64); err != nil { + return nil, fmt.Errorf("convert gauge value %s: %w", mValue, err) + } else { + return val, nil + } + case mtr.CounterType: + if val, err := strconv.ParseInt(mValue, 10, 64); err != nil { + return nil, fmt.Errorf("convert counter value %s: %w", mValue, err) + } else { + return val, nil + } + default: + return nil, fmt.Errorf("unknown metric type: %s", mType) } - - return mtr.NewGauge(name, val), nil } func GetMetric(storage ms.Collector) http.HandlerFunc { @@ -77,7 +74,7 @@ func GetMetric(storage ms.Collector) http.HandlerFunc { valueStr = strconv.FormatInt(v, 10) default: http.Error(res, "an unexpected type of metric", http.StatusInternalServerError) - return //FIXME fix this case + return } res.Header().Set("Content-Type", "text/plain") @@ -181,43 +178,52 @@ func GetAllMetrics(storage ms.Collector) http.HandlerFunc { func UpdateMetric(storage ms.Collector) http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { - log.Debug().Msgf("Incoming POST request: %s %s", req.Method, req.URL.Path) + log.Debug(). + Str("method", req.Method). + Str("url", req.URL.Path). + Msg("Incoming POST request") mType := chi.URLParam(req, "mType") mName := chi.URLParam(req, "mName") mValue := chi.URLParam(req, "mValue") - log.Debug().Msgf("Parsed metric: type=%s, name=%s, value=%s", mType, mName, mValue) + log.Debug(). + Str("metric_type", mType). + Str("metric_name", mName). + Str("metric_value", mValue). + Msg("Parsed metric") if mName == "" { - log.Error().Msgf("the metric name is not specified") + log.Error(). + Str("metric_type", mType). + Msg("Metric name is not specified") + http.Error(res, "the metric name is not specified", http.StatusBadRequest) return } - var metric mtr.Metric - var err error + val, err := ConvertByType(mType, mValue) + if err != nil { + log.Error(). + Err(err). + Str("metric_type", mType). + Str("metric_value", mValue). + Msg("Failed to convert metric value") - switch mType { - case mtr.GaugeType: - metric, err = NewGauge(res, mName, mValue) - case mtr.CounterType: - metric, err = NewCounter(res, mName, mValue) - default: - http.Error(res, "unknown type metric!", http.StatusBadRequest) + http.Error(res, fmt.Sprintf("invalid metric value: %v", err), http.StatusBadRequest) return } - if err != nil { + if err := storage.UpdateMetric(mType, mName, val); err != nil { http.Error(res, err.Error(), http.StatusBadRequest) return } - if err := storage.UpdateMetric(metric); err != nil { - http.Error(res, err.Error(), http.StatusBadRequest) - return - } + log.Info(). + Str("metric_type", mType). + Str("metric_name", mName). + Interface("value", val). + Msg("Metric updated successfully") - log.Info().Msgf("Metric updated successfully: %s %s = %s", mType, mName, mValue) res.WriteHeader(http.StatusOK) } } diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index 09b9ab3..556a30c 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -82,13 +82,14 @@ func TestUpdateMetric(t *testing.T) { func TestGetMetric(t *testing.T) { storage := ms.NewMemStorage() - if err := storage.UpdateMetric(mtr.NewGauge("cpu_usage", 75.5)); err != nil { + if err := storage.UpdateMetric(mtr.GaugeType, "cpu_usage", 75.5); err != nil { log.Error().Msgf("Failed to update metric cpu_usage: %v", err) } - if err := storage.UpdateMetric(mtr.NewCounter("requests_total", 100)); err != nil { + if err := storage.UpdateMetric(mtr.CounterType, "requests_total", int64(100)); err != nil { log.Error().Msgf("Failed to update metric requests_total: %v", err) } + router := NewRouter(storage) tests := []struct { diff --git a/internal/mem-storage/storage.go b/internal/mem-storage/storage.go index 54f7140..2ad0232 100644 --- a/internal/mem-storage/storage.go +++ b/internal/mem-storage/storage.go @@ -1,49 +1,62 @@ package memstorage import ( - "sync" "maps" + "sync" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" ) type Collector interface { GetMetric(mType, mName string) (any, bool) - GetAllMetrics() map[string]map[string]metrics.Metric - UpdateMetric(mtr metrics.Metric) error + GetAllMetrics() map[string]map[string]mtr.Metric + UpdateMetric(mType, mName string, mValue any) error } type MemStorage struct { - mutex sync.RWMutex - storage map[string]map[string]metrics.Metric + mutex sync.RWMutex + storage map[string]map[string]mtr.Metric } func NewMemStorage() *MemStorage { return &MemStorage{ - storage: make(map[string]map[string]metrics.Metric), + storage: make(map[string]map[string]mtr.Metric), } } -func (ms *MemStorage) UpdateMetric(metric metrics.Metric) error { +func (ms *MemStorage) UpdateMetric(mType, mName string, mValue any) error { ms.mutex.Lock() defer ms.mutex.Unlock() - mType := metric.Type() - mName := metric.Name() - if _, ok := ms.storage[mType]; !ok { - ms.storage[mType] = make(map[string]metrics.Metric) + ms.storage[mType] = make(map[string]mtr.Metric) } if oldMetric, ok := ms.storage[mType][mName]; ok { - err := oldMetric.Update(metric) - if err != nil { - return err + return oldMetric.Update(mType, mName, mValue) + } + + var newMetric mtr.Metric + + switch mType { + case mtr.GaugeType: + value, ok := mValue.(float64) + if !ok { + return mtr.ErrInvalidValueType + } + newMetric = mtr.NewGauge(mName, value) + case mtr.CounterType: + value, ok := mValue.(int64) + if !ok { + return mtr.ErrInvalidValueType } - } else { - ms.storage[mType][mName] = metric + newMetric = mtr.NewCounter(mName, value) + default: + return mtr.ErrInvalidMetricsType } + ms.storage[mType][mName] = newMetric + return nil } @@ -58,11 +71,11 @@ func (ms *MemStorage) GetMetric(mType, mName string) (any, bool) { return nil, false } -func (ms *MemStorage) GetAllMetrics() map[string]map[string]metrics.Metric { +func (ms *MemStorage) GetAllMetrics() map[string]map[string]mtr.Metric { ms.mutex.RLock() defer ms.mutex.RUnlock() - result := make(map[string]map[string]metrics.Metric, len(ms.storage)) + result := make(map[string]map[string]mtr.Metric, len(ms.storage)) for mType, innerMap := range ms.storage { innerCopy := maps.Clone(innerMap) @@ -71,4 +84,3 @@ func (ms *MemStorage) GetAllMetrics() map[string]map[string]metrics.Metric { return result } - diff --git a/internal/metrics/counter.go b/internal/metrics/counter.go index cc38dcf..5f3e92d 100644 --- a/internal/metrics/counter.go +++ b/internal/metrics/counter.go @@ -24,12 +24,12 @@ func (c *counter) Value() any { return c.value } -func (c *counter) Update(mtr Metric) error { - if mtr.Type() != c.Type() { +func (c *counter) Update(mType, mName string, mValue any) error { + if mType != c.Type() { return ErrInvalidMetricsType } - mtrValue, ok := mtr.Value().(int64) + mtrValue, ok := mValue.(int64) if !ok { return ErrInvalidValueType } diff --git a/internal/metrics/gauge.go b/internal/metrics/gauge.go index 734d765..e808322 100644 --- a/internal/metrics/gauge.go +++ b/internal/metrics/gauge.go @@ -24,12 +24,12 @@ func (g *gauge) Value() any { return g.value } -func (g *gauge) Update(mtr Metric) error { - if mtr.Type() != g.Type() { +func (g *gauge) Update(mType, mName string, mValue any) error { + if mType != g.Type() { return ErrInvalidMetricsType } - mtrValue, ok := mtr.Value().(float64) + mtrValue, ok := mValue.(float64) if !ok { return ErrInvalidValueType } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index b58f416..0a1ec77 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -8,7 +8,7 @@ type Metric interface { Value() any Name() string Type() string - Update(mtr Metric) error + Update(mType, mName string, mValue any) error } const ( From 74cfb61d60f7a6031965fb6dad5abb872e4c5587 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Wed, 9 Jul 2025 02:35:30 +0300 Subject: [PATCH 042/126] ufter review for merge to main --- internal/handlers/agent/agent.go | 37 ++++++------- internal/handlers/server/server.go | 84 +++++++++++++++--------------- internal/mem-storage/storage.go | 20 ++++--- internal/metrics/counter.go | 6 +-- internal/metrics/gauge.go | 6 +-- internal/metrics/metrics.go | 2 +- 6 files changed, 76 insertions(+), 79 deletions(-) diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 69369be..757a03a 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -43,27 +43,28 @@ func UpdateAllMetrics(storage *ms.MemStorage) { func sendAllMetrics(client *resty.Client, storage *ms.MemStorage) { allMetrics := storage.GetAllMetrics() - for mType, innerMap := range allMetrics { - for mName, metric := range innerMap { - switch mType { - case mtr.GaugeType: - val, ok := metric.Value().(float64) - if !ok { - log.Error().Str("metric_name", mName).Str("metric_type", mType). + for _, metric := range allMetrics { + mType := metric.Type() + mName := metric.Name() + + switch mType { + case mtr.GaugeType: + val, ok := metric.Value().(float64) + if !ok { + log.Error().Str("metric_name", mName).Str("metric_type", mType). Msg("Invalid metric value type") - continue - } - sendMetric(client, mtr.GaugeType, mName, val) - - case mtr.CounterType: - val, ok := metric.Value().(int64) - if !ok { - log.Error().Str("metric_name", mName).Str("metric_type", mType). + continue + } + sendMetric(client, mtr.GaugeType, mName, val) + + case mtr.CounterType: + val, ok := metric.Value().(int64) + if !ok { + log.Error().Str("metric_name", mName).Str("metric_type", mType). Msg("Invalid metric value type") - continue - } - sendMetric(client, mtr.CounterType, mName, val) + continue } + sendMetric(client, mtr.CounterType, mName, val) } } } diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index 81318da..bdaff27 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -35,20 +35,20 @@ func NewRouter(storage ms.Collector) http.Handler { func ConvertByType(mType, mValue string) (any, error) { switch mType { - case mtr.GaugeType: - if val, err := strconv.ParseFloat(mValue, 64); err != nil { - return nil, fmt.Errorf("convert gauge value %s: %w", mValue, err) - } else { - return val, nil - } - case mtr.CounterType: - if val, err := strconv.ParseInt(mValue, 10, 64); err != nil { - return nil, fmt.Errorf("convert counter value %s: %w", mValue, err) - } else { - return val, nil - } - default: - return nil, fmt.Errorf("unknown metric type: %s", mType) + case mtr.GaugeType: + if val, err := strconv.ParseFloat(mValue, 64); err != nil { + return nil, fmt.Errorf("convert gauge value %s: %w", mValue, err) + } else { + return val, nil + } + case mtr.CounterType: + if val, err := strconv.ParseInt(mValue, 10, 64); err != nil { + return nil, fmt.Errorf("convert counter value %s: %w", mValue, err) + } else { + return val, nil + } + default: + return nil, fmt.Errorf("unknown metric type: %s", mType) } } @@ -96,36 +96,36 @@ func GetAllMetrics(storage ms.Collector) http.HandlerFunc { var metricsToTable []MetricTable - for mType, innerMap := range allMetrics { - for mName, metric := range innerMap { - var valStr string - - switch metric.Type() { - case mtr.GaugeType: - val, ok := metric.Value().(float64) - if !ok { - log.Error().Str("metric_name", mName).Str("metric_type", mType). - Msg("Invalid metric value type") - continue - } - valStr = strconv.FormatFloat(val, 'f', -1, 64) - - case mtr.CounterType: - val, ok := metric.Value().(int64) - if !ok { - log.Error().Str("metric_name", mName).Str("metric_type", mType). - Msg("Invalid metric value type") - continue - } - valStr = strconv.FormatInt(val, 10) + for _, metric := range allMetrics { + var valStr string + mName := metric.Name() + mType := metric.Type() + + switch mType { + case mtr.GaugeType: + val, ok := metric.Value().(float64) + if !ok { + log.Error().Str("metric_name", mName).Str("metric_type", mType). + Msg("Invalid metric value type") + continue } - - metricsToTable = append(metricsToTable, MetricTable{ - Name: mName, - Type: mType, - Value: valStr, - }) + valStr = strconv.FormatFloat(val, 'f', -1, 64) + + case mtr.CounterType: + val, ok := metric.Value().(int64) + if !ok { + log.Error().Str("metric_name", mName).Str("metric_type", mType). + Msg("Invalid metric value type") + continue + } + valStr = strconv.FormatInt(val, 10) } + + metricsToTable = append(metricsToTable, MetricTable{ + Name: mName, + Type: mType, + Value: valStr, + }) } const htmlTemplate = ` diff --git a/internal/mem-storage/storage.go b/internal/mem-storage/storage.go index 2ad0232..7374f2e 100644 --- a/internal/mem-storage/storage.go +++ b/internal/mem-storage/storage.go @@ -1,7 +1,6 @@ package memstorage import ( - "maps" "sync" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" @@ -9,7 +8,7 @@ import ( type Collector interface { GetMetric(mType, mName string) (any, bool) - GetAllMetrics() map[string]map[string]mtr.Metric + GetAllMetrics() []mtr.Metric UpdateMetric(mType, mName string, mValue any) error } @@ -33,7 +32,7 @@ func (ms *MemStorage) UpdateMetric(mType, mName string, mValue any) error { } if oldMetric, ok := ms.storage[mType][mName]; ok { - return oldMetric.Update(mType, mName, mValue) + return oldMetric.Update(mName, mValue) } var newMetric mtr.Metric @@ -71,15 +70,20 @@ func (ms *MemStorage) GetMetric(mType, mName string) (any, bool) { return nil, false } -func (ms *MemStorage) GetAllMetrics() map[string]map[string]mtr.Metric { +func (ms *MemStorage) GetAllMetrics() []mtr.Metric { ms.mutex.RLock() defer ms.mutex.RUnlock() - result := make(map[string]map[string]mtr.Metric, len(ms.storage)) + total := 0 + for _, innerMap := range ms.storage { + total += len(innerMap) + } + result := make([]mtr.Metric, 0, total) - for mType, innerMap := range ms.storage { - innerCopy := maps.Clone(innerMap) - result[mType] = innerCopy + for _, innerMap := range ms.storage { + for _, metric := range innerMap { + result = append(result, metric) + } } return result diff --git a/internal/metrics/counter.go b/internal/metrics/counter.go index 5f3e92d..09b06bd 100644 --- a/internal/metrics/counter.go +++ b/internal/metrics/counter.go @@ -24,11 +24,7 @@ func (c *counter) Value() any { return c.value } -func (c *counter) Update(mType, mName string, mValue any) error { - if mType != c.Type() { - return ErrInvalidMetricsType - } - +func (c *counter) Update(mName string, mValue any) error { mtrValue, ok := mValue.(int64) if !ok { return ErrInvalidValueType diff --git a/internal/metrics/gauge.go b/internal/metrics/gauge.go index e808322..80efe1a 100644 --- a/internal/metrics/gauge.go +++ b/internal/metrics/gauge.go @@ -24,11 +24,7 @@ func (g *gauge) Value() any { return g.value } -func (g *gauge) Update(mType, mName string, mValue any) error { - if mType != g.Type() { - return ErrInvalidMetricsType - } - +func (g *gauge) Update(mName string, mValue any) error { mtrValue, ok := mValue.(float64) if !ok { return ErrInvalidValueType diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 0a1ec77..5718b26 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -8,7 +8,7 @@ type Metric interface { Value() any Name() string Type() string - Update(mType, mName string, mValue any) error + Update(mName string, mValue any) error } const ( From 74abce77913238ad4dc9078942c03bcfd73f3695 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Sat, 5 Jul 2025 21:51:34 +0300 Subject: [PATCH 043/126] wrote a log for the server --- internal/handlers/agent/agent.go | 5 ++- internal/handlers/server/logging.go | 62 +++++++++++++++++++++++++++++ internal/handlers/server/server.go | 19 +-------- 3 files changed, 68 insertions(+), 18 deletions(-) create mode 100644 internal/handlers/server/logging.go diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 757a03a..0bdca24 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -80,7 +80,10 @@ func sendMetric(client *resty.Client, mType string, mName string, mValue interfa Post("update/{mType}/{mName}/{mValue}") if err != nil { - log.Error().Msgf("Error creating a request for %s: %v", mName, err) + log.Error(). + Err(err). + Str("metric", mName). + Msg("Error sending request") return } diff --git a/internal/handlers/server/logging.go b/internal/handlers/server/logging.go new file mode 100644 index 0000000..2169501 --- /dev/null +++ b/internal/handlers/server/logging.go @@ -0,0 +1,62 @@ +package server + +import ( + "net/http" + "time" + + "github.com/rs/zerolog/log" +) + +type ( + loggingResponseData struct { + status int + size int + } + + loggingResponseWriter struct { + http.ResponseWriter + responseData *loggingResponseData + } +) + +func (res *loggingResponseWriter) Write(body []byte) (int, error) { + size, err := res.ResponseWriter.Write(body) + res.responseData.size += size + + return size, err +} + +func (res *loggingResponseWriter) WriteHeader(statusCode int) { + res.ResponseWriter.WriteHeader(statusCode) + res.responseData.status = statusCode +} + +func WithLogging(h http.Handler) http.Handler { + logfn := func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + + responseData := &loggingResponseData{ + status: 0, + size: 0, + } + lw := loggingResponseWriter{ + ResponseWriter: w, + responseData: responseData, + } + + h.ServeHTTP(&lw, r) + + duration := time.Since(start) + + log.Info(). + Str("uri", r.RequestURI). + Str("method", r.Method). + Int("status", responseData.status). + Dur("duration", duration). + Int("size", responseData.size). + Msg("new request") + + } + + return http.HandlerFunc(logfn) +} diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index bdaff27..139ff16 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -22,6 +22,8 @@ type MetricTable struct { func NewRouter(storage ms.Collector) http.Handler { r := chi.NewRouter() + r.Use(WithLogging) + r.Route("/", func(r chi.Router) { r.Get("/", GetAllMetrics(storage)) r.Route("/", func(r chi.Router) { @@ -54,12 +56,8 @@ func ConvertByType(mType, mValue string) (any, error) { func GetMetric(storage ms.Collector) http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { - log.Debug().Msgf("Incoming GET request: %s %s", req.Method, req.URL.Path) - mType := chi.URLParam(req, "mType") mName := chi.URLParam(req, "mName") - log.Debug().Msgf("Incoming request for metric: Type=%s, Name=%s", mType, mName) - value, found := storage.GetMetric(mType, mName) if !found { http.Error(res, fmt.Sprintf("Metric %s was not found", mName), http.StatusNotFound) @@ -85,8 +83,6 @@ func GetMetric(storage ms.Collector) http.HandlerFunc { log.Error().Msgf("Failed to write response: %v", err) } - - log.Debug().Msg("the metric has been send") } } @@ -171,26 +167,15 @@ func GetAllMetrics(storage ms.Collector) http.HandlerFunc { log.Error().Msgf("failed complete template: %v", err) } - log.Debug().Msg("the metrics has been send") res.WriteHeader(http.StatusOK) } } func UpdateMetric(storage ms.Collector) http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { - log.Debug(). - Str("method", req.Method). - Str("url", req.URL.Path). - Msg("Incoming POST request") - mType := chi.URLParam(req, "mType") mName := chi.URLParam(req, "mName") mValue := chi.URLParam(req, "mValue") - log.Debug(). - Str("metric_type", mType). - Str("metric_name", mName). - Str("metric_value", mValue). - Msg("Parsed metric") if mName == "" { log.Error(). From f4406dfb5586df33a0f142c9802d46c1601f83fb Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Mon, 7 Jul 2025 22:56:35 +0300 Subject: [PATCH 044/126] first easyjson-JSON version --- go.mod | 2 + go.sum | 4 + internal/handlers/agent/agent.go | 42 ++--- internal/handlers/server/server.go | 108 ++++++++++++ internal/handlers/server/server_easyjson.go | 180 ++++++++++++++++++++ internal/mem-storage/storage.go | 4 + internal/metrics/counter.go | 16 +- internal/metrics/gauge.go | 16 +- internal/metrics/metrics.go | 2 +- pkg/runtime-stats/runtime-stats.go | 57 ++++--- 10 files changed, 366 insertions(+), 65 deletions(-) create mode 100644 internal/handlers/server/server_easyjson.go diff --git a/go.mod b/go.mod index 7b7d3b1..6bd5e6a 100644 --- a/go.mod +++ b/go.mod @@ -6,12 +6,14 @@ require ( github.com/caarlos0/env/v6 v6.10.1 github.com/go-chi/chi/v5 v5.2.2 github.com/go-resty/resty/v2 v2.16.5 + github.com/mailru/easyjson v0.9.0 github.com/rs/zerolog v1.34.0 github.com/spf13/cobra v1.9.1 ) require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/spf13/pflag v1.0.6 // indirect diff --git a/go.sum b/go.sum index 6ed1198..265104c 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,10 @@ github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 0bdca24..e7c5c26 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -9,6 +9,7 @@ import ( "github.com/go-resty/resty/v2" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" @@ -29,9 +30,11 @@ func UpdateAllMetrics(storage *ms.MemStorage) { Str("type", fmt.Sprintf("%T", val)). Msg("Failed to update runtime metric") } + + log.Debug().Msgf("update metric %s", stat.Name) } - if err := storage.UpdateMetric(mtr.CounterType, "PollCount", 1); err != nil { + if err := storage.UpdateMetric(mtr.CounterType, "PollCount", int64(1)); err != nil { log.Error().Msgf("Failed to update PollCount metric: %v", err) } @@ -69,26 +72,25 @@ func sendAllMetrics(client *resty.Client, storage *ms.MemStorage) { } } -func sendMetric(client *resty.Client, mType string, mName string, mValue interface{}) { - res, err := client.R(). - SetHeader("Content-Type", "text/plain"). - SetPathParams(map[string]string{ - "mType": mType, - "mName": mName, - "mValue": fmt.Sprintf("%v", mValue), - }). - Post("update/{mType}/{mName}/{mValue}") - - if err != nil { - log.Error(). - Err(err). - Str("metric", mName). - Msg("Error sending request") - return +func sendMetric(client *resty.Client, metricJSON *server.Metrics) { + backoffSchedule := []time.Duration{ + 100 * time.Millisecond, + 500 * time.Millisecond, + 1 * time.Second, } - if res.StatusCode() != http.StatusOK { - log.Error().Msgf("Server returned non-OK status for %s/%s: %d %s", mType, mName, res.StatusCode(), res.String()) + for _, backoff := range backoffSchedule { + res, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetBody(metricJSON). + Post("update/") + + if err != nil || res.StatusCode() != http.StatusOK { + } else { + break + } + + time.Sleep(backoff) } } @@ -103,7 +105,7 @@ func CollectionLoop(storage *ms.MemStorage, interval time.Duration) { func ReportLoop(client *resty.Client, storage *ms.MemStorage, interval time.Duration) { log.Debug().Msg("reportLoop ...") for { - time.Sleep(interval) sendAllMetrics(client, storage) + time.Sleep(interval) } } diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index 139ff16..e1d1b45 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -7,6 +7,7 @@ import ( "strconv" "github.com/go-chi/chi/v5" + "github.com/mailru/easyjson" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" @@ -27,6 +28,8 @@ func NewRouter(storage ms.Collector) http.Handler { r.Route("/", func(r chi.Router) { r.Get("/", GetAllMetrics(storage)) r.Route("/", func(r chi.Router) { + r.Post("/update/", UpdateMetricsHandlerJSON(storage)) + r.Post("/value/", GetMetricsHandlerJSON(storage)) r.Get("/value/{mType}/{mName}", GetMetric(storage)) r.Post("/update/{mType}/{mName}/{mValue}", UpdateMetric(storage)) }) @@ -212,3 +215,108 @@ func UpdateMetric(storage ms.Collector) http.HandlerFunc { res.WriteHeader(http.StatusOK) } } + +type Metrics struct { + ID string `json:"id"` + MType string `json:"type"` + Delta *int64 `json:"delta,omitempty"` + Value *float64 `json:"value,omitempty"` +} + +func FillMetricValueFromStorage(storage ms.Collector, metric *Metrics) bool { + value, ok := storage.GetMetric(metric.MType, metric.ID) + if !ok { + return false + } + + switch v := value.(type) { + case float64: + metric.Value = &v + case int64: + metric.Delta = &v + default: + log.Error(). + Str("metricType", metric.MType). + Str("metricName", metric.ID). + Msg("unsupported metric type") + + return false + } + + return true +} + +func GetMetricsHandlerJSON(storage ms.Collector) http.HandlerFunc { + return func(resp http.ResponseWriter, req *http.Request) { + var metric Metrics + + if req.Header.Get("Content-Type") != "application/json" { + http.Error(resp, "Content-Type must be application/json", http.StatusUnsupportedMediaType) + return + } + + if err := easyjson.UnmarshalFromReader(req.Body, &metric); err != nil { + http.Error(resp, fmt.Sprintf("invalid json body: %v", err), http.StatusBadRequest) + return + } + + if !FillMetricValueFromStorage(storage, &metric) { + http.Error(resp, fmt.Sprintf("metric %s not found", metric.ID), http.StatusNotFound) + return + } + + started, n, err := easyjson.MarshalToHTTPResponseWriter(&metric, resp) + if err != nil { + if !started { + http.Error(resp, "failed to encode json", http.StatusInternalServerError) + } else { + log.Error().Err(err). + Int("written_bytes", n). + Msg("error while writing json response") + } + } + } +} + +func UpdateMetricsHandlerJSON(storage ms.Collector) http.HandlerFunc { + return func(resp http.ResponseWriter, req *http.Request) { + var metric Metrics + + if req.Header.Get("Content-Type") != "application/json" { + http.Error(resp, "Content-Type must be application/json", http.StatusUnsupportedMediaType) + return + } + + if err := easyjson.UnmarshalFromReader(req.Body, &metric); err != nil { + http.Error(resp, fmt.Sprintf("invalid json body: %v", err), http.StatusBadRequest) + return + } + + var value any + if metric.MType == mtr.GaugeType { + value = *metric.Value + } else { + value = *metric.Delta + } + + if err := storage.UpdateMetric(metric.MType, metric.ID, value); err != nil { + http.Error(resp, fmt.Sprintf("invalid update metric %s: %v", metric.ID, err), http.StatusBadRequest) + } + + if !FillMetricValueFromStorage(storage, &metric) { + http.Error(resp, fmt.Sprintf("metric %s not found", metric.ID), http.StatusNotFound) + return + } + + started, n, err := easyjson.MarshalToHTTPResponseWriter(&metric, resp) + if err != nil { + if !started { + http.Error(resp, "failed to encode json", http.StatusInternalServerError) + } else { + log.Error().Err(err). + Int("written_bytes", n). + Msg("error while writing json response") + } + } + } +} diff --git a/internal/handlers/server/server_easyjson.go b/internal/handlers/server/server_easyjson.go new file mode 100644 index 0000000..094fb13 --- /dev/null +++ b/internal/handlers/server/server_easyjson.go @@ -0,0 +1,180 @@ +// Code generated by easyjson for marshaling/unmarshaling. DO NOT EDIT. + +package server + +import ( + json "encoding/json" + easyjson "github.com/mailru/easyjson" + jlexer "github.com/mailru/easyjson/jlexer" + jwriter "github.com/mailru/easyjson/jwriter" +) + +// suppress unused package warning +var ( + _ *json.RawMessage + _ *jlexer.Lexer + _ *jwriter.Writer + _ easyjson.Marshaler +) + +func easyjson22b57fa5DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalHandlersServer(in *jlexer.Lexer, out *Metrics) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "id": + out.ID = string(in.String()) + case "type": + out.MType = string(in.String()) + case "delta": + if in.IsNull() { + in.Skip() + out.Delta = nil + } else { + if out.Delta == nil { + out.Delta = new(int64) + } + *out.Delta = int64(in.Int64()) + } + case "value": + if in.IsNull() { + in.Skip() + out.Value = nil + } else { + if out.Value == nil { + out.Value = new(float64) + } + *out.Value = float64(in.Float64()) + } + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjson22b57fa5EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalHandlersServer(out *jwriter.Writer, in Metrics) { + out.RawByte('{') + first := true + _ = first + { + const prefix string = ",\"id\":" + out.RawString(prefix[1:]) + out.String(string(in.ID)) + } + { + const prefix string = ",\"type\":" + out.RawString(prefix) + out.String(string(in.MType)) + } + if in.Delta != nil { + const prefix string = ",\"delta\":" + out.RawString(prefix) + out.Int64(int64(*in.Delta)) + } + if in.Value != nil { + const prefix string = ",\"value\":" + out.RawString(prefix) + out.Float64(float64(*in.Value)) + } + out.RawByte('}') +} + +// MarshalJSON supports json.Marshaler interface +func (v Metrics) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + easyjson22b57fa5EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalHandlersServer(&w, v) + return w.Buffer.BuildBytes(), w.Error +} + +// MarshalEasyJSON supports easyjson.Marshaler interface +func (v Metrics) MarshalEasyJSON(w *jwriter.Writer) { + easyjson22b57fa5EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalHandlersServer(w, v) +} + +// UnmarshalJSON supports json.Unmarshaler interface +func (v *Metrics) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + easyjson22b57fa5DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalHandlersServer(&r, v) + return r.Error() +} + +// UnmarshalEasyJSON supports easyjson.Unmarshaler interface +func (v *Metrics) UnmarshalEasyJSON(l *jlexer.Lexer) { + easyjson22b57fa5DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalHandlersServer(l, v) +} +func easyjson22b57fa5DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalHandlersServer1(in *jlexer.Lexer, out *MetricTable) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "Name": + out.Name = string(in.String()) + case "Type": + out.Type = string(in.String()) + case "Value": + out.Value = string(in.String()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjson22b57fa5EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalHandlersServer1(out *jwriter.Writer, in MetricTable) { + out.RawByte('{') + first := true + _ = first + { + const prefix string = ",\"Name\":" + out.RawString(prefix[1:]) + out.String(string(in.Name)) + } + { + const prefix string = ",\"Type\":" + out.RawString(prefix) + out.String(string(in.Type)) + } + { + const prefix string = ",\"Value\":" + out.RawString(prefix) + out.String(string(in.Value)) + } + out.RawByte('}') +} + + diff --git a/internal/mem-storage/storage.go b/internal/mem-storage/storage.go index 7374f2e..dee0d84 100644 --- a/internal/mem-storage/storage.go +++ b/internal/mem-storage/storage.go @@ -32,7 +32,11 @@ func (ms *MemStorage) UpdateMetric(mType, mName string, mValue any) error { } if oldMetric, ok := ms.storage[mType][mName]; ok { +<<<<<<< HEAD return oldMetric.Update(mName, mValue) +======= + return oldMetric.Update(mValue) +>>>>>>> c8d8784 (first easyjson-JSON version) } var newMetric mtr.Metric diff --git a/internal/metrics/counter.go b/internal/metrics/counter.go index 09b06bd..4c791f6 100644 --- a/internal/metrics/counter.go +++ b/internal/metrics/counter.go @@ -24,15 +24,13 @@ func (c *counter) Value() any { return c.value } -func (c *counter) Update(mName string, mValue any) error { - mtrValue, ok := mValue.(int64) - if !ok { - return ErrInvalidValueType - } - - c.value += mtrValue - - return nil +func (c *counter) Update(mValue any) error { + value, ok := mValue.(int64) + if !ok { + return ErrInvalidValueType + } + c.value += value + return nil } func (c *counter) SetValue(v any) error { diff --git a/internal/metrics/gauge.go b/internal/metrics/gauge.go index 80efe1a..04fce82 100644 --- a/internal/metrics/gauge.go +++ b/internal/metrics/gauge.go @@ -24,15 +24,13 @@ func (g *gauge) Value() any { return g.value } -func (g *gauge) Update(mName string, mValue any) error { - mtrValue, ok := mValue.(float64) - if !ok { - return ErrInvalidValueType - } - - g.value = mtrValue - - return nil +func (g *gauge) Update(mValue any) error { + value, ok := mValue.(float64) + if !ok { + return ErrInvalidValueType + } + g.value = value + return nil } func (g *gauge) SetValue(v any) error { diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 5718b26..5f82b17 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -8,7 +8,7 @@ type Metric interface { Value() any Name() string Type() string - Update(mName string, mValue any) error + Update(mValue any) error } const ( diff --git a/pkg/runtime-stats/runtime-stats.go b/pkg/runtime-stats/runtime-stats.go index d8b0629..5c024d7 100644 --- a/pkg/runtime-stats/runtime-stats.go +++ b/pkg/runtime-stats/runtime-stats.go @@ -16,131 +16,136 @@ var MemRuntimeStats []MemRuntimeStat = []MemRuntimeStat{ { Name: "Alloc", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.Alloc }, + Get: func(m *runtime.MemStats) any { return float64(m.Alloc) }, }, { Name: "BuckHashSys", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.BuckHashSys }, + Get: func(m *runtime.MemStats) any { return float64(m.BuckHashSys) }, }, { Name: "Frees", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.Frees }, + Get: func(m *runtime.MemStats) any { return float64(m.Frees) }, }, { Name: "GCCPUFraction", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.GCCPUFraction }, + Get: func(m *runtime.MemStats) any { return float64(m.GCCPUFraction) }, + }, + { + Name: "GCSys", + Type: mtr.GaugeType, + Get: func(m *runtime.MemStats) any { return float64(m.GCSys) }, }, { Name: "HeapAlloc", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.HeapAlloc }, + Get: func(m *runtime.MemStats) any { return float64(m.HeapAlloc) }, }, { Name: "HeapIdle", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.HeapIdle }, + Get: func(m *runtime.MemStats) any { return float64(m.HeapIdle) }, }, { Name: "HeapInuse", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.HeapInuse }, + Get: func(m *runtime.MemStats) any { return float64(m.HeapInuse) }, }, { Name: "HeapObjects", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.HeapObjects }, + Get: func(m *runtime.MemStats) any { return float64(m.HeapObjects) }, }, { Name: "HeapReleased", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.HeapReleased }, + Get: func(m *runtime.MemStats) any { return float64(m.HeapReleased) }, }, { Name: "HeapSys", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.HeapSys }, + Get: func(m *runtime.MemStats) any { return float64(m.HeapSys) }, }, { Name: "LastGC", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.LastGC }, + Get: func(m *runtime.MemStats) any { return float64(m.LastGC) }, }, { Name: "Lookups", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.Lookups }, + Get: func(m *runtime.MemStats) any { return float64(m.Lookups) }, }, { Name: "MCacheInuse", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.MCacheInuse }, + Get: func(m *runtime.MemStats) any { return float64(m.MCacheInuse) }, }, { Name: "MCacheSys", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.MCacheSys }, + Get: func(m *runtime.MemStats) any { return float64(m.MCacheSys) }, }, { Name: "MSpanInuse", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.MSpanInuse }, + Get: func(m *runtime.MemStats) any { return float64(m.MSpanInuse) }, }, { Name: "MSpanSys", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.MSpanSys }, + Get: func(m *runtime.MemStats) any { return float64(m.MSpanSys) }, }, { Name: "Mallocs", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.Mallocs }, + Get: func(m *runtime.MemStats) any { return float64(m.Mallocs) }, }, { Name: "NextGC", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.NextGC }, + Get: func(m *runtime.MemStats) any { return float64(m.NextGC) }, }, { Name: "NumForcedGC", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.NumForcedGC }, + Get: func(m *runtime.MemStats) any { return float64(m.NumForcedGC) }, }, { Name: "NumGC", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.NumGC }, + Get: func(m *runtime.MemStats) any { return float64(m.NumGC) }, }, { Name: "OtherSys", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.OtherSys }, + Get: func(m *runtime.MemStats) any { return float64(m.OtherSys) }, }, { Name: "PauseTotalNs", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.PauseTotalNs }, + Get: func(m *runtime.MemStats) any { return float64(m.PauseTotalNs) }, }, { Name: "StackInuse", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.StackInuse }, + Get: func(m *runtime.MemStats) any { return float64(m.StackInuse) }, }, { Name: "StackSys", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.StackSys }, + Get: func(m *runtime.MemStats) any { return float64(m.StackSys) }, }, { Name: "Sys", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.Sys }, + Get: func(m *runtime.MemStats) any { return float64(m.Sys) }, }, { Name: "TotalAlloc", Type: mtr.GaugeType, - Get: func(m *runtime.MemStats) any { return m.TotalAlloc }, + Get: func(m *runtime.MemStats) any { return float64(m.TotalAlloc) }, }, } From f2698a4a649d648fdfb9674e3ddebf9cb03e03a3 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Tue, 8 Jul 2025 02:00:00 +0300 Subject: [PATCH 045/126] add tests --- internal/handlers/server/server.go | 14 +- internal/mem-storage/storage_test.go | 261 +++++++++++++++++++++++++++ internal/metrics/counter_test.go | 84 +++++++++ internal/metrics/gauge_test.go | 60 ++++++ 4 files changed, 412 insertions(+), 7 deletions(-) create mode 100644 internal/mem-storage/storage_test.go create mode 100644 internal/metrics/counter_test.go create mode 100644 internal/metrics/gauge_test.go diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index e1d1b45..23ebd05 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -14,6 +14,13 @@ import ( log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) +type Metrics struct { + ID string `json:"id"` + MType string `json:"type"` + Delta *int64 `json:"delta,omitempty"` + Value *float64 `json:"value,omitempty"` +} + type MetricTable struct { Name string Type string @@ -216,13 +223,6 @@ func UpdateMetric(storage ms.Collector) http.HandlerFunc { } } -type Metrics struct { - ID string `json:"id"` - MType string `json:"type"` - Delta *int64 `json:"delta,omitempty"` - Value *float64 `json:"value,omitempty"` -} - func FillMetricValueFromStorage(storage ms.Collector, metric *Metrics) bool { value, ok := storage.GetMetric(metric.MType, metric.ID) if !ok { diff --git a/internal/mem-storage/storage_test.go b/internal/mem-storage/storage_test.go new file mode 100644 index 0000000..ed435ed --- /dev/null +++ b/internal/mem-storage/storage_test.go @@ -0,0 +1,261 @@ +package memstorage + +import ( + "reflect" + "sync" + "testing" + + mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" +) + +func TestMemStorage_UpdateMetric(t *testing.T) { + type fields struct { + storage map[string]map[string]mtr.Metric + } + type args struct { + mType string + mName string + mValue any + } + tests := []struct { + name string + fields fields + args args + wantErr bool + wantResult any + }{ + { + name: "Add new gauge", + fields: fields{ + storage: make(map[string]map[string]mtr.Metric), + }, + args: args{ + mType: mtr.GaugeType, + mName: "g1", + mValue: 3.14, + }, + wantErr: false, + wantResult: 3.14, + }, + { + name: "Add new counter", + fields: fields{ + storage: make(map[string]map[string]mtr.Metric), + }, + args: args{ + mType: mtr.CounterType, + mName: "c1", + mValue: int64(10), + }, + wantErr: false, + wantResult: int64(10), + }, + { + name: "Invalid gauge value", + fields: fields{ + storage: make(map[string]map[string]mtr.Metric), + }, + args: args{ + mType: mtr.GaugeType, + mName: "g2", + mValue: "not a float", + }, + wantErr: true, + }, + { + name: "Update existing counter", + fields: fields{ + storage: map[string]map[string]mtr.Metric{ + mtr.CounterType: { + "c2": mtr.NewCounter("c2", 5), + }, + }, + }, + args: args{ + mType: mtr.CounterType, + mName: "c2", + mValue: int64(7), + }, + wantErr: false, + wantResult: int64(12), + }, + { + name: "Unknown metric type", + fields: fields{ + storage: make(map[string]map[string]mtr.Metric), + }, + args: args{ + mType: "unknown", + mName: "whatever", + mValue: 1, + }, + wantErr: true, + }, + } + for i := range tests { + tt := &tests[i] + t.Run(tt.name, func(t *testing.T) { + ms := &MemStorage{ + mutex: sync.RWMutex{}, + storage: tt.fields.storage, + } + if err := ms.UpdateMetric(tt.args.mType, tt.args.mName, tt.args.mValue); (err != nil) != tt.wantErr { + t.Errorf("MemStorage.UpdateMetric() error = %v, wantErr %v", err, tt.wantErr) + } + + if !tt.wantErr { + val, ok := ms.GetMetric(tt.args.mType, tt.args.mName) + if !ok { + t.Errorf("metric not found") + return + } + if val != tt.wantResult { + t.Errorf("got = %v, want = %v", val, tt.wantResult) + } + } + }) + } +} + +func TestMemStorage_GetMetric(t *testing.T) { + counter := mtr.NewCounter("requests", 42) + gauge := mtr.NewGauge("temperature", 36.6) + + tests := []struct { + name string + fields map[string]map[string]mtr.Metric + mType string + mName string + wantVal any + wantOk bool + }{ + { + name: "existing counter", + fields: map[string]map[string]mtr.Metric{ + mtr.CounterType: {"requests": counter}, + }, + mType: mtr.CounterType, + mName: "requests", + wantVal: int64(42), + wantOk: true, + }, + { + name: "existing gauge", + fields: map[string]map[string]mtr.Metric{ + mtr.GaugeType: {"temperature": gauge}, + }, + mType: mtr.GaugeType, + mName: "temperature", + wantVal: float64(36.6), + wantOk: true, + }, + { + name: "missing metric name", + fields: map[string]map[string]mtr.Metric{ + mtr.CounterType: {"requests": counter}, + }, + mType: mtr.CounterType, + mName: "nonexistent", + wantVal: nil, + wantOk: false, + }, + { + name: "missing metric type", + fields: map[string]map[string]mtr.Metric{}, + mType: "unknown_type", + mName: "anything", + wantVal: nil, + wantOk: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ms := &MemStorage{ + mutex: sync.RWMutex{}, + storage: tt.fields, + } + gotVal, gotOk := ms.GetMetric(tt.mType, tt.mName) + if gotVal != tt.wantVal { + t.Errorf("GetMetric() gotVal = %v, want %v", gotVal, tt.wantVal) + } + if gotOk != tt.wantOk { + t.Errorf("GetMetric() gotOk = %v, want %v", gotOk, tt.wantOk) + } + }) + } +} + +func TestMemStorage_GetAllMetrics(t *testing.T) { + counter := mtr.NewCounter("requests", 100) + gauge := mtr.NewGauge("temperature", 25.5) + type fields struct { + mutex sync.RWMutex + storage map[string]map[string]mtr.Metric + } + tests := []struct { + name string + fields fields + want map[string]map[string]mtr.Metric + }{ + { + name: "empty storage", + fields: fields{ + mutex: sync.RWMutex{}, + storage: map[string]map[string]mtr.Metric{}, + }, + want: map[string]map[string]mtr.Metric{}, + }, + { + name: "storage with one counter", + fields: fields{ + mutex: sync.RWMutex{}, + storage: map[string]map[string]mtr.Metric{ + mtr.CounterType: { + "requests": counter, + }, + }, + }, + want: map[string]map[string]mtr.Metric{ + mtr.CounterType: { + "requests": counter, + }, + }, + }, + { + name: "storage with gauge and counter", + fields: fields{ + mutex: sync.RWMutex{}, + storage: map[string]map[string]mtr.Metric{ + mtr.CounterType: { + "requests": counter, + }, + mtr.GaugeType: { + "temperature": gauge, + }, + }, + }, + want: map[string]map[string]mtr.Metric{ + mtr.CounterType: { + "requests": counter, + }, + mtr.GaugeType: { + "temperature": gauge, + }, + }, + }, + } + + for i := range tests { + tt := &tests[i] + t.Run(tt.name, func(t *testing.T) { + ms := &MemStorage{ + mutex: sync.RWMutex{}, + storage: tt.fields.storage, + } + if got := ms.GetAllMetrics(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("MemStorage.GetAllMetrics() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/metrics/counter_test.go b/internal/metrics/counter_test.go new file mode 100644 index 0000000..60536d0 --- /dev/null +++ b/internal/metrics/counter_test.go @@ -0,0 +1,84 @@ +package metrics + +import "testing" + +func TestCounterUpdate(t *testing.T) { + type fields struct { + name string + value int64 + } + type args struct { + mValue any + } + tests := []struct { + name string + fields fields + args args + wantErr bool + wantValue int64 + }{ + { + name: "valid increment", + fields: fields{ + name: "test_counter", + value: 10, + }, + args: args{ + mValue: int64(5), + }, + wantErr: false, + wantValue: 15, + }, + { + name: "zero increment", + fields: fields{ + name: "test_counter", + value: 100, + }, + args: args{ + mValue: int64(0), + }, + wantErr: false, + wantValue: 100, + }, + { + name: "negative increment", + fields: fields{ + name: "test_counter", + value: 20, + }, + args: args{ + mValue: int64(-5), + }, + wantErr: false, + wantValue: 15, + }, + { + name: "invalid type (string)", + fields: fields{ + name: "test_counter", + value: 42, + }, + args: args{ + mValue: "not an int", + }, + wantErr: true, + wantValue: 42, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &counter{ + name: tt.fields.name, + value: tt.fields.value, + } + err := c.Update(tt.args.mValue) + if (err != nil) != tt.wantErr { + t.Errorf("Update() error = %v, wantErr = %v", err, tt.wantErr) + } + if c.value != tt.wantValue { + t.Errorf("Update() value = %v, wantValue = %v", c.value, tt.wantValue) + } + }) + } +} diff --git a/internal/metrics/gauge_test.go b/internal/metrics/gauge_test.go new file mode 100644 index 0000000..9b6dc19 --- /dev/null +++ b/internal/metrics/gauge_test.go @@ -0,0 +1,60 @@ +package metrics + +import "testing" + +func TestGaugeUpdate(t *testing.T) { + type fields struct { + name string + value float64 + } + type args struct { + mValue any + } + tests := []struct { + name string + fields fields + args args + want float64 + wantErr bool + }{ + { + name: "valid update", + fields: fields{ + name: "TestGauge", + value: 10.0, + }, + args: args{ + mValue: 42.5, + }, + want: 42.5, + wantErr: false, + }, + { + name: "invalid type", + fields: fields{ + name: "TestGaugeInvalid", + value: 3.14, + }, + args: args{ + mValue: "not a float", + }, + want: 3.14, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := &gauge{ + name: tt.fields.name, + value: tt.fields.value, + } + err := g.Update(tt.args.mValue) + if (err != nil) != tt.wantErr { + t.Errorf("gauge.Update() error = %v, wantErr %v", err, tt.wantErr) + } + if !tt.wantErr && g.value != tt.want { + t.Errorf("gauge.value = %v, want %v", g.value, tt.want) + } + }) + } +} From 439daa1ee2e051fee7b3c200bc50612c3e9caa15 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Wed, 9 Jul 2025 03:29:04 +0300 Subject: [PATCH 046/126] after rebase iter6, change --- internal/handlers/agent/agent.go | 13 ++- internal/mem-storage/storage.go | 4 - internal/mem-storage/storage_test.go | 149 +++++++++++++-------------- 3 files changed, 85 insertions(+), 81 deletions(-) diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index e7c5c26..3dcc72b 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -50,6 +50,11 @@ func sendAllMetrics(client *resty.Client, storage *ms.MemStorage) { mType := metric.Type() mName := metric.Name() + metricJSON := server.Metrics{ + ID: mName, + MType: mType, + } + switch mType { case mtr.GaugeType: val, ok := metric.Value().(float64) @@ -58,7 +63,9 @@ func sendAllMetrics(client *resty.Client, storage *ms.MemStorage) { Msg("Invalid metric value type") continue } - sendMetric(client, mtr.GaugeType, mName, val) + + metricJSON.Value = &val + sendMetric(client, &metricJSON) case mtr.CounterType: val, ok := metric.Value().(int64) @@ -67,7 +74,9 @@ func sendAllMetrics(client *resty.Client, storage *ms.MemStorage) { Msg("Invalid metric value type") continue } - sendMetric(client, mtr.CounterType, mName, val) + + metricJSON.Delta = &val + sendMetric(client, &metricJSON) } } } diff --git a/internal/mem-storage/storage.go b/internal/mem-storage/storage.go index dee0d84..de0aa74 100644 --- a/internal/mem-storage/storage.go +++ b/internal/mem-storage/storage.go @@ -32,11 +32,7 @@ func (ms *MemStorage) UpdateMetric(mType, mName string, mValue any) error { } if oldMetric, ok := ms.storage[mType][mName]; ok { -<<<<<<< HEAD - return oldMetric.Update(mName, mValue) -======= return oldMetric.Update(mValue) ->>>>>>> c8d8784 (first easyjson-JSON version) } var newMetric mtr.Metric diff --git a/internal/mem-storage/storage_test.go b/internal/mem-storage/storage_test.go index ed435ed..db651bf 100644 --- a/internal/mem-storage/storage_test.go +++ b/internal/mem-storage/storage_test.go @@ -1,7 +1,6 @@ package memstorage import ( - "reflect" "sync" "testing" @@ -93,7 +92,7 @@ func TestMemStorage_UpdateMetric(t *testing.T) { }, } for i := range tests { - tt := &tests[i] + tt := &tests[i] t.Run(tt.name, func(t *testing.T) { ms := &MemStorage{ mutex: sync.RWMutex{}, @@ -186,76 +185,76 @@ func TestMemStorage_GetMetric(t *testing.T) { } } -func TestMemStorage_GetAllMetrics(t *testing.T) { - counter := mtr.NewCounter("requests", 100) - gauge := mtr.NewGauge("temperature", 25.5) - type fields struct { - mutex sync.RWMutex - storage map[string]map[string]mtr.Metric - } - tests := []struct { - name string - fields fields - want map[string]map[string]mtr.Metric - }{ - { - name: "empty storage", - fields: fields{ - mutex: sync.RWMutex{}, - storage: map[string]map[string]mtr.Metric{}, - }, - want: map[string]map[string]mtr.Metric{}, - }, - { - name: "storage with one counter", - fields: fields{ - mutex: sync.RWMutex{}, - storage: map[string]map[string]mtr.Metric{ - mtr.CounterType: { - "requests": counter, - }, - }, - }, - want: map[string]map[string]mtr.Metric{ - mtr.CounterType: { - "requests": counter, - }, - }, - }, - { - name: "storage with gauge and counter", - fields: fields{ - mutex: sync.RWMutex{}, - storage: map[string]map[string]mtr.Metric{ - mtr.CounterType: { - "requests": counter, - }, - mtr.GaugeType: { - "temperature": gauge, - }, - }, - }, - want: map[string]map[string]mtr.Metric{ - mtr.CounterType: { - "requests": counter, - }, - mtr.GaugeType: { - "temperature": gauge, - }, - }, - }, - } - - for i := range tests { - tt := &tests[i] - t.Run(tt.name, func(t *testing.T) { - ms := &MemStorage{ - mutex: sync.RWMutex{}, - storage: tt.fields.storage, - } - if got := ms.GetAllMetrics(); !reflect.DeepEqual(got, tt.want) { - t.Errorf("MemStorage.GetAllMetrics() = %v, want %v", got, tt.want) - } - }) - } -} +// func TestMemStorage_GetAllMetrics(t *testing.T) { +// counter := mtr.NewCounter("requests", 100) +// gauge := mtr.NewGauge("temperature", 25.5) +// type fields struct { +// mutex sync.RWMutex +// storage map[string]map[string]mtr.Metric +// } +// tests := []struct { +// name string +// fields fields +// want map[string]map[string]mtr.Metric +// }{ +// { +// name: "empty storage", +// fields: fields{ +// mutex: sync.RWMutex{}, +// storage: map[string]map[string]mtr.Metric{}, +// }, +// want: map[string]map[string]mtr.Metric{}, +// }, +// { +// name: "storage with one counter", +// fields: fields{ +// mutex: sync.RWMutex{}, +// storage: map[string]map[string]mtr.Metric{ +// mtr.CounterType: { +// "requests": counter, +// }, +// }, +// }, +// want: map[string]map[string]mtr.Metric{ +// mtr.CounterType: { +// "requests": counter, +// }, +// }, +// }, +// { +// name: "storage with gauge and counter", +// fields: fields{ +// mutex: sync.RWMutex{}, +// storage: map[string]map[string]mtr.Metric{ +// mtr.CounterType: { +// "requests": counter, +// }, +// mtr.GaugeType: { +// "temperature": gauge, +// }, +// }, +// }, +// want: map[string]map[string]mtr.Metric{ +// mtr.CounterType: { +// "requests": counter, +// }, +// mtr.GaugeType: { +// "temperature": gauge, +// }, +// }, +// }, +// } +// +// for i := range tests { +// tt := &tests[i] +// t.Run(tt.name, func(t *testing.T) { +// ms := &MemStorage{ +// mutex: sync.RWMutex{}, +// storage: tt.fields.storage, +// } +// if got := ms.GetAllMetrics(); !reflect.DeepEqual(got, tt.want) { +// t.Errorf("MemStorage.GetAllMetrics() = %v, want %v", got, tt.want) +// } +// }) +// } +// } From ce1e133c19ae03df900b3de0be97f48587fc1ef4 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Mon, 7 Jul 2025 22:56:35 +0300 Subject: [PATCH 047/126] first easyjson-JSON version --- internal/handlers/agent/agent.go | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 3dcc72b..39ed1e6 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -46,6 +46,7 @@ func UpdateAllMetrics(storage *ms.MemStorage) { func sendAllMetrics(client *resty.Client, storage *ms.MemStorage) { allMetrics := storage.GetAllMetrics() +<<<<<<< HEAD for _, metric := range allMetrics { mType := metric.Type() mName := metric.Name() @@ -62,6 +63,38 @@ func sendAllMetrics(client *resty.Client, storage *ms.MemStorage) { log.Error().Str("metric_name", mName).Str("metric_type", mType). Msg("Invalid metric value type") continue +======= + for mType, innerMap := range allMetrics { + for mName, metric := range innerMap { + + metricJSON := server.Metrics{ + ID: metric.Name(), + MType: metric.Type(), + } + + switch mType { + case mtr.GaugeType: + val, ok := metric.Value().(float64) + if !ok { + log.Error().Str("metric_name", mName).Str("metric_type", mType). + Msg("Invalid metric value type") + continue + } + + metricJSON.Value = &val + sendMetric(client, &metricJSON) + + case mtr.CounterType: + val, ok := metric.Value().(int64) + if !ok { + log.Error().Str("metric_name", mName).Str("metric_type", mType). + Msg("Invalid metric value type") + continue + } + + metricJSON.Delta = &val + sendMetric(client, &metricJSON) +>>>>>>> c8d8784 (first easyjson-JSON version) } metricJSON.Value = &val From 68039a5fc81ed44ec12cc04ae034e64f74f6daca Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Tue, 8 Jul 2025 02:00:00 +0300 Subject: [PATCH 048/126] add tests --- internal/mem-storage/storage_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/mem-storage/storage_test.go b/internal/mem-storage/storage_test.go index db651bf..b34ad01 100644 --- a/internal/mem-storage/storage_test.go +++ b/internal/mem-storage/storage_test.go @@ -258,3 +258,4 @@ func TestMemStorage_GetMetric(t *testing.T) { // }) // } // } + From 2a39f46338ee371a9e3bf1e70b6f7c2a762d1449 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Tue, 8 Jul 2025 21:30:37 +0300 Subject: [PATCH 049/126] first version compress file gzip --- internal/handlers/agent/agent.go | 35 +++++++++++++++++ internal/handlers/server/gzip.go | 61 +++++++++++++++++++++++++++++ internal/handlers/server/logging.go | 5 --- internal/handlers/server/server.go | 39 ++++++------------ 4 files changed, 109 insertions(+), 31 deletions(-) create mode 100644 internal/handlers/server/gzip.go diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 39ed1e6..3afeabc 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -1,6 +1,8 @@ package agent import ( + "bytes" + "compress/gzip" "fmt" "math/rand" "net/http" @@ -8,6 +10,7 @@ import ( "time" "github.com/go-resty/resty/v2" + "github.com/mailru/easyjson" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" @@ -121,9 +124,16 @@ func sendMetric(client *resty.Client, metricJSON *server.Metrics) { 1 * time.Second, } + // buf, err := ConvertToGzipData(metricJSON) + // if err != nil { + // log.Error().Err(err).Msg("Failed to convert metric to gzip") + // return + // } + for _, backoff := range backoffSchedule { res, err := client.R(). SetHeader("Content-Type", "application/json"). + SetHeader("Content-Encoding", "gzip"). SetBody(metricJSON). Post("update/") @@ -136,6 +146,31 @@ func sendMetric(client *resty.Client, metricJSON *server.Metrics) { } } +func ConvertToGzipData(metricJSON *server.Metrics) (*bytes.Buffer, error) { + jsonData, err := easyjson.Marshal(*metricJSON) + if err != nil { + log.Error().Err(err).Msg("Failed to marshal metricJSON") + return nil, err + } + + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + defer func() { + err := gz.Close() + if err != nil { + log.Error().Err(err).Msg("Failed to close gzip writer") + } + }() + + _, err = gz.Write(jsonData) + if err != nil { + log.Error().Err(err).Msg("Failed to write gzip data") + return nil, err + } + + return &buf, nil +} + func CollectionLoop(storage *ms.MemStorage, interval time.Duration) { log.Debug().Msg("collectionLoop ...") for { diff --git a/internal/handlers/server/gzip.go b/internal/handlers/server/gzip.go new file mode 100644 index 0000000..061c536 --- /dev/null +++ b/internal/handlers/server/gzip.go @@ -0,0 +1,61 @@ +package server + +import ( + "compress/gzip" + "io" + "net/http" + "slices" + + log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" +) + +var supportedContentType = map[string]bool{ + "application/json": true, + "text/html; charset=utf-8": true, +} + +type gzipWriter struct { + http.ResponseWriter + Writer io.Writer +} + +func (w *gzipWriter) Write(b []byte) (int, error) { + contentType := w.Header().Get("Content-Type") + + if _, ok := supportedContentType[contentType]; !ok { + w.Header().Del("Content-Encoding") + return w.ResponseWriter.Write(b) + } + + w.Header().Set("Content-Encoding", "gzip") + return w.Writer.Write(b) +} + +func WithGzipCompress(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + encodingValues := req.Header.Values("Accept-Encoding") + + if !slices.Contains(encodingValues, "gzip") { + next.ServeHTTP(w, req) + return + } + + gz, err := gzip.NewWriterLevel(w, gzip.BestSpeed) + if err != nil { + http.Error(w, "Failed to create gzip writer", http.StatusInternalServerError) + return + } + defer func() { + if err := gz.Close(); err != nil { + log.Error().Err(err).Msg("Failed to close gzip writer") + } + }() + + gw := &gzipWriter{ + ResponseWriter: w, + Writer: gz, + } + + next.ServeHTTP(gw, req) + }) +} diff --git a/internal/handlers/server/logging.go b/internal/handlers/server/logging.go index 2169501..92930d9 100644 --- a/internal/handlers/server/logging.go +++ b/internal/handlers/server/logging.go @@ -26,11 +26,6 @@ func (res *loggingResponseWriter) Write(body []byte) (int, error) { return size, err } -func (res *loggingResponseWriter) WriteHeader(statusCode int) { - res.ResponseWriter.WriteHeader(statusCode) - res.responseData.status = statusCode -} - func WithLogging(h http.Handler) http.Handler { logfn := func(w http.ResponseWriter, r *http.Request) { start := time.Now() diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index 23ebd05..6f4ffe9 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -15,10 +15,10 @@ import ( ) type Metrics struct { - ID string `json:"id"` - MType string `json:"type"` - Delta *int64 `json:"delta,omitempty"` - Value *float64 `json:"value,omitempty"` + ID string `json:"id"` + MType string `json:"type"` + Delta *int64 `json:"delta,omitempty"` + Value *float64 `json:"value,omitempty"` } type MetricTable struct { @@ -31,6 +31,7 @@ func NewRouter(storage ms.Collector) http.Handler { r := chi.NewRouter() r.Use(WithLogging) + r.Use(WithGzipCompress) r.Route("/", func(r chi.Router) { r.Get("/", GetAllMetrics(storage)) @@ -176,8 +177,6 @@ func GetAllMetrics(storage ms.Collector) http.HandlerFunc { if err := template.Execute(res, metricsToTable); err != nil { log.Error().Msgf("failed complete template: %v", err) } - - res.WriteHeader(http.StatusOK) } } @@ -218,8 +217,6 @@ func UpdateMetric(storage ms.Collector) http.HandlerFunc { Str("metric_name", mName). Interface("value", val). Msg("Metric updated successfully") - - res.WriteHeader(http.StatusOK) } } @@ -265,15 +262,10 @@ func GetMetricsHandlerJSON(storage ms.Collector) http.HandlerFunc { return } - started, n, err := easyjson.MarshalToHTTPResponseWriter(&metric, resp) - if err != nil { - if !started { - http.Error(resp, "failed to encode json", http.StatusInternalServerError) - } else { - log.Error().Err(err). - Int("written_bytes", n). - Msg("error while writing json response") - } + resp.Header().Set("Content-Type", "application/json") + if _, err := easyjson.MarshalToWriter(&metric, resp); err != nil { + http.Error(resp, fmt.Sprintf("failed to encode json: %v", err), http.StatusInternalServerError) + return } } } @@ -308,15 +300,10 @@ func UpdateMetricsHandlerJSON(storage ms.Collector) http.HandlerFunc { return } - started, n, err := easyjson.MarshalToHTTPResponseWriter(&metric, resp) - if err != nil { - if !started { - http.Error(resp, "failed to encode json", http.StatusInternalServerError) - } else { - log.Error().Err(err). - Int("written_bytes", n). - Msg("error while writing json response") - } + resp.Header().Set("Content-Type", "application/json") + if _, err := easyjson.MarshalToWriter(&metric, resp); err != nil { + http.Error(resp, fmt.Sprintf("failed to encode json: %v", err), http.StatusInternalServerError) + return } } } From f41b3613f62cd387ead371ddb582770c650c21b8 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Wed, 9 Jul 2025 03:40:36 +0300 Subject: [PATCH 050/126] after rebase iter7 (from change in iter5 to merge) --- internal/handlers/agent/agent.go | 33 -------------------------------- 1 file changed, 33 deletions(-) diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 3afeabc..bc78406 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -49,7 +49,6 @@ func UpdateAllMetrics(storage *ms.MemStorage) { func sendAllMetrics(client *resty.Client, storage *ms.MemStorage) { allMetrics := storage.GetAllMetrics() -<<<<<<< HEAD for _, metric := range allMetrics { mType := metric.Type() mName := metric.Name() @@ -66,38 +65,6 @@ func sendAllMetrics(client *resty.Client, storage *ms.MemStorage) { log.Error().Str("metric_name", mName).Str("metric_type", mType). Msg("Invalid metric value type") continue -======= - for mType, innerMap := range allMetrics { - for mName, metric := range innerMap { - - metricJSON := server.Metrics{ - ID: metric.Name(), - MType: metric.Type(), - } - - switch mType { - case mtr.GaugeType: - val, ok := metric.Value().(float64) - if !ok { - log.Error().Str("metric_name", mName).Str("metric_type", mType). - Msg("Invalid metric value type") - continue - } - - metricJSON.Value = &val - sendMetric(client, &metricJSON) - - case mtr.CounterType: - val, ok := metric.Value().(int64) - if !ok { - log.Error().Str("metric_name", mName).Str("metric_type", mType). - Msg("Invalid metric value type") - continue - } - - metricJSON.Delta = &val - sendMetric(client, &metricJSON) ->>>>>>> c8d8784 (first easyjson-JSON version) } metricJSON.Value = &val From 9dfa52e0dfb7b402717d47915d84d7c6d5611134 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Wed, 9 Jul 2025 23:05:53 +0300 Subject: [PATCH 051/126] empty commit --- cmd/agent/main.go | 2 -- cmd/server/main.go | 2 -- 2 files changed, 4 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 839d48d..068527e 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -5,9 +5,7 @@ import ( ) func main() { - if err := rootCmd.Execute(); err != nil { os.Exit(1) } - } diff --git a/cmd/server/main.go b/cmd/server/main.go index 839d48d..068527e 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -5,9 +5,7 @@ import ( ) func main() { - if err := rootCmd.Execute(); err != nil { os.Exit(1) } - } From 33cf189cda11c345c874f777b36e2f455a1436d1 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Thu, 10 Jul 2025 21:45:44 +0300 Subject: [PATCH 052/126] add os data-base, need fix test (imports cycle) --- cmd/agent/cfg_agent.go | 24 +- cmd/server/cfg_server.go | 138 ++++++++-- internal/handlers/agent/agent.go | 32 +-- internal/handlers/server/server.go | 45 ++-- internal/handlers/server/server_test.go | 322 +++++++++++++----------- pkg/data-base/DBfunctions.go | 95 +++++++ 6 files changed, 449 insertions(+), 207 deletions(-) create mode 100644 pkg/data-base/DBfunctions.go diff --git a/cmd/agent/cfg_agent.go b/cmd/agent/cfg_agent.go index fb2e346..e78e028 100644 --- a/cmd/agent/cfg_agent.go +++ b/cmd/agent/cfg_agent.go @@ -4,6 +4,8 @@ import ( "fmt" "net" "os" + "os/signal" + "syscall" "time" "github.com/caarlos0/env/v6" @@ -102,8 +104,24 @@ func startAgent() { log.Info().Msg("Starting collection and reporting loops") - go agent.CollectionLoop(storage, time.Duration(opts.pollInterval)*time.Second) - go agent.ReportLoop(client, storage, time.Duration(opts.reportInterval)*time.Second) + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) - select {} + pollTimer := time.NewTicker(time.Duration(opts.pollInterval) * time.Second) + reportTimer := time.NewTicker(time.Duration(opts.reportInterval) * time.Second) + + defer pollTimer.Stop() + defer reportTimer.Stop() + + for { + select { + case <-stop: + return + + case <-pollTimer.C: + agent.UpdateAllMetrics(storage) + case <-reportTimer.C: + agent.SendAllMetrics(client, storage) + } + } } diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 2c7f731..7f3a309 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -1,53 +1,62 @@ package main import ( + "errors" "fmt" "net" "net/http" "os" + "os/signal" + "syscall" + "time" "github.com/caarlos0/env/v6" + "github.com/go-chi/chi/v5" "github.com/spf13/cobra" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" + db "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/data-base" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) const ( - defaultEndpoint = "localhost:8080" + defaultEndpoint = "localhost:8080" + defaultStoreInterval = 300 + defaultFileStoragePath = "/temp/metrics-db.json" + defaultRestoreOnStart = true ) type options struct { - endPointAddr string + endPointAddr string + storeInterval int + fileStoragePath string + restoreOnStart bool } type envConfig struct { - EndPointAddr string `env:"ADDRESS"` + EndPointAddr string `env:"ADDRESS"` + StoreInterval int `env:"STORE_INTERVAL"` + FileStoragePath string `env:"FILE_STORAGE_PATH"` + RestoreOnStart bool `env:"RESTORE"` } -var opts = &options{} +var opts = &options{ + endPointAddr: defaultEndpoint, + storeInterval: defaultStoreInterval, + fileStoragePath: defaultFileStoragePath, + restoreOnStart: defaultRestoreOnStart, +} var rootCmd = &cobra.Command{ Use: "server", Short: "MetricService", Long: "MetricService", Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - logFile, err := log.InitLogger("logFileServer.log") - if err != nil { - fmt.Fprintf(os.Stderr, "Logger init error: %v\n", err) - os.Exit(1) - } - - defer func() { - if err := logFile.Close(); err != nil { - log.Error().Err(err).Msg("Failed to close log file") - } - }() + PreRunE: func(cmd *cobra.Command, args []string) error { var cfg envConfig - err = env.Parse(&cfg) + err := env.Parse(&cfg) if err != nil { return fmt.Errorf("poll and report intervals must be > 0") } @@ -60,29 +69,114 @@ var rootCmd = &cobra.Command{ return fmt.Errorf("invalid address %s: %w", opts.endPointAddr, err) } - startServer() + if cfg.FileStoragePath != "" { + opts.fileStoragePath = cfg.FileStoragePath + } + + if cfg.StoreInterval != 0 { + opts.storeInterval = cfg.StoreInterval + } + + opts.restoreOnStart = cfg.RestoreOnStart + + return nil + }, + + RunE: func(cmd *cobra.Command, args []string) error { + logFile, err := log.InitLogger("logFileServer.log") + if err != nil { + return fmt.Errorf("logger init error: %w", err) + } + + defer func() { + if err := logFile.Close(); err != nil { + log.Error().Err(err).Msg("Failed to close log file") + } + }() + + if err := startServer(opts); err != nil { + return err + } return nil }, } func init() { - opts.endPointAddr = defaultEndpoint + rootCmd.Flags().StringVarP(&opts.endPointAddr, "a", "a", defaultEndpoint, "endpoint HTTP-server addr") + rootCmd.Flags().IntVarP(&opts.storeInterval, "i", "i", defaultStoreInterval, "store interval in seconds (0 = sync)") + rootCmd.Flags().StringVarP(&opts.fileStoragePath, "f", "f", defaultFileStoragePath, "file to store metrics") + rootCmd.Flags().BoolVarP(&opts.restoreOnStart, "r", "r", defaultRestoreOnStart, "restore metrics from file on start") +} + +func NewRouter(storage ms.Collector, opts *options) http.Handler { + r := chi.NewRouter() + + r.Use(server.WithLogging) + r.Use(server.WithGzipCompress) - rootCmd.Flags().StringVarP(&opts.endPointAddr, "a", "a", opts.endPointAddr, "endpoint HTTP-server addr") + r.Route("/", func(r chi.Router) { + r.Get("/", server.GetAllMetrics(storage)) + r.Route("/update", func(r chi.Router) { + if opts.storeInterval == 0 { + r.Use(db.WithSaveToDB(storage, opts.fileStoragePath)) + } + + r.Post("/", server.UpdateMetricsHandlerJSON(storage)) + r.Post("/{mType}/{mName}/{mValue}", server.UpdateMetric(storage)) + }) + + r.Route("/value", func(r chi.Router) { + r.Post("/", server.GetMetricsHandlerJSON(storage)) + r.Get("/{mType}/{mName}", server.GetMetric(storage)) + }) + }) + + return r } -func startServer() { + +func startServer(opts *options) error { log.Info(). Str("address", opts.endPointAddr). Msg("Server configuration") storage := ms.NewMemStorage() - r := server.NewRouter(storage) + r := NewRouter(storage, opts) + + if opts.restoreOnStart { + if err := db.LoadFromDB(storage, opts.fileStoragePath); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("LoadFromDB error %w", err) + } + } + + go func() { + ticker := time.NewTicker(time.Duration(opts.storeInterval) * time.Second) + defer ticker.Stop() + + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) + + for { + select { + case <-ticker.C: + if err := db.SaveToDB(storage, opts.fileStoragePath); err != nil { + log.Error().Err(err).Msg("failed to save DB") + } + case <-stop: + log.Info().Msg("Shutting down server, saving metrics") + if err := db.SaveToDB(storage, opts.fileStoragePath); err != nil { + log.Error().Err(err).Msg("Failed to save metrics during shutdown") + } + os.Exit(0) + } + } + }() if err := http.ListenAndServe(opts.endPointAddr, r); err != nil { fmt.Fprintf(os.Stderr, "HTTP-server didn't start: %v", err) panic(err) } + return nil } diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index bc78406..ecbdde6 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -46,7 +46,7 @@ func UpdateAllMetrics(storage *ms.MemStorage) { } } -func sendAllMetrics(client *resty.Client, storage *ms.MemStorage) { +func SendAllMetrics(client *resty.Client, storage *ms.MemStorage) { allMetrics := storage.GetAllMetrics() for _, metric := range allMetrics { @@ -138,18 +138,18 @@ func ConvertToGzipData(metricJSON *server.Metrics) (*bytes.Buffer, error) { return &buf, nil } -func CollectionLoop(storage *ms.MemStorage, interval time.Duration) { - log.Debug().Msg("collectionLoop ...") - for { - UpdateAllMetrics(storage) - time.Sleep(interval) - } -} - -func ReportLoop(client *resty.Client, storage *ms.MemStorage, interval time.Duration) { - log.Debug().Msg("reportLoop ...") - for { - sendAllMetrics(client, storage) - time.Sleep(interval) - } -} +// func CollectionLoop(storage *ms.MemStorage, interval time.Duration) { +// log.Debug().Msg("collectionLoop ...") +// for { +// UpdateAllMetrics(storage) +// time.Sleep(interval) +// } +// } +// +// func ReportLoop(client *resty.Client, storage *ms.MemStorage, interval time.Duration) { +// log.Debug().Msg("reportLoop ...") +// for { +// sendAllMetrics(client, storage) +// time.Sleep(interval) +// } +// } diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index 6f4ffe9..c317ba5 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -27,24 +27,29 @@ type MetricTable struct { Value string } -func NewRouter(storage ms.Collector) http.Handler { - r := chi.NewRouter() - - r.Use(WithLogging) - r.Use(WithGzipCompress) - - r.Route("/", func(r chi.Router) { - r.Get("/", GetAllMetrics(storage)) - r.Route("/", func(r chi.Router) { - r.Post("/update/", UpdateMetricsHandlerJSON(storage)) - r.Post("/value/", GetMetricsHandlerJSON(storage)) - r.Get("/value/{mType}/{mName}", GetMetric(storage)) - r.Post("/update/{mType}/{mName}/{mValue}", UpdateMetric(storage)) - }) - }) - - return r -} +// func NewRouter(storage ms.Collector) http.Handler { +// r := chi.NewRouter() +// +// r.Use(WithLogging) +// r.Use(WithGzipCompress) +// +// r.Route("/", func(r chi.Router) { +// r.Get("/", GetAllMetrics(storage)) +// r.Route("/update", func(r chi.Router) { +// r.Post("/", UpdateMetricsHandlerJSON(storage)) +// r.Post("/{mType}/{mName}/{mValue}", UpdateMetric(storage)) +// +// if +// }) +// +// r.Route("/value", func(r chi.Router) { +// r.Post("/", GetMetricsHandlerJSON(storage)) +// r.Get("/{mType}/{mName}", GetMetric(storage)) +// }) +// }) +// +// return r +// } func ConvertByType(mType, mValue string) (any, error) { switch mType { @@ -247,6 +252,7 @@ func GetMetricsHandlerJSON(storage ms.Collector) http.HandlerFunc { return func(resp http.ResponseWriter, req *http.Request) { var metric Metrics + log.Info().Msg("GetMetricsHandlerJSON called\n\n") if req.Header.Get("Content-Type") != "application/json" { http.Error(resp, "Content-Type must be application/json", http.StatusUnsupportedMediaType) return @@ -256,6 +262,7 @@ func GetMetricsHandlerJSON(storage ms.Collector) http.HandlerFunc { http.Error(resp, fmt.Sprintf("invalid json body: %v", err), http.StatusBadRequest) return } + log.Info().Msgf("Received metric request: %+v", metric) if !FillMetricValueFromStorage(storage, &metric) { http.Error(resp, fmt.Sprintf("metric %s not found", metric.ID), http.StatusNotFound) @@ -267,6 +274,7 @@ func GetMetricsHandlerJSON(storage ms.Collector) http.HandlerFunc { http.Error(resp, fmt.Sprintf("failed to encode json: %v", err), http.StatusInternalServerError) return } + log.Info().Msg("Metric successfully returned\n\n") } } @@ -307,3 +315,4 @@ func UpdateMetricsHandlerJSON(storage ms.Collector) http.HandlerFunc { } } } + diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index 556a30c..3bb5384 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -1,149 +1,175 @@ -package server +package server_test +// +// import ( +// "io" +// "net/http" +// "net/http/httptest" +// "testing" +// +// ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" +// mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" +// log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" +// ) +// +// const ( +// defaultEndpoint = "localhost:8080" +// defaultStoreInterval = 300 +// defaultFileStoragePath = "/temp/metrics-db.json" +// defaultRestoreOnStart = true +// ) +// +// type options struct { +// endPointAddr string +// storeInterval int +// fileStoragePath string +// restoreOnStart bool +// } +// +// var DefaultOpts = &options{ +// endPointAddr: defaultEndpoint, +// storeInterval: defaultStoreInterval, +// fileStoragePath: defaultFileStoragePath, +// restoreOnStart: defaultRestoreOnStart, +// } +// +// func TestUpdateMetric(t *testing.T) { +// opts := *DefaultOpts +// +// storage := ms.NewMemStorage() +// router := server.NewRouter(storage, opts) +// +// tests := []struct { +// name string +// method string +// url string +// wantStatus int +// }{ +// { +// name: "Valid Gauge Update", +// method: http.MethodPost, +// url: "/update/gauge/testGauge/123.45", +// wantStatus: http.StatusOK, +// }, +// { +// name: "Valid Counter Update", +// method: http.MethodPost, +// url: "/update/counter/testCounter/100", +// wantStatus: http.StatusOK, +// }, +// { +// name: "Update Counter with another value", +// method: http.MethodPost, +// url: "/update/counter/testCounter/50", +// wantStatus: http.StatusOK, +// }, +// { +// name: "Invalid Gauge Value", +// method: http.MethodPost, +// url: "/update/gauge/invalidGauge/abc", +// wantStatus: http.StatusBadRequest, +// }, +// { +// name: "Invalid Counter Value", +// method: http.MethodPost, +// url: "/update/counter/invalidCounter/xyz", +// wantStatus: http.StatusBadRequest, +// }, +// { +// name: "Unknown Metric Type", +// method: http.MethodPost, +// url: "/update/unknown/testMetric/123", +// wantStatus: http.StatusBadRequest, +// }, +// { +// name: "Missing Metric Name", +// method: http.MethodPost, +// url: "/update/gauge//123.45", +// wantStatus: http.StatusBadRequest, +// }, +// } +// +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// req := httptest.NewRequest(tt.method, tt.url, nil) +// rr := httptest.NewRecorder() +// +// router.ServeHTTP(rr, req) +// +// if rr.Code != tt.wantStatus { +// t.Errorf("Test %s: expected status %d, got %d", tt.name, tt.wantStatus, rr.Code) +// } +// }) +// } +// } +// +// func TestGetMetric(t *testing.T) { +// opts := *DefaultOpts +// +// storage := ms.NewMemStorage() +// +// if err := storage.UpdateMetric(mtr.GaugeType, "cpu_usage", 75.5); err != nil { +// log.Error().Msgf("Failed to update metric cpu_usage: %v", err) +// } +// +// if err := storage.UpdateMetric(mtr.CounterType, "requests_total", int64(100)); err != nil { +// log.Error().Msgf("Failed to update metric requests_total: %v", err) +// } +// +// router := server.NewRouter(storage, opts) +// +// tests := []struct { +// name string +// url string +// wantStatus int +// wantBody string +// }{ +// { +// name: "Get Existing Gauge", +// url: "/value/gauge/cpu_usage", +// wantStatus: http.StatusOK, +// wantBody: "75.5", +// }, +// { +// name: "Get Existing Counter", +// url: "/value/counter/requests_total", +// wantStatus: http.StatusOK, +// wantBody: "100", +// }, +// { +// name: "Get Non-Existing Metric", +// url: "/value/gauge/non_existent", +// wantStatus: http.StatusNotFound, +// wantBody: "Metric non_existent was not found\n", +// }, +// { +// name: "Get Unknown Metric Type", +// url: "/value/invalid_type/some_metric", +// wantStatus: http.StatusNotFound, +// wantBody: "Metric some_metric was not found\n", +// }, +// { +// name: "GET with missing name", +// url: "/value/gauge/", +// wantStatus: http.StatusNotFound, +// wantBody: "404 page not found\n", +// }, +// } +// +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// req := httptest.NewRequest(http.MethodGet, tt.url, nil) +// rr := httptest.NewRecorder() +// +// router.ServeHTTP(rr, req) +// +// if rr.Code != tt.wantStatus { +// t.Errorf("Test %s: expected status %d, got %d", tt.name, tt.wantStatus, rr.Code) +// } +// body, _ := io.ReadAll(rr.Body) +// if string(body) != tt.wantBody { +// t.Errorf("Test %s: expected body %q, got %q", tt.name, tt.wantBody, string(body)) +// } +// }) +// } +// } -import ( - "io" - "net/http" - "net/http/httptest" - "testing" - - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" - mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" - log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" -) - -func TestUpdateMetric(t *testing.T) { - storage := ms.NewMemStorage() - router := NewRouter(storage) - - tests := []struct { - name string - method string - url string - wantStatus int - }{ - { - name: "Valid Gauge Update", - method: http.MethodPost, - url: "/update/gauge/testGauge/123.45", - wantStatus: http.StatusOK, - }, - { - name: "Valid Counter Update", - method: http.MethodPost, - url: "/update/counter/testCounter/100", - wantStatus: http.StatusOK, - }, - { - name: "Update Counter with another value", - method: http.MethodPost, - url: "/update/counter/testCounter/50", - wantStatus: http.StatusOK, - }, - { - name: "Invalid Gauge Value", - method: http.MethodPost, - url: "/update/gauge/invalidGauge/abc", - wantStatus: http.StatusBadRequest, - }, - { - name: "Invalid Counter Value", - method: http.MethodPost, - url: "/update/counter/invalidCounter/xyz", - wantStatus: http.StatusBadRequest, - }, - { - name: "Unknown Metric Type", - method: http.MethodPost, - url: "/update/unknown/testMetric/123", - wantStatus: http.StatusBadRequest, - }, - { - name: "Missing Metric Name", - method: http.MethodPost, - url: "/update/gauge//123.45", - wantStatus: http.StatusBadRequest, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - req := httptest.NewRequest(tt.method, tt.url, nil) - rr := httptest.NewRecorder() - - router.ServeHTTP(rr, req) - - if rr.Code != tt.wantStatus { - t.Errorf("Test %s: expected status %d, got %d", tt.name, tt.wantStatus, rr.Code) - } - }) - } -} - -func TestGetMetric(t *testing.T) { - storage := ms.NewMemStorage() - - if err := storage.UpdateMetric(mtr.GaugeType, "cpu_usage", 75.5); err != nil { - log.Error().Msgf("Failed to update metric cpu_usage: %v", err) - } - - if err := storage.UpdateMetric(mtr.CounterType, "requests_total", int64(100)); err != nil { - log.Error().Msgf("Failed to update metric requests_total: %v", err) - } - - router := NewRouter(storage) - - tests := []struct { - name string - url string - wantStatus int - wantBody string - }{ - { - name: "Get Existing Gauge", - url: "/value/gauge/cpu_usage", - wantStatus: http.StatusOK, - wantBody: "75.5", - }, - { - name: "Get Existing Counter", - url: "/value/counter/requests_total", - wantStatus: http.StatusOK, - wantBody: "100", - }, - { - name: "Get Non-Existing Metric", - url: "/value/gauge/non_existent", - wantStatus: http.StatusNotFound, - wantBody: "Metric non_existent was not found\n", - }, - { - name: "Get Unknown Metric Type", - url: "/value/invalid_type/some_metric", - wantStatus: http.StatusNotFound, - wantBody: "Metric some_metric was not found\n", - }, - { - name: "GET with missing name", - url: "/value/gauge/", - wantStatus: http.StatusNotFound, - wantBody: "404 page not found\n", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - req := httptest.NewRequest(http.MethodGet, tt.url, nil) - rr := httptest.NewRecorder() - - router.ServeHTTP(rr, req) - - if rr.Code != tt.wantStatus { - t.Errorf("Test %s: expected status %d, got %d", tt.name, tt.wantStatus, rr.Code) - } - body, _ := io.ReadAll(rr.Body) - if string(body) != tt.wantBody { - t.Errorf("Test %s: expected body %q, got %q", tt.name, tt.wantBody, string(body)) - } - }) - } -} diff --git a/pkg/data-base/DBfunctions.go b/pkg/data-base/DBfunctions.go new file mode 100644 index 0000000..f7e6098 --- /dev/null +++ b/pkg/data-base/DBfunctions.go @@ -0,0 +1,95 @@ +package database + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" + + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" + mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" +) + +func SaveToDB(collector ms.Collector, path string) error { + allMetrics := collector.GetAllMetrics() + + data := make([]server.Metrics, 0, len(allMetrics)) + + for _, metric := range allMetrics { + var newMetric server.Metrics + + newMetric.MType = metric.Type() + newMetric.ID = metric.Name() + + switch metric.Type() { + case mtr.GaugeType: + val, ok := metric.Value().(float64) + if !ok { + return fmt.Errorf("invalid type metric") + } + newMetric.Value = &val + case mtr.CounterType: + val, ok := metric.Value().(int64) + if !ok { + return fmt.Errorf("invalid type metric") + } + newMetric.Delta = &val + default: + log.Error().Msg("unknown metric type") + } + + data = append(data, newMetric) + } + + bytes, err := json.MarshalIndent(data, "", " ") + if err != nil { + return fmt.Errorf("json Marshal Indent err: %w", err) + } + + return os.WriteFile(path, bytes, 0666) +} + +func LoadFromDB(collector ms.Collector, path string) error { + bytes, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("can't read file %s with DB %w", path, err) + } + + var data []server.Metrics + err = json.Unmarshal(bytes, &data) + if err != nil { + return fmt.Errorf("can't parse json format from DB %w", err) + } + + for _, metric := range data { + switch metric.MType { + case mtr.GaugeType: + if err := collector.UpdateMetric(metric.MType, metric.ID, *metric.Value); err != nil { + log.Error().Err(err).Msg("update metric error") + return fmt.Errorf("update metric error %w", err) + } + case mtr.CounterType: + if err := collector.UpdateMetric(metric.MType, metric.ID, *metric.Delta); err != nil { + log.Error().Err(err).Msg("update metric error") + return fmt.Errorf("update metric error %w", err) + } + } + } + + return nil +} + +func WithSaveToDB(collector ms.Collector, filePath string) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + next.ServeHTTP(w, req) + + if err := SaveToDB(collector, filePath); err != nil { + log.Error().Err(err).Msg("Failed to save metrics synchronously") + } + }) + } +} From 1ab2d841f9dacd6b2a62acaba6fe8947728b44b9 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Thu, 10 Jul 2025 22:54:40 +0300 Subject: [PATCH 053/126] Squash merge main into current branch --- .gitignore | 2 + cmd/server/main.go | 1 + go.mod | 1 + internal/memstorage/storage.go | 105 +++++++++++++++++++++++++++++++++ logger/logger.go | 97 ++++++++++++++++++++++++++++++ 5 files changed, 206 insertions(+) create mode 100644 internal/memstorage/storage.go create mode 100644 logger/logger.go diff --git a/.gitignore b/.gitignore index 245363d..48cc58f 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ go.work.sum .idea .vscode logFile.log + logFileServer.log logFileAgent.log cmd/server/server @@ -38,3 +39,4 @@ cmd/agent/agent log.txt + diff --git a/cmd/server/main.go b/cmd/server/main.go index 068527e..4467103 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -9,3 +9,4 @@ func main() { os.Exit(1) } } + diff --git a/go.mod b/go.mod index 6bd5e6a..02ac444 100644 --- a/go.mod +++ b/go.mod @@ -20,3 +20,4 @@ require ( golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.33.0 // indirect ) + diff --git a/internal/memstorage/storage.go b/internal/memstorage/storage.go new file mode 100644 index 0000000..de69ec8 --- /dev/null +++ b/internal/memstorage/storage.go @@ -0,0 +1,105 @@ +package memstorage + +import ( + "sync" + + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" +) + +type Collector interface { + GetMetric(mtr metrics.Metric) (interface{}, error) + UpdateMetric(mtr metrics.Metric) error +} + +type MemStorage struct { + mutex sync.RWMutex + gauges map[string]float64 + counters map[string]int64 +} + +func NewMemStorage() *MemStorage { + return &MemStorage{ + gauges: make(map[string]float64), + counters: make(map[string]int64), + } +} + +func (ms *MemStorage) UpdateMetric(metric metrics.Metric) error { + switch metric.Type() { + case metrics.CounterType: + { + val, ok := metric.Value().(int64) + if !ok { + return metrics.ErrInvalidValueType + } + + ms.UpdateCounter(metric, val) + return nil + } + case metrics.GaugeType: + { + val, ok := metric.Value().(float64) + if !ok { + return metrics.ErrInvalidValueType + } + + ms.UpdateGauge(metric, val) + return nil + } + default: + return metrics.ErrInvalidMetricsType + } +} + +func (ms *MemStorage) UpdateGauge(metric metrics.Metric, value float64) { + ms.mutex.Lock() + defer ms.mutex.Unlock() + ms.gauges[metric.Name()] = value +} + +func (ms *MemStorage) UpdateCounter(metric metrics.Metric, value int64) { + ms.mutex.Lock() + defer ms.mutex.Unlock() + ms.counters[metric.Name()] += value +} + +func (ms *MemStorage) GetMetric(metric metrics.Metric) (interface{}, error) { + switch metric.Type() { + case metrics.CounterType: + { + val, ok := ms.GetCounter(metric.Name()) + if !ok { + return nil, metrics.ErrMetricsNotFound + } + + return val, nil + } + case metrics.GaugeType: + { + val, ok := ms.GetGauges(metric.Name()) + if !ok { + return nil, metrics.ErrMetricsNotFound + } + + return val, nil + } + default: + return nil, metrics.ErrInvalidMetricsType + } +} + +func (ms *MemStorage) GetGauges(name string) (float64, bool) { + ms.mutex.RLock() + defer ms.mutex.RUnlock() + + val, ok := ms.gauges[name] + return val, ok +} + +func (ms *MemStorage) GetCounter(name string) (int64, bool) { + ms.mutex.RLock() + defer ms.mutex.RUnlock() + + val, ok := ms.counters[name] + return val, ok +} diff --git a/logger/logger.go b/logger/logger.go new file mode 100644 index 0000000..69ab32c --- /dev/null +++ b/logger/logger.go @@ -0,0 +1,97 @@ +package logger + +import ( + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "sync" + "time" +) + +type LogLevel int + +const ( + DebugLevel LogLevel = iota + InfoLevel + ErrorLevel +) + +var levelMap = map[LogLevel]string{ + DebugLevel: "[DEBUG]", + InfoLevel: "[INFO]", + ErrorLevel: "[ERROR]", +} + +type logger struct { + level LogLevel + out io.Writer +} + +var ( + instance *logger + once sync.Once +) + +func getLogger() *logger { + once.Do(func() { + instance = &logger{ + level: DebugLevel, + out: os.Stdout, + } + }) + return instance +} + +func Init(level LogLevel, logFileName string) error { + log := getLogger() + log.level = level + + if logFileName != "" { + file, err := os.Create(logFileName) + if err != nil { + return fmt.Errorf("failed to open log file %s: %w", logFileName, err) + } + log.out = file + } else { + log.out = os.Stdout + } + + return nil +} + +func Destroy() { + log := getLogger() + if file, ok := log.out.(*os.File); ok { + file.Close() + } +} + +func (log *logger) log(level LogLevel, format string, args ...interface{}) { + if log.level > level { + return + } + + _, file, line, ok := runtime.Caller(2) + if !ok { + file, line = "---", 0 + } + time := time.Now().Format("2006-01-02 15:04:05") + levelStr := levelMap[level] + msg := fmt.Sprintf(format, args...) + + fmt.Fprintf(log.out, "[%s]%s[%s:%d]: %s \n", time, levelStr, filepath.Base(file), line, msg) +} + +func Debug(format string, args ...interface{}) { + getLogger().log(DebugLevel, format, args...) +} + +func Info(format string, args ...interface{}) { + getLogger().log(InfoLevel, format, args...) +} + +func Error(format string, args ...interface{}) { + getLogger().log(ErrorLevel, format, args...) +} From dac7d685ed963dcedb4ec9aae630577ce9761ed2 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Thu, 10 Jul 2025 23:40:08 +0300 Subject: [PATCH 054/126] resolves conflicts --- logger/logger.go | 97 --------------------------- pkg/runtime-stats/runtime-stats.go | 104 ----------------------------- 2 files changed, 201 deletions(-) delete mode 100644 logger/logger.go diff --git a/logger/logger.go b/logger/logger.go deleted file mode 100644 index 69ab32c..0000000 --- a/logger/logger.go +++ /dev/null @@ -1,97 +0,0 @@ -package logger - -import ( - "fmt" - "io" - "os" - "path/filepath" - "runtime" - "sync" - "time" -) - -type LogLevel int - -const ( - DebugLevel LogLevel = iota - InfoLevel - ErrorLevel -) - -var levelMap = map[LogLevel]string{ - DebugLevel: "[DEBUG]", - InfoLevel: "[INFO]", - ErrorLevel: "[ERROR]", -} - -type logger struct { - level LogLevel - out io.Writer -} - -var ( - instance *logger - once sync.Once -) - -func getLogger() *logger { - once.Do(func() { - instance = &logger{ - level: DebugLevel, - out: os.Stdout, - } - }) - return instance -} - -func Init(level LogLevel, logFileName string) error { - log := getLogger() - log.level = level - - if logFileName != "" { - file, err := os.Create(logFileName) - if err != nil { - return fmt.Errorf("failed to open log file %s: %w", logFileName, err) - } - log.out = file - } else { - log.out = os.Stdout - } - - return nil -} - -func Destroy() { - log := getLogger() - if file, ok := log.out.(*os.File); ok { - file.Close() - } -} - -func (log *logger) log(level LogLevel, format string, args ...interface{}) { - if log.level > level { - return - } - - _, file, line, ok := runtime.Caller(2) - if !ok { - file, line = "---", 0 - } - time := time.Now().Format("2006-01-02 15:04:05") - levelStr := levelMap[level] - msg := fmt.Sprintf(format, args...) - - fmt.Fprintf(log.out, "[%s]%s[%s:%d]: %s \n", time, levelStr, filepath.Base(file), line, msg) -} - -func Debug(format string, args ...interface{}) { - getLogger().log(DebugLevel, format, args...) -} - -func Info(format string, args ...interface{}) { - getLogger().log(InfoLevel, format, args...) -} - -func Error(format string, args ...interface{}) { - getLogger().log(ErrorLevel, format, args...) -} diff --git a/pkg/runtime-stats/runtime-stats.go b/pkg/runtime-stats/runtime-stats.go index 40a00db..5c024d7 100644 --- a/pkg/runtime-stats/runtime-stats.go +++ b/pkg/runtime-stats/runtime-stats.go @@ -16,240 +16,136 @@ var MemRuntimeStats []MemRuntimeStat = []MemRuntimeStat{ { Name: "Alloc", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.Alloc) }, -======= - Get: func(m *runtime.MemStats) any { return m.Alloc }, ->>>>>>> main }, { Name: "BuckHashSys", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.BuckHashSys) }, -======= - Get: func(m *runtime.MemStats) any { return m.BuckHashSys }, ->>>>>>> main }, { Name: "Frees", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.Frees) }, -======= - Get: func(m *runtime.MemStats) any { return m.Frees }, ->>>>>>> main }, { Name: "GCCPUFraction", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.GCCPUFraction) }, }, { Name: "GCSys", Type: mtr.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.GCSys) }, -======= - Get: func(m *runtime.MemStats) any { return m.GCCPUFraction }, ->>>>>>> main }, { Name: "HeapAlloc", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.HeapAlloc) }, -======= - Get: func(m *runtime.MemStats) any { return m.HeapAlloc }, ->>>>>>> main }, { Name: "HeapIdle", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.HeapIdle) }, -======= - Get: func(m *runtime.MemStats) any { return m.HeapIdle }, ->>>>>>> main }, { Name: "HeapInuse", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.HeapInuse) }, -======= - Get: func(m *runtime.MemStats) any { return m.HeapInuse }, ->>>>>>> main }, { Name: "HeapObjects", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.HeapObjects) }, -======= - Get: func(m *runtime.MemStats) any { return m.HeapObjects }, ->>>>>>> main }, { Name: "HeapReleased", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.HeapReleased) }, -======= - Get: func(m *runtime.MemStats) any { return m.HeapReleased }, ->>>>>>> main }, { Name: "HeapSys", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.HeapSys) }, -======= - Get: func(m *runtime.MemStats) any { return m.HeapSys }, ->>>>>>> main }, { Name: "LastGC", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.LastGC) }, -======= - Get: func(m *runtime.MemStats) any { return m.LastGC }, ->>>>>>> main }, { Name: "Lookups", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.Lookups) }, -======= - Get: func(m *runtime.MemStats) any { return m.Lookups }, ->>>>>>> main }, { Name: "MCacheInuse", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.MCacheInuse) }, -======= - Get: func(m *runtime.MemStats) any { return m.MCacheInuse }, ->>>>>>> main }, { Name: "MCacheSys", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.MCacheSys) }, -======= - Get: func(m *runtime.MemStats) any { return m.MCacheSys }, ->>>>>>> main }, { Name: "MSpanInuse", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.MSpanInuse) }, -======= - Get: func(m *runtime.MemStats) any { return m.MSpanInuse }, ->>>>>>> main }, { Name: "MSpanSys", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.MSpanSys) }, -======= - Get: func(m *runtime.MemStats) any { return m.MSpanSys }, ->>>>>>> main }, { Name: "Mallocs", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.Mallocs) }, -======= - Get: func(m *runtime.MemStats) any { return m.Mallocs }, ->>>>>>> main }, { Name: "NextGC", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.NextGC) }, -======= - Get: func(m *runtime.MemStats) any { return m.NextGC }, ->>>>>>> main }, { Name: "NumForcedGC", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.NumForcedGC) }, -======= - Get: func(m *runtime.MemStats) any { return m.NumForcedGC }, ->>>>>>> main }, { Name: "NumGC", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.NumGC) }, -======= - Get: func(m *runtime.MemStats) any { return m.NumGC }, ->>>>>>> main }, { Name: "OtherSys", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.OtherSys) }, -======= - Get: func(m *runtime.MemStats) any { return m.OtherSys }, ->>>>>>> main }, { Name: "PauseTotalNs", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.PauseTotalNs) }, -======= - Get: func(m *runtime.MemStats) any { return m.PauseTotalNs }, ->>>>>>> main }, { Name: "StackInuse", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.StackInuse) }, -======= - Get: func(m *runtime.MemStats) any { return m.StackInuse }, ->>>>>>> main }, { Name: "StackSys", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.StackSys) }, -======= - Get: func(m *runtime.MemStats) any { return m.StackSys }, ->>>>>>> main }, { Name: "Sys", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.Sys) }, -======= - Get: func(m *runtime.MemStats) any { return m.Sys }, ->>>>>>> main }, { Name: "TotalAlloc", Type: mtr.GaugeType, -<<<<<<< HEAD Get: func(m *runtime.MemStats) any { return float64(m.TotalAlloc) }, -======= - Get: func(m *runtime.MemStats) any { return m.TotalAlloc }, ->>>>>>> main }, } From 3d09f163b9a475654022c13c45581267529572e5 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Fri, 11 Jul 2025 13:54:13 +0300 Subject: [PATCH 055/126] start write funcs options --- cmd/server/cfg_server.go | 103 +++++++++++++++---------------- internal/config/server-config.go | 65 +++++++++++++++++++ 2 files changed, 114 insertions(+), 54 deletions(-) create mode 100644 internal/config/server-config.go diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 30316f4..3dd796a 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -14,70 +14,65 @@ import ( "github.com/go-chi/chi/v5" "github.com/spf13/cobra" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" db "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/data-base" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) -const ( - defaultEndpoint = "localhost:8080" - defaultStoreInterval = 300 - defaultFileStoragePath = "/temp/metrics-db.json" - defaultRestoreOnStart = true -) - -type options struct { +var ( endPointAddr string storeInterval int fileStoragePath string restoreOnStart bool -} - -type envConfig struct { - EndPointAddr string `env:"ADDRESS"` - StoreInterval int `env:"STORE_INTERVAL"` - FileStoragePath string `env:"FILE_STORAGE_PATH"` - RestoreOnStart bool `env:"RESTORE"` -} - -var opts = &options{ - endPointAddr: defaultEndpoint, - storeInterval: defaultStoreInterval, - fileStoragePath: defaultFileStoragePath, - restoreOnStart: defaultRestoreOnStart, -} + opts *config.Options +) var rootCmd = &cobra.Command{ Use: "server", Short: "MetricService", Long: "MetricService", Args: cobra.NoArgs, - PreRunE: func(cmd *cobra.Command, args []string) error { - var cfg envConfig - err := env.Parse(&cfg) - if err != nil { - return fmt.Errorf("poll and report intervals must be > 0") + var envCfg config.EnvConfig + if err := env.Parse(&envCfg); err != nil { + return fmt.Errorf("failed to parse environment: %w", err) } - if cfg.EndPointAddr != "" { - opts.endPointAddr = cfg.EndPointAddr - } + var envOpts []func(*config.Options) - if _, _, err := net.SplitHostPort(opts.endPointAddr); err != nil { - return fmt.Errorf("invalid address %s: %w", opts.endPointAddr, err) + if envCfg.EndPointAddr != "" { + envOpts = append(envOpts, config.WithAddress(envCfg.EndPointAddr)) } - - if cfg.FileStoragePath != "" { - opts.fileStoragePath = cfg.FileStoragePath + if envCfg.StoreInterval != 0 { + envOpts = append(envOpts, config.WithStoreInterval(envCfg.StoreInterval)) + } + if envCfg.FileStoragePath != "" { + envOpts = append(envOpts, config.WithFileStoragePath(envCfg.FileStoragePath)) } + envOpts = append(envOpts, config.WithRestoreOnStart(envCfg.RestoreOnStart)) - if cfg.StoreInterval != 0 { - opts.storeInterval = cfg.StoreInterval + var flagOpts []func(*config.Options) + + if cmd.Flags().Changed("a") { + flagOpts = append(flagOpts, config.WithAddress(endPointAddr)) + } + if cmd.Flags().Changed("i") { + flagOpts = append(flagOpts, config.WithStoreInterval(storeInterval)) + } + if cmd.Flags().Changed("f") { + flagOpts = append(flagOpts, config.WithFileStoragePath(fileStoragePath)) + } + if cmd.Flags().Changed("r") { + flagOpts = append(flagOpts, config.WithRestoreOnStart(restoreOnStart)) } - opts.restoreOnStart = cfg.RestoreOnStart + opts = config.NewOptions(envOpts, flagOpts) + + if _, _, err := net.SplitHostPort(opts.EndPointAddr); err != nil { + return fmt.Errorf("invalid address %s: %w", opts.EndPointAddr, err) + } return nil }, @@ -103,13 +98,13 @@ var rootCmd = &cobra.Command{ } func init() { - rootCmd.Flags().StringVarP(&opts.endPointAddr, "a", "a", defaultEndpoint, "endpoint HTTP-server addr") - rootCmd.Flags().IntVarP(&opts.storeInterval, "i", "i", defaultStoreInterval, "store interval in seconds (0 = sync)") - rootCmd.Flags().StringVarP(&opts.fileStoragePath, "f", "f", defaultFileStoragePath, "file to store metrics") - rootCmd.Flags().BoolVarP(&opts.restoreOnStart, "r", "r", defaultRestoreOnStart, "restore metrics from file on start") + rootCmd.Flags().StringVarP(&endPointAddr, "a", "a", config.DefaultEndpoint, "endpoint HTTP-server addr") + rootCmd.Flags().IntVarP(&storeInterval, "i", "i", config.DefaultStoreInterval, "store interval in seconds (0 = sync)") + rootCmd.Flags().StringVarP(&fileStoragePath, "f", "f", config.DefaultFileStoragePath, "file to store metrics") + rootCmd.Flags().BoolVarP(&restoreOnStart, "r", "r", config.DefaultRestoreOnStart, "restore metrics from file on start") } -func NewRouter(storage ms.Collector, opts *options) http.Handler { +func NewRouter(storage ms.Collector, opts *config.Options) http.Handler { r := chi.NewRouter() r.Use(server.WithLogging) @@ -118,8 +113,8 @@ func NewRouter(storage ms.Collector, opts *options) http.Handler { r.Route("/", func(r chi.Router) { r.Get("/", server.GetAllMetrics(storage)) r.Route("/update", func(r chi.Router) { - if opts.storeInterval == 0 { - r.Use(db.WithSaveToDB(storage, opts.fileStoragePath)) + if opts.StoreInterval == 0 { + r.Use(db.WithSaveToDB(storage, opts.FileStoragePath)) } r.Post("/", server.UpdateMetricsHandlerJSON(storage)) @@ -135,22 +130,22 @@ func NewRouter(storage ms.Collector, opts *options) http.Handler { return r } -func startServer(opts *options) error { +func startServer(opts *config.Options) error { log.Info(). - Str("address", opts.endPointAddr). + Str("address", opts.EndPointAddr). Msg("Server configuration") storage := ms.NewMemStorage() r := NewRouter(storage, opts) - if opts.restoreOnStart { - if err := db.LoadFromDB(storage, opts.fileStoragePath); err != nil && !errors.Is(err, os.ErrNotExist) { + if opts.RestoreOnStart { + if err := db.LoadFromDB(storage, opts.FileStoragePath); err != nil && !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("LoadFromDB error %w", err) } } go func() { - ticker := time.NewTicker(time.Duration(opts.storeInterval) * time.Second) + ticker := time.NewTicker(time.Duration(opts.StoreInterval) * time.Second) defer ticker.Stop() stop := make(chan os.Signal, 1) @@ -159,12 +154,12 @@ func startServer(opts *options) error { for { select { case <-ticker.C: - if err := db.SaveToDB(storage, opts.fileStoragePath); err != nil { + if err := db.SaveToDB(storage, opts.FileStoragePath); err != nil { log.Error().Err(err).Msg("failed to save DB") } case <-stop: log.Info().Msg("Shutting down server, saving metrics") - if err := db.SaveToDB(storage, opts.fileStoragePath); err != nil { + if err := db.SaveToDB(storage, opts.FileStoragePath); err != nil { log.Error().Err(err).Msg("Failed to save metrics during shutdown") } os.Exit(0) @@ -172,7 +167,7 @@ func startServer(opts *options) error { } }() - if err := http.ListenAndServe(opts.endPointAddr, r); err != nil { + if err := http.ListenAndServe(opts.EndPointAddr, r); err != nil { fmt.Fprintf(os.Stderr, "HTTP-server didn't start: %v", err) panic(err) } diff --git a/internal/config/server-config.go b/internal/config/server-config.go new file mode 100644 index 0000000..94a5190 --- /dev/null +++ b/internal/config/server-config.go @@ -0,0 +1,65 @@ +package config + +const ( + DefaultEndpoint = "localhost:8080" + DefaultStoreInterval = 300 + DefaultFileStoragePath = "/temp/metrics-db.json" + DefaultRestoreOnStart = true +) + +type Options struct { + EndPointAddr string + StoreInterval int + FileStoragePath string + RestoreOnStart bool +} + +type EnvConfig struct { + EndPointAddr string `env:"ADDRESS"` + StoreInterval int `env:"STORE_INTERVAL"` + FileStoragePath string `env:"FILE_STORAGE_PATH"` + RestoreOnStart bool `env:"RESTORE"` +} + +func NewOptions(envOpts, flagOpts []func(*Options)) *Options { + opts := &Options{ + EndPointAddr: DefaultEndpoint, + StoreInterval: DefaultStoreInterval, + FileStoragePath: DefaultFileStoragePath, + RestoreOnStart: DefaultRestoreOnStart, + } + + for _, opt := range flagOpts { + opt(opts) + } + + for _, opt := range envOpts { + opt(opts) + } + + return opts +} + +func WithAddress(addr string) func(*Options) { + return func(o *Options) { + o.EndPointAddr = addr + } +} + +func WithStoreInterval(interval int) func(*Options) { + return func(o *Options) { + o.StoreInterval = interval + } +} + +func WithFileStoragePath(path string) func(*Options) { + return func(o *Options) { + o.FileStoragePath = path + } +} + +func WithRestoreOnStart(restore bool) func(*Options) { + return func(o *Options) { + o.RestoreOnStart = restore + } +} From 073585720f4481a036a82a176626218b475a2407 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Fri, 11 Jul 2025 14:52:35 +0300 Subject: [PATCH 056/126] add functional options pattern for server --- cmd/server/cfg_server.go | 76 +----- internal/config/server-config.go | 64 ++++- internal/handlers/server/server_test.go | 342 ++++++++++++------------ internal/router/roter.go | 38 +++ 4 files changed, 275 insertions(+), 245 deletions(-) create mode 100644 internal/router/roter.go diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 3dd796a..cd0ae04 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -3,20 +3,17 @@ package main import ( "errors" "fmt" - "net" "net/http" "os" "os/signal" "syscall" "time" - "github.com/caarlos0/env/v6" - "github.com/go-chi/chi/v5" "github.com/spf13/cobra" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/router" db "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/data-base" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -35,46 +32,9 @@ var rootCmd = &cobra.Command{ Long: "MetricService", Args: cobra.NoArgs, PreRunE: func(cmd *cobra.Command, args []string) error { - var envCfg config.EnvConfig - if err := env.Parse(&envCfg); err != nil { - return fmt.Errorf("failed to parse environment: %w", err) - } - - var envOpts []func(*config.Options) - - if envCfg.EndPointAddr != "" { - envOpts = append(envOpts, config.WithAddress(envCfg.EndPointAddr)) - } - if envCfg.StoreInterval != 0 { - envOpts = append(envOpts, config.WithStoreInterval(envCfg.StoreInterval)) - } - if envCfg.FileStoragePath != "" { - envOpts = append(envOpts, config.WithFileStoragePath(envCfg.FileStoragePath)) - } - envOpts = append(envOpts, config.WithRestoreOnStart(envCfg.RestoreOnStart)) - - var flagOpts []func(*config.Options) - - if cmd.Flags().Changed("a") { - flagOpts = append(flagOpts, config.WithAddress(endPointAddr)) - } - if cmd.Flags().Changed("i") { - flagOpts = append(flagOpts, config.WithStoreInterval(storeInterval)) - } - if cmd.Flags().Changed("f") { - flagOpts = append(flagOpts, config.WithFileStoragePath(fileStoragePath)) - } - if cmd.Flags().Changed("r") { - flagOpts = append(flagOpts, config.WithRestoreOnStart(restoreOnStart)) - } - - opts = config.NewOptions(envOpts, flagOpts) - - if _, _, err := net.SplitHostPort(opts.EndPointAddr); err != nil { - return fmt.Errorf("invalid address %s: %w", opts.EndPointAddr, err) - } - - return nil + var err error + opts, err = config.ParseOptionsFromCmd(cmd, endPointAddr, storeInterval, fileStoragePath, restoreOnStart) + return err }, RunE: func(cmd *cobra.Command, args []string) error { @@ -104,39 +64,13 @@ func init() { rootCmd.Flags().BoolVarP(&restoreOnStart, "r", "r", config.DefaultRestoreOnStart, "restore metrics from file on start") } -func NewRouter(storage ms.Collector, opts *config.Options) http.Handler { - r := chi.NewRouter() - - r.Use(server.WithLogging) - r.Use(server.WithGzipCompress) - - r.Route("/", func(r chi.Router) { - r.Get("/", server.GetAllMetrics(storage)) - r.Route("/update", func(r chi.Router) { - if opts.StoreInterval == 0 { - r.Use(db.WithSaveToDB(storage, opts.FileStoragePath)) - } - - r.Post("/", server.UpdateMetricsHandlerJSON(storage)) - r.Post("/{mType}/{mName}/{mValue}", server.UpdateMetric(storage)) - }) - - r.Route("/value", func(r chi.Router) { - r.Post("/", server.GetMetricsHandlerJSON(storage)) - r.Get("/{mType}/{mName}", server.GetMetric(storage)) - }) - }) - - return r -} - func startServer(opts *config.Options) error { log.Info(). Str("address", opts.EndPointAddr). Msg("Server configuration") storage := ms.NewMemStorage() - r := NewRouter(storage, opts) + r := router.NewRouter(storage, opts) if opts.RestoreOnStart { if err := db.LoadFromDB(storage, opts.FileStoragePath); err != nil && !errors.Is(err, os.ErrNotExist) { diff --git a/internal/config/server-config.go b/internal/config/server-config.go index 94a5190..cdb4807 100644 --- a/internal/config/server-config.go +++ b/internal/config/server-config.go @@ -1,5 +1,13 @@ package config +import ( + "fmt" + "net" + + "github.com/caarlos0/env/v6" + "github.com/spf13/cobra" +) + const ( DefaultEndpoint = "localhost:8080" DefaultStoreInterval = 300 @@ -21,7 +29,7 @@ type EnvConfig struct { RestoreOnStart bool `env:"RESTORE"` } -func NewOptions(envOpts, flagOpts []func(*Options)) *Options { +func NewServerOptions(envOpts, flagOpts []func(*Options)) *Options { opts := &Options{ EndPointAddr: DefaultEndpoint, StoreInterval: DefaultStoreInterval, @@ -63,3 +71,57 @@ func WithRestoreOnStart(restore bool) func(*Options) { o.RestoreOnStart = restore } } + +func ParseOptionsFromCmd(cmd *cobra.Command, endPointAddr string, storeInterval int, fileStoragePath string, restoreOnStart bool) (*Options, error) { + var envCfg EnvConfig + if err := env.Parse(&envCfg); err != nil { + return nil, fmt.Errorf("failed to parse environment: %w", err) + } + + var envOpts []func(*Options) + + if envCfg.EndPointAddr != "" { + envOpts = append(envOpts, WithAddress(envCfg.EndPointAddr)) + } + if envCfg.StoreInterval < 0 { + return nil, fmt.Errorf("store interval must be >= 0, got %d", envCfg.StoreInterval) + } else { + envOpts = append(envOpts, WithStoreInterval(envCfg.StoreInterval)) + } + if envCfg.FileStoragePath != "" { + envOpts = append(envOpts, WithFileStoragePath(envCfg.FileStoragePath)) + } + envOpts = append(envOpts, WithRestoreOnStart(envCfg.RestoreOnStart)) + + var flagOpts []func(*Options) + + if cmd.Flags().Changed("a") { + flagOpts = append(flagOpts, WithAddress(endPointAddr)) + } + + if cmd.Flags().Changed("i") { + if storeInterval < 0 { + return nil, fmt.Errorf("store interval flag must be >= 0, got %d", storeInterval) + } + flagOpts = append(flagOpts, WithStoreInterval(storeInterval)) + } + + if cmd.Flags().Changed("f") { + if fileStoragePath == "" { + return nil, fmt.Errorf("file storage path flag cannot be empty") + } + flagOpts = append(flagOpts, WithFileStoragePath(fileStoragePath)) + } + + if cmd.Flags().Changed("r") { + flagOpts = append(flagOpts, WithRestoreOnStart(restoreOnStart)) + } + + opts := NewServerOptions(envOpts, flagOpts) + + if _, _, err := net.SplitHostPort(opts.EndPointAddr); err != nil { + return nil, fmt.Errorf("invalid address %s: %w", opts.EndPointAddr, err) + } + + return opts, nil +} diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index 3bb5384..f0f0d78 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -1,175 +1,171 @@ package server_test -// -// import ( -// "io" -// "net/http" -// "net/http/httptest" -// "testing" -// -// ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" -// mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" -// log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" -// ) -// -// const ( -// defaultEndpoint = "localhost:8080" -// defaultStoreInterval = 300 -// defaultFileStoragePath = "/temp/metrics-db.json" -// defaultRestoreOnStart = true -// ) -// -// type options struct { -// endPointAddr string -// storeInterval int -// fileStoragePath string -// restoreOnStart bool -// } -// -// var DefaultOpts = &options{ -// endPointAddr: defaultEndpoint, -// storeInterval: defaultStoreInterval, -// fileStoragePath: defaultFileStoragePath, -// restoreOnStart: defaultRestoreOnStart, -// } -// -// func TestUpdateMetric(t *testing.T) { -// opts := *DefaultOpts -// -// storage := ms.NewMemStorage() -// router := server.NewRouter(storage, opts) -// -// tests := []struct { -// name string -// method string -// url string -// wantStatus int -// }{ -// { -// name: "Valid Gauge Update", -// method: http.MethodPost, -// url: "/update/gauge/testGauge/123.45", -// wantStatus: http.StatusOK, -// }, -// { -// name: "Valid Counter Update", -// method: http.MethodPost, -// url: "/update/counter/testCounter/100", -// wantStatus: http.StatusOK, -// }, -// { -// name: "Update Counter with another value", -// method: http.MethodPost, -// url: "/update/counter/testCounter/50", -// wantStatus: http.StatusOK, -// }, -// { -// name: "Invalid Gauge Value", -// method: http.MethodPost, -// url: "/update/gauge/invalidGauge/abc", -// wantStatus: http.StatusBadRequest, -// }, -// { -// name: "Invalid Counter Value", -// method: http.MethodPost, -// url: "/update/counter/invalidCounter/xyz", -// wantStatus: http.StatusBadRequest, -// }, -// { -// name: "Unknown Metric Type", -// method: http.MethodPost, -// url: "/update/unknown/testMetric/123", -// wantStatus: http.StatusBadRequest, -// }, -// { -// name: "Missing Metric Name", -// method: http.MethodPost, -// url: "/update/gauge//123.45", -// wantStatus: http.StatusBadRequest, -// }, -// } -// -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// req := httptest.NewRequest(tt.method, tt.url, nil) -// rr := httptest.NewRecorder() -// -// router.ServeHTTP(rr, req) -// -// if rr.Code != tt.wantStatus { -// t.Errorf("Test %s: expected status %d, got %d", tt.name, tt.wantStatus, rr.Code) -// } -// }) -// } -// } -// -// func TestGetMetric(t *testing.T) { -// opts := *DefaultOpts -// -// storage := ms.NewMemStorage() -// -// if err := storage.UpdateMetric(mtr.GaugeType, "cpu_usage", 75.5); err != nil { -// log.Error().Msgf("Failed to update metric cpu_usage: %v", err) -// } -// -// if err := storage.UpdateMetric(mtr.CounterType, "requests_total", int64(100)); err != nil { -// log.Error().Msgf("Failed to update metric requests_total: %v", err) -// } -// -// router := server.NewRouter(storage, opts) -// -// tests := []struct { -// name string -// url string -// wantStatus int -// wantBody string -// }{ -// { -// name: "Get Existing Gauge", -// url: "/value/gauge/cpu_usage", -// wantStatus: http.StatusOK, -// wantBody: "75.5", -// }, -// { -// name: "Get Existing Counter", -// url: "/value/counter/requests_total", -// wantStatus: http.StatusOK, -// wantBody: "100", -// }, -// { -// name: "Get Non-Existing Metric", -// url: "/value/gauge/non_existent", -// wantStatus: http.StatusNotFound, -// wantBody: "Metric non_existent was not found\n", -// }, -// { -// name: "Get Unknown Metric Type", -// url: "/value/invalid_type/some_metric", -// wantStatus: http.StatusNotFound, -// wantBody: "Metric some_metric was not found\n", -// }, -// { -// name: "GET with missing name", -// url: "/value/gauge/", -// wantStatus: http.StatusNotFound, -// wantBody: "404 page not found\n", -// }, -// } -// -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// req := httptest.NewRequest(http.MethodGet, tt.url, nil) -// rr := httptest.NewRecorder() -// -// router.ServeHTTP(rr, req) -// -// if rr.Code != tt.wantStatus { -// t.Errorf("Test %s: expected status %d, got %d", tt.name, tt.wantStatus, rr.Code) -// } -// body, _ := io.ReadAll(rr.Body) -// if string(body) != tt.wantBody { -// t.Errorf("Test %s: expected body %q, got %q", tt.name, tt.wantBody, string(body)) -// } -// }) -// } -// } +import ( + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" + mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/router" + log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" +) + +func TestUpdateMetric(t *testing.T) { + opts := &config.Options{} + for _, opt := range []func(*config.Options){ + config.WithAddress("localhost:8080"), + config.WithStoreInterval(300), + config.WithFileStoragePath("/tmp/metrics-db.json"), + config.WithRestoreOnStart(true), + } { + opt(opts) + } + + storage := ms.NewMemStorage() + router := router.NewRouter(storage, opts) + + tests := []struct { + name string + method string + url string + wantStatus int + }{ + { + name: "Valid Gauge Update", + method: http.MethodPost, + url: "/update/gauge/testGauge/123.45", + wantStatus: http.StatusOK, + }, + { + name: "Valid Counter Update", + method: http.MethodPost, + url: "/update/counter/testCounter/100", + wantStatus: http.StatusOK, + }, + { + name: "Update Counter with another value", + method: http.MethodPost, + url: "/update/counter/testCounter/50", + wantStatus: http.StatusOK, + }, + { + name: "Invalid Gauge Value", + method: http.MethodPost, + url: "/update/gauge/invalidGauge/abc", + wantStatus: http.StatusBadRequest, + }, + { + name: "Invalid Counter Value", + method: http.MethodPost, + url: "/update/counter/invalidCounter/xyz", + wantStatus: http.StatusBadRequest, + }, + { + name: "Unknown Metric Type", + method: http.MethodPost, + url: "/update/unknown/testMetric/123", + wantStatus: http.StatusBadRequest, + }, + { + name: "Missing Metric Name", + method: http.MethodPost, + url: "/update/gauge//123.45", + wantStatus: http.StatusBadRequest, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(tt.method, tt.url, nil) + rr := httptest.NewRecorder() + + router.ServeHTTP(rr, req) + + if rr.Code != tt.wantStatus { + t.Errorf("Test %s: expected status %d, got %d", tt.name, tt.wantStatus, rr.Code) + } + }) + } +} + +func TestGetMetric(t *testing.T) { + opts := &config.Options{} + for _, opt := range []func(*config.Options){ + config.WithAddress("localhost:8080"), + config.WithStoreInterval(300), + config.WithFileStoragePath("/tmp/metrics-db.json"), + config.WithRestoreOnStart(true), + } { + opt(opts) + } + + storage := ms.NewMemStorage() + + if err := storage.UpdateMetric(mtr.GaugeType, "cpu_usage", 75.5); err != nil { + log.Error().Msgf("Failed to update metric cpu_usage: %v", err) + } + + if err := storage.UpdateMetric(mtr.CounterType, "requests_total", int64(100)); err != nil { + log.Error().Msgf("Failed to update metric requests_total: %v", err) + } + + router := router.NewRouter(storage, opts) + + tests := []struct { + name string + url string + wantStatus int + wantBody string + }{ + { + name: "Get Existing Gauge", + url: "/value/gauge/cpu_usage", + wantStatus: http.StatusOK, + wantBody: "75.5", + }, + { + name: "Get Existing Counter", + url: "/value/counter/requests_total", + wantStatus: http.StatusOK, + wantBody: "100", + }, + { + name: "Get Non-Existing Metric", + url: "/value/gauge/non_existent", + wantStatus: http.StatusNotFound, + wantBody: "Metric non_existent was not found\n", + }, + { + name: "Get Unknown Metric Type", + url: "/value/invalid_type/some_metric", + wantStatus: http.StatusNotFound, + wantBody: "Metric some_metric was not found\n", + }, + { + name: "GET with missing name", + url: "/value/gauge/", + wantStatus: http.StatusNotFound, + wantBody: "404 page not found\n", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, tt.url, nil) + rr := httptest.NewRecorder() + + router.ServeHTTP(rr, req) + + if rr.Code != tt.wantStatus { + t.Errorf("Test %s: expected status %d, got %d", tt.name, tt.wantStatus, rr.Code) + } + body, _ := io.ReadAll(rr.Body) + if string(body) != tt.wantBody { + t.Errorf("Test %s: expected body %q, got %q", tt.name, tt.wantBody, string(body)) + } + }) + } +} diff --git a/internal/router/roter.go b/internal/router/roter.go new file mode 100644 index 0000000..269f1a5 --- /dev/null +++ b/internal/router/roter.go @@ -0,0 +1,38 @@ +package router + +import ( + "net/http" + + "github.com/go-chi/chi/v5" + + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" + ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" + db "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/data-base" +) + +func NewRouter(storage ms.Collector, opts *config.Options) http.Handler { + r := chi.NewRouter() + + r.Use(server.WithLogging) + r.Use(server.WithGzipCompress) + + r.Route("/", func(r chi.Router) { + r.Get("/", server.GetAllMetrics(storage)) + r.Route("/update", func(r chi.Router) { + if opts.StoreInterval == 0 { + r.Use(db.WithSaveToDB(storage, opts.FileStoragePath)) + } + + r.Post("/", server.UpdateMetricsHandlerJSON(storage)) + r.Post("/{mType}/{mName}/{mValue}", server.UpdateMetric(storage)) + }) + + r.Route("/value", func(r chi.Router) { + r.Post("/", server.GetMetricsHandlerJSON(storage)) + r.Get("/{mType}/{mName}", server.GetMetric(storage)) + }) + }) + + return r +} From 621ba8ef3b55b24a28b4622040d16ae492fa0592 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Fri, 11 Jul 2025 15:48:56 +0300 Subject: [PATCH 057/126] add functional options pattern --- cmd/server/cfg_server.go | 44 ++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index cd0ae04..582a4f3 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -78,30 +78,34 @@ func startServer(opts *config.Options) error { } } - go func() { - ticker := time.NewTicker(time.Duration(opts.StoreInterval) * time.Second) - defer ticker.Stop() - - stop := make(chan os.Signal, 1) - signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) - - for { - select { - case <-ticker.C: - if err := db.SaveToDB(storage, opts.FileStoragePath); err != nil { - log.Error().Err(err).Msg("failed to save DB") + if opts.StoreInterval > 0 { + go func() { + ticker := time.NewTicker(time.Duration(opts.StoreInterval) * time.Second) + defer ticker.Stop() + + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) + + for { + select { + case <-ticker.C: + if err := db.SaveToDB(storage, opts.FileStoragePath); err != nil { + log.Error().Err(err).Msg("failed to save DB") + } + case <-stop: + log.Info().Msg("Shutting down server, saving metrics") + if err := db.SaveToDB(storage, opts.FileStoragePath); err != nil { + log.Error().Err(err).Msg("Failed to save metrics during shutdown") + } + os.Exit(0) } - case <-stop: - log.Info().Msg("Shutting down server, saving metrics") - if err := db.SaveToDB(storage, opts.FileStoragePath); err != nil { - log.Error().Err(err).Msg("Failed to save metrics during shutdown") - } - os.Exit(0) } - } - }() + }() + } + log.Info().Msg("Starting HTTP server...") if err := http.ListenAndServe(opts.EndPointAddr, r); err != nil { + log.Error().Err(err).Msg("HTTP server failed to start") fmt.Fprintf(os.Stderr, "HTTP-server didn't start: %v", err) panic(err) } From 89acb00fcd9873bc2089b83fec81b2eda3a6ea7c Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Fri, 11 Jul 2025 15:49:58 +0300 Subject: [PATCH 058/126] add go mod download --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index c9bb312..5b467f0 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ test: @go test ./... -v lint: + @go mod download @golangci-lint run clean: From 220626ef86f69ca08f1479ec27f9a624ef710cca Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Fri, 11 Jul 2025 18:10:43 +0300 Subject: [PATCH 059/126] add context --- cmd/agent/cfg_agent.go | 21 +++++++++++----- cmd/server/cfg_server.go | 54 ++++++++++++++++++++++++++++++++-------- 2 files changed, 58 insertions(+), 17 deletions(-) diff --git a/cmd/agent/cfg_agent.go b/cmd/agent/cfg_agent.go index e78e028..2721551 100644 --- a/cmd/agent/cfg_agent.go +++ b/cmd/agent/cfg_agent.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "net" "os" @@ -79,7 +80,18 @@ var rootCmd = &cobra.Command{ return fmt.Errorf("invalid address %s: %w", opts.endPointAddr, err) } - startAgent() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) + + go func(){ + <-stop + cancel() + }() + + startAgent(ctx) return nil }, @@ -95,7 +107,7 @@ func init() { rootCmd.Flags().IntVarP(&opts.reportInterval, "r", "r", opts.reportInterval, "PollInterval value") } -func startAgent() { +func startAgent(ctx context.Context) { storage := ms.NewMemStorage() client := resty.New(). @@ -104,9 +116,6 @@ func startAgent() { log.Info().Msg("Starting collection and reporting loops") - stop := make(chan os.Signal, 1) - signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) - pollTimer := time.NewTicker(time.Duration(opts.pollInterval) * time.Second) reportTimer := time.NewTicker(time.Duration(opts.reportInterval) * time.Second) @@ -115,7 +124,7 @@ func startAgent() { for { select { - case <-stop: + case <-ctx.Done(): return case <-pollTimer.C: diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 582a4f3..d21ea72 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -1,6 +1,7 @@ package main import ( + "context" "errors" "fmt" "net/http" @@ -49,11 +50,26 @@ var rootCmd = &cobra.Command{ } }() - if err := startServer(opts); err != nil { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) + + serverErrCh := make(chan error, 1) + go func() { + serverErrCh <- startServer(ctx, opts) + }() + + select { + case sig := <-stop: + log.Info().Str("signal", sig.String()).Msg("Received shutdown signal") + cancel() + err := <-serverErrCh + return err + case err := <-serverErrCh: return err } - - return nil }, } @@ -64,7 +80,7 @@ func init() { rootCmd.Flags().BoolVarP(&restoreOnStart, "r", "r", config.DefaultRestoreOnStart, "restore metrics from file on start") } -func startServer(opts *config.Options) error { +func startServer(ctx context.Context, opts *config.Options) error { log.Info(). Str("address", opts.EndPointAddr). Msg("Server configuration") @@ -92,23 +108,39 @@ func startServer(opts *config.Options) error { if err := db.SaveToDB(storage, opts.FileStoragePath); err != nil { log.Error().Err(err).Msg("failed to save DB") } - case <-stop: + case <-ctx.Done(): log.Info().Msg("Shutting down server, saving metrics") if err := db.SaveToDB(storage, opts.FileStoragePath); err != nil { log.Error().Err(err).Msg("Failed to save metrics during shutdown") } - os.Exit(0) + return } } }() } - log.Info().Msg("Starting HTTP server...") - if err := http.ListenAndServe(opts.EndPointAddr, r); err != nil { - log.Error().Err(err).Msg("HTTP server failed to start") - fmt.Fprintf(os.Stderr, "HTTP-server didn't start: %v", err) - panic(err) + srv := &http.Server{ + Addr: opts.EndPointAddr, + Handler: r, } + go func() { + log.Info().Msg("Starting HTTP server...") + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatal().Err(err).Msg("HTTP server failed unexpectedly") + } + }() + + <-ctx.Done() + shutdownCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + + if err := srv.Shutdown(shutdownCtx); err != nil { + log.Error().Err(err).Msg("Failed to gracefully shutdown server") + return err + } + + log.Info().Msg("Server gracefully stopped") + return nil } From 08107ed4517258ca453edd80e9462af1c1170617 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Sat, 12 Jul 2025 00:30:45 +0300 Subject: [PATCH 060/126] first version pingDataBase --- cmd/server/cfg_server.go | 28 ++++++++++++++++++------- go.mod | 8 ++++++- go.sum | 26 +++++++++++++++++++++++ internal/config/server-config.go | 20 +++++++++++++++--- internal/handlers/server/database.go | 31 ++++++++++++++++++++++++++++ internal/router/roter.go | 11 +++++++--- 6 files changed, 109 insertions(+), 15 deletions(-) create mode 100644 internal/handlers/server/database.go diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index d21ea72..70afcb8 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -2,6 +2,7 @@ package main import ( "context" + "database/sql" "errors" "fmt" "net/http" @@ -10,12 +11,13 @@ import ( "syscall" "time" + _ "github.com/jackc/pgx/v5/stdlib" "github.com/spf13/cobra" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/router" - db "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/data-base" + database "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/data-base" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -24,6 +26,7 @@ var ( storeInterval int fileStoragePath string restoreOnStart bool + dataBaseDSN string opts *config.Options ) @@ -34,7 +37,7 @@ var rootCmd = &cobra.Command{ Args: cobra.NoArgs, PreRunE: func(cmd *cobra.Command, args []string) error { var err error - opts, err = config.ParseOptionsFromCmd(cmd, endPointAddr, storeInterval, fileStoragePath, restoreOnStart) + opts, err = config.ParseOptionsFromCmd(cmd, endPointAddr, storeInterval, fileStoragePath, restoreOnStart, dataBaseDSN) return err }, @@ -50,6 +53,14 @@ var rootCmd = &cobra.Command{ } }() + log.Info().Msgf("DSN: %q", opts.DataBaseDSN) + db, err := sql.Open("pgx", opts.DataBaseDSN) + if err != nil { + log.Error().Err(err).Msg("sql.Open error") + panic(err) + } + defer db.Close() + ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -58,7 +69,7 @@ var rootCmd = &cobra.Command{ serverErrCh := make(chan error, 1) go func() { - serverErrCh <- startServer(ctx, opts) + serverErrCh <- startServer(ctx, opts, db) }() select { @@ -78,18 +89,19 @@ func init() { rootCmd.Flags().IntVarP(&storeInterval, "i", "i", config.DefaultStoreInterval, "store interval in seconds (0 = sync)") rootCmd.Flags().StringVarP(&fileStoragePath, "f", "f", config.DefaultFileStoragePath, "file to store metrics") rootCmd.Flags().BoolVarP(&restoreOnStart, "r", "r", config.DefaultRestoreOnStart, "restore metrics from file on start") + rootCmd.Flags().StringVarP(&dataBaseDSN, "d", "d", config.DefaultDataBaseDSN, "database dsn") } -func startServer(ctx context.Context, opts *config.Options) error { +func startServer(ctx context.Context, opts *config.Options, db *sql.DB) error { log.Info(). Str("address", opts.EndPointAddr). Msg("Server configuration") storage := ms.NewMemStorage() - r := router.NewRouter(storage, opts) + r := router.NewRouter(storage, opts, db) if opts.RestoreOnStart { - if err := db.LoadFromDB(storage, opts.FileStoragePath); err != nil && !errors.Is(err, os.ErrNotExist) { + if err := database.LoadFromDB(storage, opts.FileStoragePath); err != nil && !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("LoadFromDB error %w", err) } } @@ -105,12 +117,12 @@ func startServer(ctx context.Context, opts *config.Options) error { for { select { case <-ticker.C: - if err := db.SaveToDB(storage, opts.FileStoragePath); err != nil { + if err := database.SaveToDB(storage, opts.FileStoragePath); err != nil { log.Error().Err(err).Msg("failed to save DB") } case <-ctx.Done(): log.Info().Msg("Shutting down server, saving metrics") - if err := db.SaveToDB(storage, opts.FileStoragePath); err != nil { + if err := database.SaveToDB(storage, opts.FileStoragePath); err != nil { log.Error().Err(err).Msg("Failed to save metrics during shutdown") } return diff --git a/go.mod b/go.mod index 02ac444..48383dc 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/caarlos0/env/v6 v6.10.1 github.com/go-chi/chi/v5 v5.2.2 github.com/go-resty/resty/v2 v2.16.5 + github.com/jackc/pgx/v5 v5.7.5 github.com/mailru/easyjson v0.9.0 github.com/rs/zerolog v1.34.0 github.com/spf13/cobra v1.9.1 @@ -13,11 +14,16 @@ require ( require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/spf13/pflag v1.0.6 // indirect + golang.org/x/crypto v0.37.0 // indirect golang.org/x/net v0.33.0 // indirect + golang.org/x/sync v0.13.0 // indirect golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.24.0 // indirect ) - diff --git a/go.sum b/go.sum index 265104c..06a96de 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,9 @@ github.com/caarlos0/env/v6 v6.10.1 h1:t1mPSxNpei6M5yAeu1qtRdPAK29Nbcf/n3G7x+b3/I github.com/caarlos0/env/v6 v6.10.1/go.mod h1:hvp/ryKXKipEkcuYjs9mI4bBCg+UI0Yhgm5Zu0ddvwc= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618= github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM= @@ -9,6 +12,14 @@ github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= +github.com/jackc/pgx/v5 v5.7.5/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= @@ -21,6 +32,8 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= @@ -29,14 +42,27 @@ github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/config/server-config.go b/internal/config/server-config.go index cdb4807..4facd72 100644 --- a/internal/config/server-config.go +++ b/internal/config/server-config.go @@ -13,6 +13,7 @@ const ( DefaultStoreInterval = 300 DefaultFileStoragePath = "/temp/metrics-db.json" DefaultRestoreOnStart = true + DefaultDataBaseDSN = "" ) type Options struct { @@ -20,6 +21,7 @@ type Options struct { StoreInterval int FileStoragePath string RestoreOnStart bool + DataBaseDSN string } type EnvConfig struct { @@ -27,6 +29,7 @@ type EnvConfig struct { StoreInterval int `env:"STORE_INTERVAL"` FileStoragePath string `env:"FILE_STORAGE_PATH"` RestoreOnStart bool `env:"RESTORE"` + DataBaseDSN string `env:"DATABASE_DSN"` } func NewServerOptions(envOpts, flagOpts []func(*Options)) *Options { @@ -72,14 +75,23 @@ func WithRestoreOnStart(restore bool) func(*Options) { } } -func ParseOptionsFromCmd(cmd *cobra.Command, endPointAddr string, storeInterval int, fileStoragePath string, restoreOnStart bool) (*Options, error) { +func WithDataBaseDSN(dataBaseDSN string) func(*Options) { + return func(o *Options) { + o.DataBaseDSN = dataBaseDSN + } +} + +func ParseOptionsFromCmd(cmd *cobra.Command, endPointAddr string, storeInterval int, fileStoragePath string, + restoreOnStart bool, dataBaseDSN string) (*Options, error) { var envCfg EnvConfig if err := env.Parse(&envCfg); err != nil { return nil, fmt.Errorf("failed to parse environment: %w", err) } var envOpts []func(*Options) - + if envCfg.DataBaseDSN != "" { + envOpts = append(envOpts, WithDataBaseDSN(envCfg.DataBaseDSN)) + } if envCfg.EndPointAddr != "" { envOpts = append(envOpts, WithAddress(envCfg.EndPointAddr)) } @@ -94,7 +106,9 @@ func ParseOptionsFromCmd(cmd *cobra.Command, endPointAddr string, storeInterval envOpts = append(envOpts, WithRestoreOnStart(envCfg.RestoreOnStart)) var flagOpts []func(*Options) - + if cmd.Flags().Changed("d") { + flagOpts = append(flagOpts, WithDataBaseDSN(dataBaseDSN)) + } if cmd.Flags().Changed("a") { flagOpts = append(flagOpts, WithAddress(endPointAddr)) } diff --git a/internal/handlers/server/database.go b/internal/handlers/server/database.go new file mode 100644 index 0000000..26d6d16 --- /dev/null +++ b/internal/handlers/server/database.go @@ -0,0 +1,31 @@ +package server + +import ( + "context" + "database/sql" + "fmt" + "net/http" + "time" + + log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" +) + +func WithDataBase(db *sql.DB, next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + if err := db.PingContext(ctx); err != nil { + log.Error().Err(err).Msg("database ping failed") + http.Error(w, fmt.Sprintf("database ping failed: %v", err), http.StatusInternalServerError) + return + } + log.Info().Msg("database ping successful") + + next(w, r) + } +} + +func PingDataBase(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} diff --git a/internal/router/roter.go b/internal/router/roter.go index 269f1a5..a0fc307 100644 --- a/internal/router/roter.go +++ b/internal/router/roter.go @@ -1,6 +1,7 @@ package router import ( + "database/sql" "net/http" "github.com/go-chi/chi/v5" @@ -8,10 +9,10 @@ import ( "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" - db "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/data-base" + database "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/data-base" ) -func NewRouter(storage ms.Collector, opts *config.Options) http.Handler { +func NewRouter(storage ms.Collector, opts *config.Options, db *sql.DB) http.Handler { r := chi.NewRouter() r.Use(server.WithLogging) @@ -21,7 +22,7 @@ func NewRouter(storage ms.Collector, opts *config.Options) http.Handler { r.Get("/", server.GetAllMetrics(storage)) r.Route("/update", func(r chi.Router) { if opts.StoreInterval == 0 { - r.Use(db.WithSaveToDB(storage, opts.FileStoragePath)) + r.Use(database.WithSaveToDB(storage, opts.FileStoragePath)) } r.Post("/", server.UpdateMetricsHandlerJSON(storage)) @@ -32,6 +33,10 @@ func NewRouter(storage ms.Collector, opts *config.Options) http.Handler { r.Post("/", server.GetMetricsHandlerJSON(storage)) r.Get("/{mType}/{mName}", server.GetMetric(storage)) }) + + r.Route("/ping", func(r chi.Router) { + r.Get("/", server.WithDataBase(db, server.PingDataBase)) + }) }) return r From d6b3471e889dac0c01edb811aa6342e31de40800 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Sat, 12 Jul 2025 01:09:17 +0300 Subject: [PATCH 061/126] fix my tests --- cmd/server/cfg_server.go | 8 +++++-- internal/handlers/server/server_test.go | 29 +++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 70afcb8..777c6ce 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -53,13 +53,17 @@ var rootCmd = &cobra.Command{ } }() - log.Info().Msgf("DSN: %q", opts.DataBaseDSN) + log.Info().Msgf("DSN: <%s>", opts.DataBaseDSN) db, err := sql.Open("pgx", opts.DataBaseDSN) if err != nil { log.Error().Err(err).Msg("sql.Open error") panic(err) } - defer db.Close() + defer func() { + if err := db.Close(); err != nil { + log.Error().Err(err).Msg("Failed to db.Close") + } + }() ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index f0f0d78..d90623c 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -1,6 +1,7 @@ package server_test import ( + "database/sql" "io" "net/http" "net/http/httptest" @@ -24,8 +25,20 @@ func TestUpdateMetric(t *testing.T) { opt(opts) } + //FIXME + db, err := sql.Open("pgx", opts.DataBaseDSN) + if err != nil { + log.Error().Err(err).Msg("sql.Open error") + panic(err) + } + defer func() { + if err := db.Close(); err != nil { + log.Error().Err(err).Msg("Failed to db.Close") + } + }() + storage := ms.NewMemStorage() - router := router.NewRouter(storage, opts) + router := router.NewRouter(storage, opts, db) tests := []struct { name string @@ -102,6 +115,18 @@ func TestGetMetric(t *testing.T) { opt(opts) } + //FIXME + db, err := sql.Open("pgx", opts.DataBaseDSN) + if err != nil { + log.Error().Err(err).Msg("sql.Open error") + panic(err) + } + defer func() { + if err := db.Close(); err != nil { + log.Error().Err(err).Msg("Failed to db.Close") + } + }() + storage := ms.NewMemStorage() if err := storage.UpdateMetric(mtr.GaugeType, "cpu_usage", 75.5); err != nil { @@ -112,7 +137,7 @@ func TestGetMetric(t *testing.T) { log.Error().Msgf("Failed to update metric requests_total: %v", err) } - router := router.NewRouter(storage, opts) + router := router.NewRouter(storage, opts, db) tests := []struct { name string From 2dbdc198e69ce091fbb77cc25f5bf69a1521f065 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Sat, 12 Jul 2025 02:20:43 +0300 Subject: [PATCH 062/126] empty commit --- cmd/server/cfg_server.go | 3 --- internal/handlers/server/server_test.go | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 777c6ce..f3efab8 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -115,9 +115,6 @@ func startServer(ctx context.Context, opts *config.Options, db *sql.DB) error { ticker := time.NewTicker(time.Duration(opts.StoreInterval) * time.Second) defer ticker.Stop() - stop := make(chan os.Signal, 1) - signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) - for { select { case <-ticker.C: diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index d90623c..410da65 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -7,6 +7,8 @@ import ( "net/http/httptest" "testing" + _ "github.com/jackc/pgx/v5/stdlib" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" From da3dcb145022441c39861acc5bd099d8c34576a2 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Sat, 12 Jul 2025 13:35:03 +0300 Subject: [PATCH 063/126] empty commit2 --- cmd/server/cfg_server.go | 1 + internal/config/server-config.go | 6 ++++-- internal/handlers/server/database.go | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index f3efab8..9885a7b 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -53,6 +53,7 @@ var rootCmd = &cobra.Command{ } }() + //opts.DataBaseDSN = "postgres://postgres:postgres@localhost:5432/mipt?sslmode=disable" log.Info().Msgf("DSN: <%s>", opts.DataBaseDSN) db, err := sql.Open("pgx", opts.DataBaseDSN) if err != nil { diff --git a/internal/config/server-config.go b/internal/config/server-config.go index 4facd72..f32280c 100644 --- a/internal/config/server-config.go +++ b/internal/config/server-config.go @@ -3,6 +3,7 @@ package config import ( "fmt" "net" + "strings" "github.com/caarlos0/env/v6" "github.com/spf13/cobra" @@ -82,7 +83,7 @@ func WithDataBaseDSN(dataBaseDSN string) func(*Options) { } func ParseOptionsFromCmd(cmd *cobra.Command, endPointAddr string, storeInterval int, fileStoragePath string, - restoreOnStart bool, dataBaseDSN string) (*Options, error) { + restoreOnStart bool, dataBaseDSN string) (*Options, error) { var envCfg EnvConfig if err := env.Parse(&envCfg); err != nil { return nil, fmt.Errorf("failed to parse environment: %w", err) @@ -90,7 +91,8 @@ func ParseOptionsFromCmd(cmd *cobra.Command, endPointAddr string, storeInterval var envOpts []func(*Options) if envCfg.DataBaseDSN != "" { - envOpts = append(envOpts, WithDataBaseDSN(envCfg.DataBaseDSN)) + cleanDSN := strings.Trim(envCfg.DataBaseDSN, "'") + envOpts = append(envOpts, WithDataBaseDSN(cleanDSN)) } if envCfg.EndPointAddr != "" { envOpts = append(envOpts, WithAddress(envCfg.EndPointAddr)) diff --git a/internal/handlers/server/database.go b/internal/handlers/server/database.go index 26d6d16..92926cf 100644 --- a/internal/handlers/server/database.go +++ b/internal/handlers/server/database.go @@ -12,7 +12,7 @@ import ( func WithDataBase(db *sql.DB, next http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() if err := db.PingContext(ctx); err != nil { From e6d73adc4fffaac66867f8247bef16b2bee7c0e7 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Sat, 12 Jul 2025 14:07:52 +0300 Subject: [PATCH 064/126] some corrections --- cmd/server/cfg_server.go | 5 ++--- internal/handlers/server/database.go | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 9885a7b..84a1fac 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -53,8 +53,7 @@ var rootCmd = &cobra.Command{ } }() - //opts.DataBaseDSN = "postgres://postgres:postgres@localhost:5432/mipt?sslmode=disable" - log.Info().Msgf("DSN: <%s>", opts.DataBaseDSN) + log.Info().Msgf("DSN: %s", opts.DataBaseDSN) db, err := sql.Open("pgx", opts.DataBaseDSN) if err != nil { log.Error().Err(err).Msg("sql.Open error") @@ -146,7 +145,7 @@ func startServer(ctx context.Context, opts *config.Options, db *sql.DB) error { }() <-ctx.Done() - shutdownCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + shutdownCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() if err := srv.Shutdown(shutdownCtx); err != nil { diff --git a/internal/handlers/server/database.go b/internal/handlers/server/database.go index 92926cf..26d6d16 100644 --- a/internal/handlers/server/database.go +++ b/internal/handlers/server/database.go @@ -12,7 +12,7 @@ import ( func WithDataBase(db *sql.DB, next http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() if err := db.PingContext(ctx); err != nil { From 9ebe508523e680eb5fbde6b50f3bdf52cc4d351f Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Sat, 12 Jul 2025 18:17:19 +0300 Subject: [PATCH 065/126] change dirs --- cmd/server/cfg_server.go | 111 +++++++++--------- internal/handlers/agent/agent.go | 7 +- internal/handlers/server/server.go | 23 +--- internal/memstorage/storage.go | 104 ---------------- internal/metrics/metrics.go | 14 ++- .../metrics_easyjson.go} | 40 +++++-- .../{DBfunctions.go => file-storage.go} | 8 +- 7 files changed, 112 insertions(+), 195 deletions(-) delete mode 100644 internal/memstorage/storage.go rename internal/{handlers/server/server_easyjson.go => metrics/metrics_easyjson.go} (63%) rename pkg/data-base/{DBfunctions.go => file-storage.go} (92%) diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 84a1fac..a4f2b37 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -31,69 +31,72 @@ var ( ) var rootCmd = &cobra.Command{ - Use: "server", - Short: "MetricService", - Long: "MetricService", - Args: cobra.NoArgs, - PreRunE: func(cmd *cobra.Command, args []string) error { - var err error - opts, err = config.ParseOptionsFromCmd(cmd, endPointAddr, storeInterval, fileStoragePath, restoreOnStart, dataBaseDSN) - return err - }, + Use: "server", + Short: "MetricService", + Long: "MetricService", + Args: cobra.NoArgs, + PreRunE: PreRunE, + RunE: RunE, +} - RunE: func(cmd *cobra.Command, args []string) error { - logFile, err := log.InitLogger("logFileServer.log") - if err != nil { - return fmt.Errorf("logger init error: %w", err) - } +func init() { + rootCmd.Flags().StringVarP(&endPointAddr, "a", "a", config.DefaultEndpoint, "endpoint HTTP-server addr") + rootCmd.Flags().IntVarP(&storeInterval, "i", "i", config.DefaultStoreInterval, "store interval in seconds (0 = sync)") + rootCmd.Flags().StringVarP(&fileStoragePath, "f", "f", config.DefaultFileStoragePath, "file to store metrics") + rootCmd.Flags().BoolVarP(&restoreOnStart, "r", "r", config.DefaultRestoreOnStart, "restore metrics from file on start") + rootCmd.Flags().StringVarP(&dataBaseDSN, "d", "d", config.DefaultDataBaseDSN, "database dsn") +} - defer func() { - if err := logFile.Close(); err != nil { - log.Error().Err(err).Msg("Failed to close log file") - } - }() +func PreRunE(cmd *cobra.Command, args []string) error { + var err error + opts, err = config.ParseOptionsFromCmd(cmd, endPointAddr, storeInterval, fileStoragePath, restoreOnStart, dataBaseDSN) + return err +} + +func RunE(cmd *cobra.Command, args []string) error { + logFile, err := log.InitLogger("logFileServer.log") + if err != nil { + return fmt.Errorf("logger init error: %w", err) + } - log.Info().Msgf("DSN: %s", opts.DataBaseDSN) - db, err := sql.Open("pgx", opts.DataBaseDSN) - if err != nil { - log.Error().Err(err).Msg("sql.Open error") - panic(err) + defer func() { + if err := logFile.Close(); err != nil { + log.Error().Err(err).Msg("Failed to close log file") } - defer func() { - if err := db.Close(); err != nil { - log.Error().Err(err).Msg("Failed to db.Close") - } - }() + }() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + log.Info().Msgf("DSN: %s", opts.DataBaseDSN) + db, err := sql.Open("pgx", opts.DataBaseDSN) + if err != nil { + log.Error().Err(err).Msg("sql.Open error") + panic(err) + } + defer func() { + if err := db.Close(); err != nil { + log.Error().Err(err).Msg("Failed to db.Close") + } + }() - stop := make(chan os.Signal, 1) - signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - serverErrCh := make(chan error, 1) - go func() { - serverErrCh <- startServer(ctx, opts, db) - }() + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) - select { - case sig := <-stop: - log.Info().Str("signal", sig.String()).Msg("Received shutdown signal") - cancel() - err := <-serverErrCh - return err - case err := <-serverErrCh: - return err - } - }, -} + serverErrCh := make(chan error, 1) + go func() { + serverErrCh <- startServer(ctx, opts, db) + }() -func init() { - rootCmd.Flags().StringVarP(&endPointAddr, "a", "a", config.DefaultEndpoint, "endpoint HTTP-server addr") - rootCmd.Flags().IntVarP(&storeInterval, "i", "i", config.DefaultStoreInterval, "store interval in seconds (0 = sync)") - rootCmd.Flags().StringVarP(&fileStoragePath, "f", "f", config.DefaultFileStoragePath, "file to store metrics") - rootCmd.Flags().BoolVarP(&restoreOnStart, "r", "r", config.DefaultRestoreOnStart, "restore metrics from file on start") - rootCmd.Flags().StringVarP(&dataBaseDSN, "d", "d", config.DefaultDataBaseDSN, "database dsn") + select { + case sig := <-stop: + log.Info().Str("signal", sig.String()).Msg("Received shutdown signal") + cancel() + err := <-serverErrCh + return err + case err := <-serverErrCh: + return err + } } func startServer(ctx context.Context, opts *config.Options, db *sql.DB) error { diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 8557f21..d81698d 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -12,7 +12,6 @@ import ( "github.com/go-resty/resty/v2" "github.com/mailru/easyjson" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" @@ -53,7 +52,7 @@ func SendAllMetrics(client *resty.Client, storage *ms.MemStorage) { mType := metric.Type() mName := metric.Name() - metricJSON := server.Metrics{ + metricJSON := mtr.Metrics{ ID: mName, MType: mType, } @@ -84,7 +83,7 @@ func SendAllMetrics(client *resty.Client, storage *ms.MemStorage) { } } -func sendMetric(client *resty.Client, metricJSON *server.Metrics) { +func sendMetric(client *resty.Client, metricJSON *mtr.Metrics) { backoffSchedule := []time.Duration{ 100 * time.Millisecond, 500 * time.Millisecond, @@ -113,7 +112,7 @@ func sendMetric(client *resty.Client, metricJSON *server.Metrics) { } } -func ConvertToGzipData(metricJSON *server.Metrics) (*bytes.Buffer, error) { +func ConvertToGzipData(metricJSON *mtr.Metrics) (*bytes.Buffer, error) { jsonData, err := easyjson.Marshal(*metricJSON) if err != nil { log.Error().Err(err).Msg("Failed to marshal metricJSON") diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index 7eb5d11..7e774c0 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -14,19 +14,6 @@ import ( log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) -type Metrics struct { - ID string `json:"id"` - MType string `json:"type"` - Delta *int64 `json:"delta,omitempty"` - Value *float64 `json:"value,omitempty"` -} - -type MetricTable struct { - Name string - Type string - Value string -} - func ConvertByType(mType, mValue string) (any, error) { switch mType { case mtr.GaugeType: @@ -82,7 +69,7 @@ func GetAllMetrics(storage ms.Collector) http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { allMetrics := storage.GetAllMetrics() - var metricsToTable []MetricTable + var metricsToTable []mtr.MetricTable for _, metric := range allMetrics { var valStr string @@ -109,7 +96,7 @@ func GetAllMetrics(storage ms.Collector) http.HandlerFunc { valStr = strconv.FormatInt(val, 10) } - metricsToTable = append(metricsToTable, MetricTable{ + metricsToTable = append(metricsToTable, mtr.MetricTable{ Name: mName, Type: mType, Value: valStr, @@ -201,7 +188,7 @@ func UpdateMetric(storage ms.Collector) http.HandlerFunc { } } -func FillMetricValueFromStorage(storage ms.Collector, metric *Metrics) bool { +func FillMetricValueFromStorage(storage ms.Collector, metric *mtr.Metrics) bool { value, ok := storage.GetMetric(metric.MType, metric.ID) if !ok { return false @@ -226,7 +213,7 @@ func FillMetricValueFromStorage(storage ms.Collector, metric *Metrics) bool { func GetMetricsHandlerJSON(storage ms.Collector) http.HandlerFunc { return func(resp http.ResponseWriter, req *http.Request) { - var metric Metrics + var metric mtr.Metrics log.Info().Msg("GetMetricsHandlerJSON called\n\n") if req.Header.Get("Content-Type") != "application/json" { @@ -256,7 +243,7 @@ func GetMetricsHandlerJSON(storage ms.Collector) http.HandlerFunc { func UpdateMetricsHandlerJSON(storage ms.Collector) http.HandlerFunc { return func(resp http.ResponseWriter, req *http.Request) { - var metric Metrics + var metric mtr.Metrics if req.Header.Get("Content-Type") != "application/json" { http.Error(resp, "Content-Type must be application/json", http.StatusUnsupportedMediaType) diff --git a/internal/memstorage/storage.go b/internal/memstorage/storage.go deleted file mode 100644 index 569e79e..0000000 --- a/internal/memstorage/storage.go +++ /dev/null @@ -1,104 +0,0 @@ -package memstorage - -import ( - "sync" - - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" -) - -type Collector interface { - GetMetric(mType, mName string) (interface{}, bool) - GetAllMetrics() (map[string]float64, map[string]int64) - UpdateMetric(mtr metrics.Metric) error -} - -type MemStorage struct { - mutex sync.RWMutex - Gauges map[string]float64 - Counters map[string]int64 -} - -func NewMemStorage() *MemStorage { - return &MemStorage{ - Gauges: make(map[string]float64), - Counters: make(map[string]int64), - } -} - -func (ms *MemStorage) UpdateMetric(metric metrics.Metric) error { - switch metric.Type() { - case metrics.CounterType: - { - val, ok := metric.Value().(int64) - if !ok { - return metrics.ErrInvalidValueType - } - - ms.UpdateCounter(metric, val) - return nil - } - case metrics.GaugeType: - { - val, ok := metric.Value().(float64) - if !ok { - return metrics.ErrInvalidValueType - } - - ms.UpdateGauge(metric, val) - return nil - } - default: - return metrics.ErrInvalidMetricsType - } -} - -func (ms *MemStorage) UpdateGauge(metric metrics.Metric, value float64) { - ms.mutex.Lock() - defer ms.mutex.Unlock() - ms.Gauges[metric.Name()] = value -} - -func (ms *MemStorage) UpdateCounter(metric metrics.Metric, value int64) { - ms.mutex.Lock() - defer ms.mutex.Unlock() - ms.Counters[metric.Name()] += value -} - -func (ms *MemStorage) GetMetric(mType, mName string) (interface{}, bool) { - ms.mutex.RLock() - defer ms.mutex.RUnlock() - - switch mType { - case metrics.GaugeType: - { - if val, ok := ms.Gauges[mName]; ok { - return val, true - } - } - case metrics.CounterType: - { - if val, ok := ms.Counters[mName]; ok { - return val, true - } - } - } - - return nil, false -} - -func (ms *MemStorage) GetAllMetrics() (map[string]float64, map[string]int64) { - ms.mutex.RLock() - defer ms.mutex.RUnlock() - - gaugesCopy := make(map[string]float64, len(ms.Gauges)) - for name, value := range ms.Gauges { - gaugesCopy[name] = value - } - - countersCopy := make(map[string]int64, len(ms.Counters)) - for name, value := range ms.Counters { - countersCopy[name] = value - } - - return gaugesCopy, countersCopy -} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 5f82b17..8dd6853 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -3,7 +3,6 @@ package metrics import ( "errors" ) - type Metric interface { Value() any Name() string @@ -11,6 +10,19 @@ type Metric interface { Update(mValue any) error } +//easyjson:json +type Metrics struct { + ID string `json:"id"` + MType string `json:"type"` + Delta *int64 `json:"delta,omitempty"` + Value *float64 `json:"value,omitempty"` +} +type MetricTable struct { + Name string + Type string + Value string +} + const ( CounterType = "counter" GaugeType = "gauge" diff --git a/internal/handlers/server/server_easyjson.go b/internal/metrics/metrics_easyjson.go similarity index 63% rename from internal/handlers/server/server_easyjson.go rename to internal/metrics/metrics_easyjson.go index 094fb13..188ce47 100644 --- a/internal/handlers/server/server_easyjson.go +++ b/internal/metrics/metrics_easyjson.go @@ -1,6 +1,6 @@ // Code generated by easyjson for marshaling/unmarshaling. DO NOT EDIT. -package server +package metrics import ( json "encoding/json" @@ -17,7 +17,7 @@ var ( _ easyjson.Marshaler ) -func easyjson22b57fa5DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalHandlersServer(in *jlexer.Lexer, out *Metrics) { +func easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(in *jlexer.Lexer, out *Metrics) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -70,7 +70,7 @@ func easyjson22b57fa5DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInte in.Consumed() } } -func easyjson22b57fa5EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalHandlersServer(out *jwriter.Writer, in Metrics) { +func easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(out *jwriter.Writer, in Metrics) { out.RawByte('{') first := true _ = first @@ -100,27 +100,27 @@ func easyjson22b57fa5EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInte // MarshalJSON supports json.Marshaler interface func (v Metrics) MarshalJSON() ([]byte, error) { w := jwriter.Writer{} - easyjson22b57fa5EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalHandlersServer(&w, v) + easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(&w, v) return w.Buffer.BuildBytes(), w.Error } // MarshalEasyJSON supports easyjson.Marshaler interface func (v Metrics) MarshalEasyJSON(w *jwriter.Writer) { - easyjson22b57fa5EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalHandlersServer(w, v) + easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(w, v) } // UnmarshalJSON supports json.Unmarshaler interface func (v *Metrics) UnmarshalJSON(data []byte) error { r := jlexer.Lexer{Data: data} - easyjson22b57fa5DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalHandlersServer(&r, v) + easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(&r, v) return r.Error() } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *Metrics) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjson22b57fa5DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalHandlersServer(l, v) + easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(l, v) } -func easyjson22b57fa5DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalHandlersServer1(in *jlexer.Lexer, out *MetricTable) { +func easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(in *jlexer.Lexer, out *MetricTable) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -155,7 +155,7 @@ func easyjson22b57fa5DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInte in.Consumed() } } -func easyjson22b57fa5EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalHandlersServer1(out *jwriter.Writer, in MetricTable) { +func easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(out *jwriter.Writer, in MetricTable) { out.RawByte('{') first := true _ = first @@ -177,4 +177,26 @@ func easyjson22b57fa5EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInte out.RawByte('}') } +// MarshalJSON supports json.Marshaler interface +func (v MetricTable) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(&w, v) + return w.Buffer.BuildBytes(), w.Error +} + +// MarshalEasyJSON supports easyjson.Marshaler interface +func (v MetricTable) MarshalEasyJSON(w *jwriter.Writer) { + easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(w, v) +} +// UnmarshalJSON supports json.Unmarshaler interface +func (v *MetricTable) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(&r, v) + return r.Error() +} + +// UnmarshalEasyJSON supports easyjson.Unmarshaler interface +func (v *MetricTable) UnmarshalEasyJSON(l *jlexer.Lexer) { + easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(l, v) +} diff --git a/pkg/data-base/DBfunctions.go b/pkg/data-base/file-storage.go similarity index 92% rename from pkg/data-base/DBfunctions.go rename to pkg/data-base/file-storage.go index f7e6098..faaaf46 100644 --- a/pkg/data-base/DBfunctions.go +++ b/pkg/data-base/file-storage.go @@ -6,8 +6,6 @@ import ( "net/http" "os" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" @@ -16,10 +14,10 @@ import ( func SaveToDB(collector ms.Collector, path string) error { allMetrics := collector.GetAllMetrics() - data := make([]server.Metrics, 0, len(allMetrics)) + data := make([]mtr.Metrics, 0, len(allMetrics)) for _, metric := range allMetrics { - var newMetric server.Metrics + var newMetric mtr.Metrics newMetric.MType = metric.Type() newMetric.ID = metric.Name() @@ -58,7 +56,7 @@ func LoadFromDB(collector ms.Collector, path string) error { return fmt.Errorf("can't read file %s with DB %w", path, err) } - var data []server.Metrics + var data []mtr.Metrics err = json.Unmarshal(bytes, &data) if err != nil { return fmt.Errorf("can't parse json format from DB %w", err) From c34339b0ee3ed469585363dded88d8a21ac084e0 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Wed, 16 Jul 2025 19:18:11 +0300 Subject: [PATCH 066/126] optimization compresstion --- internal/handlers/agent/agent.go | 41 ++++++++++++++++++++---------- internal/handlers/server/server.go | 22 ++++++++++++++-- 2 files changed, 47 insertions(+), 16 deletions(-) diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index d81698d..3b8f7fa 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -90,18 +90,22 @@ func sendMetric(client *resty.Client, metricJSON *mtr.Metrics) { 1 * time.Second, } - // buf, err := ConvertToGzipData(metricJSON) - // if err != nil { - // log.Error().Err(err).Msg("Failed to convert metric to gzip") - // return - // } + buf, ok, err := ConvertToGzipData(metricJSON) + if err != nil { + log.Error().Err(err).Msg("Failed to convert metric to gzip") + return + } for _, backoff := range backoffSchedule { - res, err := client.R(). + req := client.R(). SetHeader("Content-Type", "application/json"). - SetHeader("Content-Encoding", "gzip"). - SetBody(metricJSON). - Post("update/") + SetBody(buf) + + if ok { + req.SetHeader("Content-Encoding", "gzip") + } + + res, err := req.Post("update/") if err != nil || res.StatusCode() != http.StatusOK { } else { @@ -112,15 +116,24 @@ func sendMetric(client *resty.Client, metricJSON *mtr.Metrics) { } } -func ConvertToGzipData(metricJSON *mtr.Metrics) (*bytes.Buffer, error) { +func ConvertToGzipData(metricJSON *mtr.Metrics) (*bytes.Buffer, bool, error) { jsonData, err := easyjson.Marshal(*metricJSON) if err != nil { log.Error().Err(err).Msg("Failed to marshal metricJSON") - return nil, err + return nil, false, err } var buf bytes.Buffer - gz := gzip.NewWriter(&buf) + if len(jsonData) <= 1024 { + buf.Write(jsonData) + return &buf, false, nil + } + + gz, err := gzip.NewWriterLevel(&buf, gzip.BestSpeed) + if err != nil { + log.Error().Err(err).Msg("Failed to create gzip writer") + return nil, false, err + } defer func() { err := gz.Close() if err != nil { @@ -131,8 +144,8 @@ func ConvertToGzipData(metricJSON *mtr.Metrics) (*bytes.Buffer, error) { _, err = gz.Write(jsonData) if err != nil { log.Error().Err(err).Msg("Failed to write gzip data") - return nil, err + return nil, false, err } - return &buf, nil + return &buf, true, nil } diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index 7e774c0..35d7053 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -1,8 +1,10 @@ package server import ( + "compress/gzip" "fmt" "html/template" + "io" "net/http" "strconv" @@ -243,6 +245,23 @@ func GetMetricsHandlerJSON(storage ms.Collector) http.HandlerFunc { func UpdateMetricsHandlerJSON(storage ms.Collector) http.HandlerFunc { return func(resp http.ResponseWriter, req *http.Request) { + var reader io.Reader + if req.Header.Get("Content-Encoding") == "gzip" { + gz, err := gzip.NewReader(req.Body) + if err != nil { + http.Error(resp, "failed to create gzip reader", http.StatusBadRequest) + return + } + defer func() { + if err := gz.Close(); err != nil { + log.Error().Err(err).Msg("failed close gz reader") + } + }() + reader = gz + } else { + reader = req.Body + } + var metric mtr.Metrics if req.Header.Get("Content-Type") != "application/json" { @@ -250,7 +269,7 @@ func UpdateMetricsHandlerJSON(storage ms.Collector) http.HandlerFunc { return } - if err := easyjson.UnmarshalFromReader(req.Body, &metric); err != nil { + if err := easyjson.UnmarshalFromReader(reader, &metric); err != nil { http.Error(resp, fmt.Sprintf("invalid json body: %v", err), http.StatusBadRequest) return } @@ -278,4 +297,3 @@ func UpdateMetricsHandlerJSON(storage ms.Collector) http.HandlerFunc { } } } - From 10949571e61edcbd72f671efdf6e2454e3d11c9a Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Wed, 16 Jul 2025 17:57:02 +0300 Subject: [PATCH 067/126] after code review --- Makefile | 8 ++- cmd/server/cfg_server.go | 18 +++++- internal/config/server-config.go | 107 +++++++++++++++++-------------- internal/handlers/server/gzip.go | 6 +- internal/metrics/counter.go | 10 +-- internal/metrics/gauge.go | 10 +-- pkg/data-base/file-storage.go | 49 +++++++++++++- 7 files changed, 138 insertions(+), 70 deletions(-) diff --git a/Makefile b/Makefile index 5b467f0..6a6c93e 100644 --- a/Makefile +++ b/Makefile @@ -15,19 +15,21 @@ all: build build: server agent -server: +server: deps @go build -o $(SERVER_FULL_PATH) $(SERVER_SRC_DIR) @echo "Built $(SERVER_FULL_PATH)" -agent: +agent: deps @go build -o $(AGENT_FULL_PATH) $(AGENT_SRC_DIR) @echo "Built $(AGENT_FULL_PATH)" test: @go test ./... -v -lint: +deps: @go mod download + +lint: deps @golangci-lint run clean: diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index a4f2b37..f843dfb 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -48,8 +48,21 @@ func init() { } func PreRunE(cmd *cobra.Command, args []string) error { - var err error - opts, err = config.ParseOptionsFromCmd(cmd, endPointAddr, storeInterval, fileStoragePath, restoreOnStart, dataBaseDSN) + opts, err := config.ParseOptionsFromCmdAndEnvs(cmd, &config.Options{ + EndPointAddr: endPointAddr, + StoreInterval: storeInterval, + FileStoragePath: fileStoragePath, + RestoreOnStart: restoreOnStart, + DataBaseDSN: dataBaseDSN}) + + opts = config.NewServerOptions( + config.WithAddress(opts.EndPointAddr), + config.WithStoreInterval(opts.StoreInterval), + config.WithFileStoragePath(opts.FileStoragePath), + config.WithRestoreOnStart(opts.RestoreOnStart), + config.WithDataBaseDSN(opts.DataBaseDSN), + ) + return err } @@ -108,6 +121,7 @@ func startServer(ctx context.Context, opts *config.Options, db *sql.DB) error { r := router.NewRouter(storage, opts, db) if opts.RestoreOnStart { + log.Debug().Msg("restoreOnStart") if err := database.LoadFromDB(storage, opts.FileStoragePath); err != nil && !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("LoadFromDB error %w", err) } diff --git a/internal/config/server-config.go b/internal/config/server-config.go index f32280c..facbb83 100644 --- a/internal/config/server-config.go +++ b/internal/config/server-config.go @@ -12,7 +12,7 @@ import ( const ( DefaultEndpoint = "localhost:8080" DefaultStoreInterval = 300 - DefaultFileStoragePath = "/temp/metrics-db.json" + DefaultFileStoragePath = "/tmp/metrics-db.json" DefaultRestoreOnStart = true DefaultDataBaseDSN = "" ) @@ -33,111 +33,120 @@ type EnvConfig struct { DataBaseDSN string `env:"DATABASE_DSN"` } -func NewServerOptions(envOpts, flagOpts []func(*Options)) *Options { +type Option func(*Options) + +func NewServerOptions(options ...Option) *Options { opts := &Options{ EndPointAddr: DefaultEndpoint, StoreInterval: DefaultStoreInterval, FileStoragePath: DefaultFileStoragePath, RestoreOnStart: DefaultRestoreOnStart, + DataBaseDSN: DefaultDataBaseDSN, } - for _, opt := range flagOpts { - opt(opts) - } - - for _, opt := range envOpts { + for _, opt := range options { opt(opts) } return opts } -func WithAddress(addr string) func(*Options) { +func WithAddress(addr string) Option { return func(o *Options) { o.EndPointAddr = addr } } -func WithStoreInterval(interval int) func(*Options) { +func WithStoreInterval(interval int) Option { return func(o *Options) { o.StoreInterval = interval } } -func WithFileStoragePath(path string) func(*Options) { +func WithFileStoragePath(path string) Option { return func(o *Options) { o.FileStoragePath = path } } -func WithRestoreOnStart(restore bool) func(*Options) { +func WithRestoreOnStart(restore bool) Option { return func(o *Options) { o.RestoreOnStart = restore } } -func WithDataBaseDSN(dataBaseDSN string) func(*Options) { +func WithDataBaseDSN(dsn string) Option { return func(o *Options) { - o.DataBaseDSN = dataBaseDSN + o.DataBaseDSN = dsn } } -func ParseOptionsFromCmd(cmd *cobra.Command, endPointAddr string, storeInterval int, fileStoragePath string, - restoreOnStart bool, dataBaseDSN string) (*Options, error) { - var envCfg EnvConfig - if err := env.Parse(&envCfg); err != nil { - return nil, fmt.Errorf("failed to parse environment: %w", err) +func ParseOptionsFromCmdAndEnvs(cmd *cobra.Command, src *Options) (*Options, error) { + opts, err := ParseFlags(cmd, src) + if err != nil { + return nil, err } - var envOpts []func(*Options) - if envCfg.DataBaseDSN != "" { - cleanDSN := strings.Trim(envCfg.DataBaseDSN, "'") - envOpts = append(envOpts, WithDataBaseDSN(cleanDSN)) - } - if envCfg.EndPointAddr != "" { - envOpts = append(envOpts, WithAddress(envCfg.EndPointAddr)) + if err := ParseEnvs(cmd, opts); err != nil { + return nil, err } - if envCfg.StoreInterval < 0 { - return nil, fmt.Errorf("store interval must be >= 0, got %d", envCfg.StoreInterval) - } else { - envOpts = append(envOpts, WithStoreInterval(envCfg.StoreInterval)) - } - if envCfg.FileStoragePath != "" { - envOpts = append(envOpts, WithFileStoragePath(envCfg.FileStoragePath)) + + if _, _, err := net.SplitHostPort(opts.EndPointAddr); err != nil { + return nil, fmt.Errorf("invalid address %s: %w", opts.EndPointAddr, err) } - envOpts = append(envOpts, WithRestoreOnStart(envCfg.RestoreOnStart)) - var flagOpts []func(*Options) + return opts, nil +} + +func ParseFlags(cmd *cobra.Command, src *Options) (*Options, error) { + opts := *src + if cmd.Flags().Changed("d") { - flagOpts = append(flagOpts, WithDataBaseDSN(dataBaseDSN)) + opts.DataBaseDSN = src.DataBaseDSN } if cmd.Flags().Changed("a") { - flagOpts = append(flagOpts, WithAddress(endPointAddr)) + opts.EndPointAddr = src.EndPointAddr } - if cmd.Flags().Changed("i") { - if storeInterval < 0 { - return nil, fmt.Errorf("store interval flag must be >= 0, got %d", storeInterval) + if src.StoreInterval < 0 { + return nil, fmt.Errorf("store interval must be >= 0, got %d", src.StoreInterval) } - flagOpts = append(flagOpts, WithStoreInterval(storeInterval)) + opts.StoreInterval = src.StoreInterval } - if cmd.Flags().Changed("f") { - if fileStoragePath == "" { + if src.FileStoragePath == "" { return nil, fmt.Errorf("file storage path flag cannot be empty") } - flagOpts = append(flagOpts, WithFileStoragePath(fileStoragePath)) + opts.FileStoragePath = src.FileStoragePath } - if cmd.Flags().Changed("r") { - flagOpts = append(flagOpts, WithRestoreOnStart(restoreOnStart)) + opts.RestoreOnStart = src.RestoreOnStart } - opts := NewServerOptions(envOpts, flagOpts) + return &opts, nil +} - if _, _, err := net.SplitHostPort(opts.EndPointAddr); err != nil { - return nil, fmt.Errorf("invalid address %s: %w", opts.EndPointAddr, err) +func ParseEnvs(cmd *cobra.Command, opts *Options) error { + var envCfg EnvConfig + if err := env.Parse(&envCfg); err != nil { + return fmt.Errorf("failed to parse environment: %w", err) } - return opts, nil + if envCfg.DataBaseDSN != "" { + cleanDSN := strings.Trim(envCfg.DataBaseDSN, "'") + opts.DataBaseDSN = cleanDSN + } + + if envCfg.EndPointAddr != "" { + opts.EndPointAddr = envCfg.EndPointAddr + } + if envCfg.StoreInterval > 0 { + opts.StoreInterval = envCfg.StoreInterval + } + if envCfg.FileStoragePath != "" { + opts.FileStoragePath = envCfg.FileStoragePath + } + opts.RestoreOnStart = envCfg.RestoreOnStart + + return nil } diff --git a/internal/handlers/server/gzip.go b/internal/handlers/server/gzip.go index 061c536..3d20480 100644 --- a/internal/handlers/server/gzip.go +++ b/internal/handlers/server/gzip.go @@ -9,9 +9,9 @@ import ( log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) -var supportedContentType = map[string]bool{ - "application/json": true, - "text/html; charset=utf-8": true, +var supportedContentType = map[string]struct{}{ + "application/json": {}, + "text/html; charset=utf-8": {}, } type gzipWriter struct { diff --git a/internal/metrics/counter.go b/internal/metrics/counter.go index 4c791f6..8c1a546 100644 --- a/internal/metrics/counter.go +++ b/internal/metrics/counter.go @@ -26,11 +26,11 @@ func (c *counter) Value() any { func (c *counter) Update(mValue any) error { value, ok := mValue.(int64) - if !ok { - return ErrInvalidValueType - } - c.value += value - return nil + if !ok { + return ErrInvalidValueType + } + c.value += value + return nil } func (c *counter) SetValue(v any) error { diff --git a/internal/metrics/gauge.go b/internal/metrics/gauge.go index 04fce82..6349490 100644 --- a/internal/metrics/gauge.go +++ b/internal/metrics/gauge.go @@ -26,11 +26,11 @@ func (g *gauge) Value() any { func (g *gauge) Update(mValue any) error { value, ok := mValue.(float64) - if !ok { - return ErrInvalidValueType - } - g.value = value - return nil + if !ok { + return ErrInvalidValueType + } + g.value = value + return nil } func (g *gauge) SetValue(v any) error { diff --git a/pkg/data-base/file-storage.go b/pkg/data-base/file-storage.go index faaaf46..abba83c 100644 --- a/pkg/data-base/file-storage.go +++ b/pkg/data-base/file-storage.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "os" + "path/filepath" ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" @@ -42,12 +43,49 @@ func SaveToDB(collector ms.Collector, path string) error { data = append(data, newMetric) } - bytes, err := json.MarshalIndent(data, "", " ") + if len(data) == 0 { + log.Warn().Msg("No metrics to save, skipping file write") + return nil + } + + bytes, err := json.MarshalIndent(data, "", " ") if err != nil { - return fmt.Errorf("json Marshal Indent err: %w", err) + return fmt.Errorf("json marshal error: %w", err) + } + + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return fmt.Errorf("failed to create directory: %w", err) + } + + tmpPath := path + ".tmp" + file, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return fmt.Errorf("failed to create temp file: %w", err) + } + defer func() { + if err := file.Close(); err != nil { + log.Error().Err(err).Msg("failed to close file") + } + }() + + if _, err := file.Write(bytes); err != nil { + return fmt.Errorf("write failed: %w", err) + } + + if err := file.Sync(); err != nil { + return fmt.Errorf("sync failed: %w", err) + } + + if err := os.Rename(tmpPath, path); err != nil { + return fmt.Errorf("rename failed: %w", err) } - return os.WriteFile(path, bytes, 0666) + log.Info(). + Str("path", path). + Int("metrics_saved", len(data)). + Msg("Metrics successfully saved") + + return nil } func LoadFromDB(collector ms.Collector, path string) error { @@ -56,6 +94,11 @@ func LoadFromDB(collector ms.Collector, path string) error { return fmt.Errorf("can't read file %s with DB %w", path, err) } + if len(bytes) == 0 { + log.Warn().Msgf("DB file %s is empty, skipping restore", path) + return nil + } + var data []mtr.Metrics err = json.Unmarshal(bytes, &data) if err != nil { From f4e15b1b3909ffdaa46fcd96226e7ce56f14f494 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Wed, 16 Jul 2025 20:58:44 +0300 Subject: [PATCH 068/126] some corrects --- cmd/server/cfg_server.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index f843dfb..2abb36d 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -48,7 +48,8 @@ func init() { } func PreRunE(cmd *cobra.Command, args []string) error { - opts, err := config.ParseOptionsFromCmdAndEnvs(cmd, &config.Options{ + var err error + opts, err = config.ParseOptionsFromCmdAndEnvs(cmd, &config.Options{ EndPointAddr: endPointAddr, StoreInterval: storeInterval, FileStoragePath: fileStoragePath, From 40b9a26571e6012168ab6aa3871b0a2eb25800ed Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Sun, 13 Jul 2025 00:55:15 +0300 Subject: [PATCH 069/126] empty commit, from iter10 version From 8264e0d9e7df176c68ef497386ebd72c087c519d Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Sun, 13 Jul 2025 18:28:56 +0300 Subject: [PATCH 070/126] Iter11 refactor (#14) * rewrote the storage logic for files and the database for the collector * fix tests * write a template for postgresql, but haven't implemented the logic * filled out the template, corrected the logic * fix name ms for method MemStorage --- cmd/agent/cfg_agent.go | 10 +- cmd/server/cfg_server.go | 86 ++++--- internal/collector/collerctor.go | 16 ++ internal/handlers/agent/agent.go | 15 +- internal/handlers/server/database.go | 31 --- internal/handlers/server/server.go | 39 ++-- internal/handlers/server/server_test.go | 17 +- internal/router/roter.go | 12 +- internal/storage/file.go | 116 ++++++++++ .../storage.go => storage/memory.go} | 23 +- internal/storage/postgres.go | 219 ++++++++++++++++++ .../{mem-storage => storage}/storage_test.go | 14 +- pkg/converter/converter.go | 31 +++ pkg/data-base/file-storage.go | 24 +- 14 files changed, 502 insertions(+), 151 deletions(-) create mode 100644 internal/collector/collerctor.go delete mode 100644 internal/handlers/server/database.go create mode 100644 internal/storage/file.go rename internal/{mem-storage/storage.go => storage/memory.go} (77%) create mode 100644 internal/storage/postgres.go rename internal/{mem-storage => storage}/storage_test.go (94%) create mode 100644 pkg/converter/converter.go diff --git a/cmd/agent/cfg_agent.go b/cmd/agent/cfg_agent.go index 2721551..c696ebb 100644 --- a/cmd/agent/cfg_agent.go +++ b/cmd/agent/cfg_agent.go @@ -14,7 +14,7 @@ import ( "github.com/spf13/cobra" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/agent" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/storage" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -86,7 +86,7 @@ var rootCmd = &cobra.Command{ stop := make(chan os.Signal, 1) signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) - go func(){ + go func() { <-stop cancel() }() @@ -108,7 +108,7 @@ func init() { } func startAgent(ctx context.Context) { - storage := ms.NewMemStorage() + storage := storage.NewMemStorage() client := resty.New(). SetTimeout(5 * time.Second). @@ -128,9 +128,9 @@ func startAgent(ctx context.Context) { return case <-pollTimer.C: - agent.UpdateAllMetrics(storage) + agent.UpdateAllMetrics(ctx, storage) case <-reportTimer.C: - agent.SendAllMetrics(client, storage) + agent.SendAllMetrics(ctx, client, storage) } } } diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 2abb36d..4a95b57 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -2,8 +2,6 @@ package main import ( "context" - "database/sql" - "errors" "fmt" "net/http" "os" @@ -14,10 +12,10 @@ import ( _ "github.com/jackc/pgx/v5/stdlib" "github.com/spf13/cobra" + col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/router" - database "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/data-base" + storage "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/storage" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -79,18 +77,6 @@ func RunE(cmd *cobra.Command, args []string) error { } }() - log.Info().Msgf("DSN: %s", opts.DataBaseDSN) - db, err := sql.Open("pgx", opts.DataBaseDSN) - if err != nil { - log.Error().Err(err).Msg("sql.Open error") - panic(err) - } - defer func() { - if err := db.Close(); err != nil { - log.Error().Err(err).Msg("Failed to db.Close") - } - }() - ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -99,7 +85,7 @@ func RunE(cmd *cobra.Command, args []string) error { serverErrCh := make(chan error, 1) go func() { - serverErrCh <- startServer(ctx, opts, db) + serverErrCh <- startServer(ctx, opts) }() select { @@ -113,42 +99,22 @@ func RunE(cmd *cobra.Command, args []string) error { } } -func startServer(ctx context.Context, opts *config.Options, db *sql.DB) error { +func startServer(ctx context.Context, opts *config.Options) error { log.Info(). Str("address", opts.EndPointAddr). Msg("Server configuration") - storage := ms.NewMemStorage() - r := router.NewRouter(storage, opts, db) - - if opts.RestoreOnStart { - log.Debug().Msg("restoreOnStart") - if err := database.LoadFromDB(storage, opts.FileStoragePath); err != nil && !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("LoadFromDB error %w", err) - } + collector, err := ChoiceCollector(ctx, opts) + if err != nil { + return fmt.Errorf("failed to create collector: %w", err) } + defer func() { + if err := collector.Close(); err != nil { + log.Error().Err(err).Msg("Failed to close collector") + } + }() - if opts.StoreInterval > 0 { - go func() { - ticker := time.NewTicker(time.Duration(opts.StoreInterval) * time.Second) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - if err := database.SaveToDB(storage, opts.FileStoragePath); err != nil { - log.Error().Err(err).Msg("failed to save DB") - } - case <-ctx.Done(): - log.Info().Msg("Shutting down server, saving metrics") - if err := database.SaveToDB(storage, opts.FileStoragePath); err != nil { - log.Error().Err(err).Msg("Failed to save metrics during shutdown") - } - return - } - } - }() - } + r := router.NewRouter(collector, opts) srv := &http.Server{ Addr: opts.EndPointAddr, @@ -175,3 +141,29 @@ func startServer(ctx context.Context, opts *config.Options, db *sql.DB) error { return nil } + +func ChoiceCollector(ctx context.Context, opts *config.Options) (col.Collector, error) { + var ( + collector col.Collector + err error + ) + + switch { + case opts.DataBaseDSN != "": + collector, err = storage.NewDatabase(ctx, opts.DataBaseDSN) + case opts.FileStoragePath != "": + collector, err = storage.NewFileStorage(ctx, &storage.FileParams{ + FileStoragePath: opts.FileStoragePath, + RestoreOnStart: opts.RestoreOnStart, + StoreInterval: opts.StoreInterval}) + default: + collector = storage.NewMemStorage() + } + + if err != nil { + log.Error().Err(err).Msg("failed create storage") + return nil, err + } + + return collector, nil +} diff --git a/internal/collector/collerctor.go b/internal/collector/collerctor.go new file mode 100644 index 0000000..d256d8d --- /dev/null +++ b/internal/collector/collerctor.go @@ -0,0 +1,16 @@ +package collector + +import ( + "context" + + mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" +) + +type Collector interface { + GetMetric(ctx context.Context, mType, mName string) (any, bool) + GetAllMetrics(ctx context.Context) []mtr.Metric + UpdateMetric(ctx context.Context, mType, mName string, mValue any) error + + Ping(ctx context.Context) error + Close() error +} diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 3b8f7fa..019baa1 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -3,6 +3,7 @@ package agent import ( "bytes" "compress/gzip" + "context" "fmt" "math/rand" "net/http" @@ -12,20 +13,20 @@ import ( "github.com/go-resty/resty/v2" "github.com/mailru/easyjson" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/storage" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" rt "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/runtime-stats" ) -func UpdateAllMetrics(storage *ms.MemStorage) { +func UpdateAllMetrics(ctx context.Context, storage *storage.MemStorage) { var memStats runtime.MemStats runtime.ReadMemStats(&memStats) for _, stat := range rt.MemRuntimeStats { val := stat.Get(&memStats) - if err := storage.UpdateMetric(stat.Type, stat.Name, val); err != nil { + if err := storage.UpdateMetric(ctx, stat.Type, stat.Name, val); err != nil { log.Error(). Err(err). Str("metric", stat.Name). @@ -36,17 +37,17 @@ func UpdateAllMetrics(storage *ms.MemStorage) { log.Debug().Msgf("update metric %s", stat.Name) } - if err := storage.UpdateMetric(mtr.CounterType, "PollCount", int64(1)); err != nil { + if err := storage.UpdateMetric(ctx, mtr.CounterType, "PollCount", int64(1)); err != nil { log.Error().Msgf("Failed to update PollCount metric: %v", err) } - if err := storage.UpdateMetric(mtr.GaugeType, "RandomValue", rand.Float64()); err != nil { + if err := storage.UpdateMetric(ctx, mtr.GaugeType, "RandomValue", rand.Float64()); err != nil { log.Error().Msgf("Failed to update RandomValue metric: %v", err) } } -func SendAllMetrics(client *resty.Client, storage *ms.MemStorage) { - allMetrics := storage.GetAllMetrics() +func SendAllMetrics(ctx context.Context, client *resty.Client, storage *storage.MemStorage) { + allMetrics := storage.GetAllMetrics(ctx) for _, metric := range allMetrics { mType := metric.Type() diff --git a/internal/handlers/server/database.go b/internal/handlers/server/database.go deleted file mode 100644 index 26d6d16..0000000 --- a/internal/handlers/server/database.go +++ /dev/null @@ -1,31 +0,0 @@ -package server - -import ( - "context" - "database/sql" - "fmt" - "net/http" - "time" - - log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" -) - -func WithDataBase(db *sql.DB, next http.HandlerFunc) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - - if err := db.PingContext(ctx); err != nil { - log.Error().Err(err).Msg("database ping failed") - http.Error(w, fmt.Sprintf("database ping failed: %v", err), http.StatusInternalServerError) - return - } - log.Info().Msg("database ping successful") - - next(w, r) - } -} - -func PingDataBase(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) -} diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index 35d7053..b4c7be4 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -2,6 +2,7 @@ package server import ( "compress/gzip" + "context" "fmt" "html/template" "io" @@ -11,7 +12,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/mailru/easyjson" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" + col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -35,11 +36,11 @@ func ConvertByType(mType, mValue string) (any, error) { } } -func GetMetric(storage ms.Collector) http.HandlerFunc { +func GetMetric(storage col.Collector) http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { mType := chi.URLParam(req, "mType") mName := chi.URLParam(req, "mName") - value, found := storage.GetMetric(mType, mName) + value, found := storage.GetMetric(req.Context(), mType, mName) if !found { http.Error(res, fmt.Sprintf("Metric %s was not found", mName), http.StatusNotFound) return @@ -67,9 +68,9 @@ func GetMetric(storage ms.Collector) http.HandlerFunc { } } -func GetAllMetrics(storage ms.Collector) http.HandlerFunc { +func GetAllMetrics(storage col.Collector) http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { - allMetrics := storage.GetAllMetrics() + allMetrics := storage.GetAllMetrics(req.Context()) var metricsToTable []mtr.MetricTable @@ -150,7 +151,7 @@ func GetAllMetrics(storage ms.Collector) http.HandlerFunc { } } -func UpdateMetric(storage ms.Collector) http.HandlerFunc { +func UpdateMetric(storage col.Collector) http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { mType := chi.URLParam(req, "mType") mName := chi.URLParam(req, "mName") @@ -177,7 +178,7 @@ func UpdateMetric(storage ms.Collector) http.HandlerFunc { return } - if err := storage.UpdateMetric(mType, mName, val); err != nil { + if err := storage.UpdateMetric(req.Context(), mType, mName, val); err != nil { http.Error(res, err.Error(), http.StatusBadRequest) return } @@ -190,8 +191,8 @@ func UpdateMetric(storage ms.Collector) http.HandlerFunc { } } -func FillMetricValueFromStorage(storage ms.Collector, metric *mtr.Metrics) bool { - value, ok := storage.GetMetric(metric.MType, metric.ID) +func FillMetricValueFromStorage(ctx context.Context, storage col.Collector, metric *mtr.Metrics) bool { + value, ok := storage.GetMetric(ctx, metric.MType, metric.ID) if !ok { return false } @@ -213,7 +214,7 @@ func FillMetricValueFromStorage(storage ms.Collector, metric *mtr.Metrics) bool return true } -func GetMetricsHandlerJSON(storage ms.Collector) http.HandlerFunc { +func GetMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { return func(resp http.ResponseWriter, req *http.Request) { var metric mtr.Metrics @@ -229,7 +230,7 @@ func GetMetricsHandlerJSON(storage ms.Collector) http.HandlerFunc { } log.Info().Msgf("Received metric request: %+v", metric) - if !FillMetricValueFromStorage(storage, &metric) { + if !FillMetricValueFromStorage(req.Context(), storage, &metric) { http.Error(resp, fmt.Sprintf("metric %s not found", metric.ID), http.StatusNotFound) return } @@ -243,7 +244,7 @@ func GetMetricsHandlerJSON(storage ms.Collector) http.HandlerFunc { } } -func UpdateMetricsHandlerJSON(storage ms.Collector) http.HandlerFunc { +func UpdateMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { return func(resp http.ResponseWriter, req *http.Request) { var reader io.Reader if req.Header.Get("Content-Encoding") == "gzip" { @@ -281,11 +282,11 @@ func UpdateMetricsHandlerJSON(storage ms.Collector) http.HandlerFunc { value = *metric.Delta } - if err := storage.UpdateMetric(metric.MType, metric.ID, value); err != nil { + if err := storage.UpdateMetric(req.Context(), metric.MType, metric.ID, value); err != nil { http.Error(resp, fmt.Sprintf("invalid update metric %s: %v", metric.ID, err), http.StatusBadRequest) } - if !FillMetricValueFromStorage(storage, &metric) { + if !FillMetricValueFromStorage(req.Context(), storage, &metric) { http.Error(resp, fmt.Sprintf("metric %s not found", metric.ID), http.StatusNotFound) return } @@ -297,3 +298,13 @@ func UpdateMetricsHandlerJSON(storage ms.Collector) http.HandlerFunc { } } } + +func PingHandler(col col.Collector) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if err := col.Ping(r.Context()); err != nil { + http.Error(w, "can't ping DB", http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + } +} diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index 410da65..99a4189 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -1,6 +1,7 @@ package server_test import ( + "context" "database/sql" "io" "net/http" @@ -10,9 +11,9 @@ import ( _ "github.com/jackc/pgx/v5/stdlib" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/router" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/storage" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -39,8 +40,8 @@ func TestUpdateMetric(t *testing.T) { } }() - storage := ms.NewMemStorage() - router := router.NewRouter(storage, opts, db) + storage := storage.NewMemStorage() + router := router.NewRouter(storage, opts) tests := []struct { name string @@ -129,17 +130,19 @@ func TestGetMetric(t *testing.T) { } }() - storage := ms.NewMemStorage() + //FIXME - maybe need mocks + ctx := context.Background() + storage := storage.NewMemStorage() - if err := storage.UpdateMetric(mtr.GaugeType, "cpu_usage", 75.5); err != nil { + if err := storage.UpdateMetric(ctx, mtr.GaugeType, "cpu_usage", 75.5); err != nil { log.Error().Msgf("Failed to update metric cpu_usage: %v", err) } - if err := storage.UpdateMetric(mtr.CounterType, "requests_total", int64(100)); err != nil { + if err := storage.UpdateMetric(ctx, mtr.CounterType, "requests_total", int64(100)); err != nil { log.Error().Msgf("Failed to update metric requests_total: %v", err) } - router := router.NewRouter(storage, opts, db) + router := router.NewRouter(storage, opts) tests := []struct { name string diff --git a/internal/router/roter.go b/internal/router/roter.go index a0fc307..859a97c 100644 --- a/internal/router/roter.go +++ b/internal/router/roter.go @@ -1,18 +1,17 @@ package router import ( - "database/sql" "net/http" "github.com/go-chi/chi/v5" + col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" - database "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/data-base" + //database "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/data-base" ) -func NewRouter(storage ms.Collector, opts *config.Options, db *sql.DB) http.Handler { +func NewRouter(storage col.Collector, opts *config.Options) http.Handler { r := chi.NewRouter() r.Use(server.WithLogging) @@ -21,9 +20,6 @@ func NewRouter(storage ms.Collector, opts *config.Options, db *sql.DB) http.Hand r.Route("/", func(r chi.Router) { r.Get("/", server.GetAllMetrics(storage)) r.Route("/update", func(r chi.Router) { - if opts.StoreInterval == 0 { - r.Use(database.WithSaveToDB(storage, opts.FileStoragePath)) - } r.Post("/", server.UpdateMetricsHandlerJSON(storage)) r.Post("/{mType}/{mName}/{mValue}", server.UpdateMetric(storage)) @@ -35,7 +31,7 @@ func NewRouter(storage ms.Collector, opts *config.Options, db *sql.DB) http.Hand }) r.Route("/ping", func(r chi.Router) { - r.Get("/", server.WithDataBase(db, server.PingDataBase)) + r.Get("/", server.PingHandler(storage)) }) }) diff --git a/internal/storage/file.go b/internal/storage/file.go new file mode 100644 index 0000000..e804d19 --- /dev/null +++ b/internal/storage/file.go @@ -0,0 +1,116 @@ +package storage + +import ( + "context" + "errors" + "fmt" + "os" + "sync" + "time" + + col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" + mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + database "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/data-base" + "github.com/rs/zerolog/log" +) + +type FileStorage struct { + mutex sync.RWMutex + filePath string + storage col.Collector + SyncRecord bool +} + +type FileParams struct { + FileStoragePath string + RestoreOnStart bool + StoreInterval int +} + +func NewFileStorage(ctx context.Context, fp *FileParams) (col.Collector, error) { + fs := &FileStorage{ + filePath: fp.FileStoragePath, + storage: NewMemStorage(), + SyncRecord: true, + } + if fp.RestoreOnStart { + if err := database.LoadFromDB(ctx, fs.storage, fp.FileStoragePath); err != nil && !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("LoadFromDB error %w", err) + } + } + + if fp.StoreInterval == 0 { + fs.SyncRecord = false + } + + if fp.StoreInterval > 0 { + go func() { + ticker := time.NewTicker(time.Duration(fp.StoreInterval) * time.Second) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + if err := database.SaveToDB(ctx, fs.storage, fp.FileStoragePath); err != nil { + log.Error().Err(err).Msg("failed to save DB") + } + case <-ctx.Done(): + log.Info().Msg("Shutting down server, saving metrics") + if err := database.SaveToDB(ctx, fs.storage, fp.FileStoragePath); err != nil { + log.Error().Err(err).Msg("Failed to save metrics during shutdown") + } + return + } + } + }() + } + + return fs, nil +} + +func (fs *FileStorage) UpdateMetric(ctx context.Context, mType, mName string, mValue any) error { + fs.mutex.Lock() + defer fs.mutex.Unlock() + + if err := fs.storage.UpdateMetric(ctx, mType, mName, mValue); err != nil { + log.Error().Err(err).Msg("failed update metric from file storage") + return fmt.Errorf("failed update metric from file storage %w", err) + } + + if fs.SyncRecord { + if err := database.SaveToDB(ctx, fs.storage, fs.filePath); err != nil { + log.Error().Err(err).Msg("failed save storage") + return fmt.Errorf("failed save storage %w", err) + } + } + + return nil +} + +func (fs *FileStorage) GetMetric(ctx context.Context, mType, mName string) (any, bool) { + fs.mutex.RLock() + defer fs.mutex.RUnlock() + + val, ok := fs.storage.GetMetric(ctx, mType, mName) + if !ok { + log.Error().Msg("can't get valid metric") + return nil, false + } + + return val, true +} + +func (fs *FileStorage) GetAllMetrics(ctx context.Context) []mtr.Metric { + fs.mutex.RLock() + defer fs.mutex.RUnlock() + + return fs.storage.GetAllMetrics(ctx) +} + +func (fs *FileStorage) Ping(ctx context.Context) error { + return nil +} + +func (fs *FileStorage) Close() error { + return nil +} diff --git a/internal/mem-storage/storage.go b/internal/storage/memory.go similarity index 77% rename from internal/mem-storage/storage.go rename to internal/storage/memory.go index de0aa74..b630127 100644 --- a/internal/mem-storage/storage.go +++ b/internal/storage/memory.go @@ -1,17 +1,12 @@ -package memstorage +package storage import ( + "context" "sync" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" ) -type Collector interface { - GetMetric(mType, mName string) (any, bool) - GetAllMetrics() []mtr.Metric - UpdateMetric(mType, mName string, mValue any) error -} - type MemStorage struct { mutex sync.RWMutex storage map[string]map[string]mtr.Metric @@ -23,7 +18,7 @@ func NewMemStorage() *MemStorage { } } -func (ms *MemStorage) UpdateMetric(mType, mName string, mValue any) error { +func (ms *MemStorage) UpdateMetric(_ context.Context, mType, mName string, mValue any) error { ms.mutex.Lock() defer ms.mutex.Unlock() @@ -59,7 +54,7 @@ func (ms *MemStorage) UpdateMetric(mType, mName string, mValue any) error { return nil } -func (ms *MemStorage) GetMetric(mType, mName string) (any, bool) { +func (ms *MemStorage) GetMetric(_ context.Context, mType, mName string) (any, bool) { ms.mutex.RLock() defer ms.mutex.RUnlock() @@ -70,7 +65,7 @@ func (ms *MemStorage) GetMetric(mType, mName string) (any, bool) { return nil, false } -func (ms *MemStorage) GetAllMetrics() []mtr.Metric { +func (ms *MemStorage) GetAllMetrics(_ context.Context) []mtr.Metric { ms.mutex.RLock() defer ms.mutex.RUnlock() @@ -88,3 +83,11 @@ func (ms *MemStorage) GetAllMetrics() []mtr.Metric { return result } + +func (ms *MemStorage) Ping(ctx context.Context) error { + return nil +} + +func (ms *MemStorage) Close() error { + return nil +} diff --git a/internal/storage/postgres.go b/internal/storage/postgres.go new file mode 100644 index 0000000..fc1f6f4 --- /dev/null +++ b/internal/storage/postgres.go @@ -0,0 +1,219 @@ +package storage + +import ( + "context" + "database/sql" + "fmt" + "sync" + + col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" + mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/converter" + "github.com/rs/zerolog/log" +) + +type Database struct { + mutex sync.RWMutex + DB *sql.DB +} + +func NewDatabase(ctx context.Context, dataBaseDSN string) (col.Collector, error) { + log.Info().Msgf("DSN: %s", dataBaseDSN) + db, err := sql.Open("pgx", dataBaseDSN) + if err != nil { + log.Error().Err(err).Msg("sql.Open error") + return nil, fmt.Errorf("failed to open database: %w", err) + } + + _, err = db.ExecContext(ctx, + "CREATE TABLE IF NOT EXISTS collector ("+ + "\"ID\" VARCHAR(250) PRIMARY KEY,"+ + "\"MType\" TEXT,"+ + "\"Delta\" INTEGER,"+ + "\"Value\" DOUBLE PRECISION"+ + ");") + + if err != nil { + if err := db.Close(); err != nil { + log.Error().Err(err).Msg("failed to close database") + } + + log.Error().Err(err).Msg("failed create table for database") + return nil, fmt.Errorf("failed create table for database %w", err) + } + + return &Database{ + mutex: sync.RWMutex{}, + DB: db, + }, nil +} + +func (db *Database) GetMetric(ctx context.Context, mType, mName string) (any, bool) { + db.mutex.RLock() + defer db.mutex.RUnlock() + + row := db.DB.QueryRowContext(ctx, + "SELECT ID, MType, Delta, Value FROM collector"+ + "WHERE ID = $1 AND MType = $2 LIMIT 1", mType, mName) + + var ( + id string + Type string + delta sql.NullInt64 + value sql.NullFloat64 + ) + + err := row.Scan(&id, &Type, &delta, &value) + if err != nil { + log.Error().Err(err).Msg("failed scan row") + return nil, false + } + + switch { + case value.Valid: + if Type != mtr.GaugeType { + return nil, false + } + + return value.Float64, true + + case delta.Valid: + if Type != mtr.CounterType { + return nil, false + } + + return delta.Int64, true + default: + log.Error().Msg("not valid value") + return nil, false + } +} + +func (db *Database) GetAllMetrics(ctx context.Context) []mtr.Metric { + db.mutex.RLock() + defer db.mutex.RUnlock() + + metrics := make([]mtr.Metrics, 0) + + rows, err := db.DB.QueryContext(ctx, + "SELECT ID, MType, Delta, Value FROM collector") + + if err != nil { + log.Error().Err(err).Msg("The request was not processed") + return nil + } + + var ( + delta sql.NullInt64 + value sql.NullFloat64 + ) + + for rows.Next() { + var metric mtr.Metrics + + err = rows.Scan(&metric.ID, &metric.MType, &delta, &value) + if err != nil { + log.Error().Err(err).Msgf("failed scan row: ID = %s, MType = %s", + metric.ID, metric.MType) + return nil + } + + switch { + case value.Valid: + if metric.MType != mtr.GaugeType { + return nil + } + + metric.Value = &value.Float64 + + case delta.Valid: + if metric.MType != mtr.CounterType { + return nil + } + + metric.Delta = &delta.Int64 + default: + log.Error().Msg("not valid value") + return nil + } + + metrics = append(metrics, metric) + } + + if rows.Err() != nil { + log.Error().Err(rows.Err()).Msg("have rows error") + return nil + } + + convertedMetrics, err := converter.ConvertMetrics(metrics) + if err != nil { + log.Error().Err(err).Msg("failed to convert metrics") + return nil + } + + return convertedMetrics +} + +func (db *Database) UpdateMetric(ctx context.Context, mType, mName string, mValue any) error { + db.mutex.Lock() + defer db.mutex.Unlock() + + m := mtr.Metrics{ + ID: mName, + MType: mType, + } + + //TODO - make method maybe for Metrics type + switch m.MType { + case mtr.GaugeType: + val, ok := mValue.(float64) + if !ok { + return fmt.Errorf("expected float64 for gauge") + } + m.Value = &val + + case mtr.CounterType: + val, ok := mValue.(int64) + if !ok { + return fmt.Errorf("expected int64 for counter") + } + m.Delta = &val + + default: + return fmt.Errorf("unknown metric type: %s", m.MType) + } + + _, err := db.DB.ExecContext(ctx, + `INSERT INTO collector ("ID", "MType", "Delta", "Value") + VALUES ($1, $2, $3, $4) + ON CONFLICT ("ID") DO UPDATE + SET "Delta" = EXCLUDED."Delta", + "Value" = EXCLUDED."Value", + "MType" = EXCLUDED."MType";`, + m.ID, m.MType, m.Delta, m.Value) + + if err != nil { + log.Error().Err(err).Msg("failed update insert into collector") + return fmt.Errorf("failed update insert into collector: %w", err) + } + + return nil +} + +func (db *Database) Ping(ctx context.Context) error { + return db.DB.PingContext(ctx) +} + +func (db *Database) Close() error { + db.mutex.Lock() + defer db.mutex.Unlock() + + if db.DB != nil { + if err := db.DB.Close(); err != nil { + log.Error().Err(err).Msg("failed to close database") + return err + } + } + + return nil +} diff --git a/internal/mem-storage/storage_test.go b/internal/storage/storage_test.go similarity index 94% rename from internal/mem-storage/storage_test.go rename to internal/storage/storage_test.go index b34ad01..aa69a5d 100644 --- a/internal/mem-storage/storage_test.go +++ b/internal/storage/storage_test.go @@ -1,6 +1,7 @@ -package memstorage +package storage import ( + "context" "sync" "testing" @@ -8,6 +9,8 @@ import ( ) func TestMemStorage_UpdateMetric(t *testing.T) { + ctx := context.Background() + type fields struct { storage map[string]map[string]mtr.Metric } @@ -98,12 +101,12 @@ func TestMemStorage_UpdateMetric(t *testing.T) { mutex: sync.RWMutex{}, storage: tt.fields.storage, } - if err := ms.UpdateMetric(tt.args.mType, tt.args.mName, tt.args.mValue); (err != nil) != tt.wantErr { + if err := ms.UpdateMetric(ctx, tt.args.mType, tt.args.mName, tt.args.mValue); (err != nil) != tt.wantErr { t.Errorf("MemStorage.UpdateMetric() error = %v, wantErr %v", err, tt.wantErr) } if !tt.wantErr { - val, ok := ms.GetMetric(tt.args.mType, tt.args.mName) + val, ok := ms.GetMetric(ctx, tt.args.mType, tt.args.mName) if !ok { t.Errorf("metric not found") return @@ -117,6 +120,8 @@ func TestMemStorage_UpdateMetric(t *testing.T) { } func TestMemStorage_GetMetric(t *testing.T) { + ctx := context.Background() + counter := mtr.NewCounter("requests", 42) gauge := mtr.NewGauge("temperature", 36.6) @@ -174,7 +179,7 @@ func TestMemStorage_GetMetric(t *testing.T) { mutex: sync.RWMutex{}, storage: tt.fields, } - gotVal, gotOk := ms.GetMetric(tt.mType, tt.mName) + gotVal, gotOk := ms.GetMetric(ctx, tt.mType, tt.mName) if gotVal != tt.wantVal { t.Errorf("GetMetric() gotVal = %v, want %v", gotVal, tt.wantVal) } @@ -258,4 +263,3 @@ func TestMemStorage_GetMetric(t *testing.T) { // }) // } // } - diff --git a/pkg/converter/converter.go b/pkg/converter/converter.go new file mode 100644 index 0000000..49ced6e --- /dev/null +++ b/pkg/converter/converter.go @@ -0,0 +1,31 @@ +package converter + +import ( + "fmt" + + mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" +) + +func ConvertMetrics(src []mtr.Metrics) ([]mtr.Metric, error) { + converted := make([]mtr.Metric, 0, len(src)) + + for _, m := range src { + switch m.MType { + case mtr.GaugeType: + if m.Value == nil { + return nil, fmt.Errorf("nil gauge value for ID: %s", m.ID) + } + converted = append(converted, mtr.NewGauge(m.ID, *m.Value)) + + case mtr.CounterType: + if m.Delta == nil { + return nil, fmt.Errorf("nil counter value for ID: %s", m.ID) + } + converted = append(converted, mtr.NewCounter(m.ID, *m.Delta)) + default: + return nil, fmt.Errorf("unsupported metric type: %s", m.MType) + } + } + + return converted, nil +} diff --git a/pkg/data-base/file-storage.go b/pkg/data-base/file-storage.go index abba83c..c4b0a6a 100644 --- a/pkg/data-base/file-storage.go +++ b/pkg/data-base/file-storage.go @@ -1,19 +1,19 @@ package database import ( + "context" "encoding/json" "fmt" - "net/http" "os" "path/filepath" - ms "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/mem-storage" + col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) -func SaveToDB(collector ms.Collector, path string) error { - allMetrics := collector.GetAllMetrics() +func SaveToDB(ctx context.Context, collector col.Collector, path string) error { + allMetrics := collector.GetAllMetrics(ctx) data := make([]mtr.Metrics, 0, len(allMetrics)) @@ -88,7 +88,7 @@ func SaveToDB(collector ms.Collector, path string) error { return nil } -func LoadFromDB(collector ms.Collector, path string) error { +func LoadFromDB(ctx context.Context, collector col.Collector, path string) error { bytes, err := os.ReadFile(path) if err != nil { return fmt.Errorf("can't read file %s with DB %w", path, err) @@ -108,12 +108,12 @@ func LoadFromDB(collector ms.Collector, path string) error { for _, metric := range data { switch metric.MType { case mtr.GaugeType: - if err := collector.UpdateMetric(metric.MType, metric.ID, *metric.Value); err != nil { + if err := collector.UpdateMetric(ctx, metric.MType, metric.ID, *metric.Value); err != nil { log.Error().Err(err).Msg("update metric error") return fmt.Errorf("update metric error %w", err) } case mtr.CounterType: - if err := collector.UpdateMetric(metric.MType, metric.ID, *metric.Delta); err != nil { + if err := collector.UpdateMetric(ctx, metric.MType, metric.ID, *metric.Delta); err != nil { log.Error().Err(err).Msg("update metric error") return fmt.Errorf("update metric error %w", err) } @@ -123,14 +123,4 @@ func LoadFromDB(collector ms.Collector, path string) error { return nil } -func WithSaveToDB(collector ms.Collector, filePath string) func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - next.ServeHTTP(w, req) - if err := SaveToDB(collector, filePath); err != nil { - log.Error().Err(err).Msg("Failed to save metrics synchronously") - } - }) - } -} From aec27e1a629fe4b77d305d6a7c549438b3c36b99 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Mon, 14 Jul 2025 00:25:57 +0300 Subject: [PATCH 071/126] fix sync record for file base --- internal/router/roter.go | 1 - internal/storage/file.go | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/router/roter.go b/internal/router/roter.go index 859a97c..f9570c4 100644 --- a/internal/router/roter.go +++ b/internal/router/roter.go @@ -8,7 +8,6 @@ import ( col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" - //database "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/data-base" ) func NewRouter(storage col.Collector, opts *config.Options) http.Handler { diff --git a/internal/storage/file.go b/internal/storage/file.go index e804d19..66a6ac7 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -31,7 +31,7 @@ func NewFileStorage(ctx context.Context, fp *FileParams) (col.Collector, error) fs := &FileStorage{ filePath: fp.FileStoragePath, storage: NewMemStorage(), - SyncRecord: true, + SyncRecord: false, } if fp.RestoreOnStart { if err := database.LoadFromDB(ctx, fs.storage, fp.FileStoragePath); err != nil && !errors.Is(err, os.ErrNotExist) { @@ -40,7 +40,7 @@ func NewFileStorage(ctx context.Context, fp *FileParams) (col.Collector, error) } if fp.StoreInterval == 0 { - fs.SyncRecord = false + fs.SyncRecord = true } if fp.StoreInterval > 0 { From a3c0746ab3705e011c76d3b4351a7d5ea3690c4d Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Mon, 14 Jul 2025 00:42:31 +0300 Subject: [PATCH 072/126] fix ChoiceCollector --- cmd/server/cfg_server.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 4a95b57..3396257 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -156,6 +156,8 @@ func ChoiceCollector(ctx context.Context, opts *config.Options) (col.Collector, FileStoragePath: opts.FileStoragePath, RestoreOnStart: opts.RestoreOnStart, StoreInterval: opts.StoreInterval}) + + log.Debug().Msg("chose file storage") default: collector = storage.NewMemStorage() } From efbe35f8030bd2d3af0dc81b926fbbd80ddf1c27 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Mon, 14 Jul 2025 01:13:47 +0300 Subject: [PATCH 073/126] delete if len() from LoadFromDB() --- internal/storage/file.go | 13 ++++++++++++- pkg/data-base/file-storage.go | 1 - 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/internal/storage/file.go b/internal/storage/file.go index 66a6ac7..f602120 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -72,6 +72,12 @@ func (fs *FileStorage) UpdateMetric(ctx context.Context, mType, mName string, mV fs.mutex.Lock() defer fs.mutex.Unlock() + log.Debug(). + Str("type", mType). + Str("name", mName). + Interface("value", mValue). + Msg("Updating metric") + if err := fs.storage.UpdateMetric(ctx, mType, mName, mValue); err != nil { log.Error().Err(err).Msg("failed update metric from file storage") return fmt.Errorf("failed update metric from file storage %w", err) @@ -84,6 +90,11 @@ func (fs *FileStorage) UpdateMetric(ctx context.Context, mType, mName string, mV } } + log.Info(). + Str("type", mType). + Str("name", mName). + Msg("Metric updated successfully") + return nil } @@ -108,7 +119,7 @@ func (fs *FileStorage) GetAllMetrics(ctx context.Context) []mtr.Metric { } func (fs *FileStorage) Ping(ctx context.Context) error { - return nil + return nil } func (fs *FileStorage) Close() error { diff --git a/pkg/data-base/file-storage.go b/pkg/data-base/file-storage.go index c4b0a6a..2fc8858 100644 --- a/pkg/data-base/file-storage.go +++ b/pkg/data-base/file-storage.go @@ -123,4 +123,3 @@ func LoadFromDB(ctx context.Context, collector col.Collector, path string) error return nil } - From ce90e6f9498f7cdb0631b30ae46f1fe1bc0be383 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Mon, 14 Jul 2025 01:23:01 +0300 Subject: [PATCH 074/126] . From 071421d3913ca190e571c90f8f3fb8527ac4f819 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Mon, 14 Jul 2025 15:31:36 +0300 Subject: [PATCH 075/126] - From b7847de415dd720e6e822c468e164fd10659f723 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Mon, 14 Jul 2025 15:47:54 +0300 Subject: [PATCH 076/126] add len(bytes) for Load --- pkg/data-base/file-storage.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/data-base/file-storage.go b/pkg/data-base/file-storage.go index 2fc8858..0a63341 100644 --- a/pkg/data-base/file-storage.go +++ b/pkg/data-base/file-storage.go @@ -98,7 +98,6 @@ func LoadFromDB(ctx context.Context, collector col.Collector, path string) error log.Warn().Msgf("DB file %s is empty, skipping restore", path) return nil } - var data []mtr.Metrics err = json.Unmarshal(bytes, &data) if err != nil { @@ -122,4 +121,3 @@ func LoadFromDB(ctx context.Context, collector col.Collector, path string) error return nil } - From 7f95a65417f6eb4e5d5463bd9e9f26b2283f6cdf Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Mon, 14 Jul 2025 22:30:33 +0300 Subject: [PATCH 077/126] didn't fix --- .gitignore | 2 +- internal/storage/file.go | 1 + pkg/data-base/file-storage.go | 5 +++++ pkg/logger/logger.go | 2 +- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 2055d36..e671eb3 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,6 @@ logFileAgent.log cmd/server/server cmd/agent/agent - +tmp log.txt diff --git a/internal/storage/file.go b/internal/storage/file.go index f602120..688bfe7 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -56,6 +56,7 @@ func NewFileStorage(ctx context.Context, fp *FileParams) (col.Collector, error) } case <-ctx.Done(): log.Info().Msg("Shutting down server, saving metrics") + if err := database.SaveToDB(ctx, fs.storage, fp.FileStoragePath); err != nil { log.Error().Err(err).Msg("Failed to save metrics during shutdown") } diff --git a/pkg/data-base/file-storage.go b/pkg/data-base/file-storage.go index 0a63341..fd8316d 100644 --- a/pkg/data-base/file-storage.go +++ b/pkg/data-base/file-storage.go @@ -89,11 +89,16 @@ func SaveToDB(ctx context.Context, collector col.Collector, path string) error { } func LoadFromDB(ctx context.Context, collector col.Collector, path string) error { + log.Info(). + Str("path", path). + Msg("Trying to load metrics from file") + bytes, err := os.ReadFile(path) if err != nil { return fmt.Errorf("can't read file %s with DB %w", path, err) } + log.Debug().Str("data", string(bytes)).Msg("metrics") if len(bytes) == 0 { log.Warn().Msgf("DB file %s is empty, skipping restore", path) return nil diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 98fca6c..3dfdcca 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -12,7 +12,7 @@ import ( var logger zerolog.Logger func InitLogger(logFilePath string) (*os.File, error) { - file, err := os.OpenFile(logFilePath, os.O_CREATE|os.O_WRONLY, 0666) + file, err := os.OpenFile(logFilePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) if err != nil { return nil, fmt.Errorf("failed to open log file %s: %w", logFilePath, err) } From 4bb4ad33b6ad0da4a3e46c3c1a5259f6ba6344ae Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Mon, 14 Jul 2025 23:41:55 +0300 Subject: [PATCH 078/126] add create file for SaveToDB() --- pkg/data-base/file-storage.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/data-base/file-storage.go b/pkg/data-base/file-storage.go index fd8316d..8a182c2 100644 --- a/pkg/data-base/file-storage.go +++ b/pkg/data-base/file-storage.go @@ -86,6 +86,13 @@ func SaveToDB(ctx context.Context, collector col.Collector, path string) error { Msg("Metrics successfully saved") return nil + // err = os.WriteFile(path, bytes, 0644) + // + // if err != nil { + // return fmt.Errorf("invalid write file %s: %w", path, err) + // } + // + // return nil } func LoadFromDB(ctx context.Context, collector col.Collector, path string) error { From 8fcb4fe772372bbe64da537eead7fed5a40bac98 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Tue, 15 Jul 2025 01:01:17 +0300 Subject: [PATCH 079/126] add logs for CI --- internal/handlers/server/server.go | 3 ++- internal/storage/file.go | 11 ++++------- pkg/data-base/file-storage.go | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index b4c7be4..f11f71d 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -229,7 +229,7 @@ func GetMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { return } log.Info().Msgf("Received metric request: %+v", metric) - + fmt.Printf("GET METRIC: Получение метрики типа '%s' с именем '%s'\n", metric.MType, metric.ID) if !FillMetricValueFromStorage(req.Context(), storage, &metric) { http.Error(resp, fmt.Sprintf("metric %s not found", metric.ID), http.StatusNotFound) return @@ -282,6 +282,7 @@ func UpdateMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { value = *metric.Delta } + fmt.Printf("UPDATE METRIC: Обновление метрики типа '%s', имя '%s', значение '%v'\n", metric.MType, metric.ID, value) if err := storage.UpdateMetric(req.Context(), metric.MType, metric.ID, value); err != nil { http.Error(resp, fmt.Sprintf("invalid update metric %s: %v", metric.ID, err), http.StatusBadRequest) } diff --git a/internal/storage/file.go b/internal/storage/file.go index 688bfe7..66722a7 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -73,12 +73,6 @@ func (fs *FileStorage) UpdateMetric(ctx context.Context, mType, mName string, mV fs.mutex.Lock() defer fs.mutex.Unlock() - log.Debug(). - Str("type", mType). - Str("name", mName). - Interface("value", mValue). - Msg("Updating metric") - if err := fs.storage.UpdateMetric(ctx, mType, mName, mValue); err != nil { log.Error().Err(err).Msg("failed update metric from file storage") return fmt.Errorf("failed update metric from file storage %w", err) @@ -105,7 +99,10 @@ func (fs *FileStorage) GetMetric(ctx context.Context, mType, mName string) (any, val, ok := fs.storage.GetMetric(ctx, mType, mName) if !ok { - log.Error().Msg("can't get valid metric") + log.Error(). + Str("type", mType). + Str("name", mName). + Msg("can't get valid metric") return nil, false } diff --git a/pkg/data-base/file-storage.go b/pkg/data-base/file-storage.go index 8a182c2..08e2aa7 100644 --- a/pkg/data-base/file-storage.go +++ b/pkg/data-base/file-storage.go @@ -105,7 +105,7 @@ func LoadFromDB(ctx context.Context, collector col.Collector, path string) error return fmt.Errorf("can't read file %s with DB %w", path, err) } - log.Debug().Str("data", string(bytes)).Msg("metrics") + fmt.Printf("load data: %s", string(bytes)) if len(bytes) == 0 { log.Warn().Msgf("DB file %s is empty, skipping restore", path) return nil From fd3638e0c150291b39f754236548c82f1f14878c Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Tue, 15 Jul 2025 16:03:41 +0300 Subject: [PATCH 080/126] maybe fix problem --- internal/handlers/server/server.go | 3 +-- pkg/data-base/file-storage.go | 16 +++------------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index f11f71d..b4c7be4 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -229,7 +229,7 @@ func GetMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { return } log.Info().Msgf("Received metric request: %+v", metric) - fmt.Printf("GET METRIC: Получение метрики типа '%s' с именем '%s'\n", metric.MType, metric.ID) + if !FillMetricValueFromStorage(req.Context(), storage, &metric) { http.Error(resp, fmt.Sprintf("metric %s not found", metric.ID), http.StatusNotFound) return @@ -282,7 +282,6 @@ func UpdateMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { value = *metric.Delta } - fmt.Printf("UPDATE METRIC: Обновление метрики типа '%s', имя '%s', значение '%v'\n", metric.MType, metric.ID, value) if err := storage.UpdateMetric(req.Context(), metric.MType, metric.ID, value); err != nil { http.Error(resp, fmt.Sprintf("invalid update metric %s: %v", metric.ID, err), http.StatusBadRequest) } diff --git a/pkg/data-base/file-storage.go b/pkg/data-base/file-storage.go index 08e2aa7..b9ef3ad 100644 --- a/pkg/data-base/file-storage.go +++ b/pkg/data-base/file-storage.go @@ -14,12 +14,10 @@ import ( func SaveToDB(ctx context.Context, collector col.Collector, path string) error { allMetrics := collector.GetAllMetrics(ctx) - data := make([]mtr.Metrics, 0, len(allMetrics)) for _, metric := range allMetrics { var newMetric mtr.Metrics - newMetric.MType = metric.Type() newMetric.ID = metric.Name() @@ -27,19 +25,18 @@ func SaveToDB(ctx context.Context, collector col.Collector, path string) error { case mtr.GaugeType: val, ok := metric.Value().(float64) if !ok { - return fmt.Errorf("invalid type metric") + return fmt.Errorf("invalid gauge metric value") } newMetric.Value = &val case mtr.CounterType: val, ok := metric.Value().(int64) if !ok { - return fmt.Errorf("invalid type metric") + return fmt.Errorf("invalid counter metric value") } newMetric.Delta = &val default: - log.Error().Msg("unknown metric type") + return fmt.Errorf("unknown metric type") } - data = append(data, newMetric) } @@ -86,13 +83,6 @@ func SaveToDB(ctx context.Context, collector col.Collector, path string) error { Msg("Metrics successfully saved") return nil - // err = os.WriteFile(path, bytes, 0644) - // - // if err != nil { - // return fmt.Errorf("invalid write file %s: %w", path, err) - // } - // - // return nil } func LoadFromDB(ctx context.Context, collector col.Collector, path string) error { From 07b2a23a627b208cb0f5ebfbfc304751b0a5e5a0 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Tue, 15 Jul 2025 17:03:22 +0300 Subject: [PATCH 081/126] fix save --- pkg/data-base/file-storage.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/data-base/file-storage.go b/pkg/data-base/file-storage.go index b9ef3ad..d6b03b8 100644 --- a/pkg/data-base/file-storage.go +++ b/pkg/data-base/file-storage.go @@ -95,7 +95,6 @@ func LoadFromDB(ctx context.Context, collector col.Collector, path string) error return fmt.Errorf("can't read file %s with DB %w", path, err) } - fmt.Printf("load data: %s", string(bytes)) if len(bytes) == 0 { log.Warn().Msgf("DB file %s is empty, skipping restore", path) return nil From 329710e557026dfc9aef9dd154e7749b1bb95cf4 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Tue, 15 Jul 2025 19:55:36 +0300 Subject: [PATCH 082/126] fix request for postgres --- internal/storage/postgres.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/storage/postgres.go b/internal/storage/postgres.go index fc1f6f4..79da5f4 100644 --- a/internal/storage/postgres.go +++ b/internal/storage/postgres.go @@ -54,7 +54,7 @@ func (db *Database) GetMetric(ctx context.Context, mType, mName string) (any, bo row := db.DB.QueryRowContext(ctx, "SELECT ID, MType, Delta, Value FROM collector"+ - "WHERE ID = $1 AND MType = $2 LIMIT 1", mType, mName) + "WHERE ID = $1 AND MType = $2 LIMIT 1", mName, mType) var ( id string @@ -102,6 +102,7 @@ func (db *Database) GetAllMetrics(ctx context.Context) []mtr.Metric { log.Error().Err(err).Msg("The request was not processed") return nil } + defer rows.Close() var ( delta sql.NullInt64 From a2897906bb2eafd76464cef8ab2a270d3cae7750 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Tue, 15 Jul 2025 22:26:37 +0300 Subject: [PATCH 083/126] fix db postgres --- cmd/server/cfg_server.go | 3 +++ internal/storage/postgres.go | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 3396257..1491d36 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -151,6 +151,9 @@ func ChoiceCollector(ctx context.Context, opts *config.Options) (col.Collector, switch { case opts.DataBaseDSN != "": collector, err = storage.NewDatabase(ctx, opts.DataBaseDSN) + if err != nil { + return nil, fmt.Errorf("DB connection failed: %w", err) + } case opts.FileStoragePath != "": collector, err = storage.NewFileStorage(ctx, &storage.FileParams{ FileStoragePath: opts.FileStoragePath, diff --git a/internal/storage/postgres.go b/internal/storage/postgres.go index 79da5f4..49fcb3f 100644 --- a/internal/storage/postgres.go +++ b/internal/storage/postgres.go @@ -53,8 +53,8 @@ func (db *Database) GetMetric(ctx context.Context, mType, mName string) (any, bo defer db.mutex.RUnlock() row := db.DB.QueryRowContext(ctx, - "SELECT ID, MType, Delta, Value FROM collector"+ - "WHERE ID = $1 AND MType = $2 LIMIT 1", mName, mType) + `SELECT "ID", "MType", "Delta", "Value" FROM collector ` + + `WHERE "ID" = $1 AND "MType" = $2 LIMIT 1`, mName, mType) var ( id string From 54f164d8f656241d3b7c5f929738d50ff6e4ba51 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Wed, 16 Jul 2025 21:18:19 +0300 Subject: [PATCH 084/126] rebase from iter10 --- internal/storage/postgres.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/storage/postgres.go b/internal/storage/postgres.go index 49fcb3f..ba369db 100644 --- a/internal/storage/postgres.go +++ b/internal/storage/postgres.go @@ -53,8 +53,8 @@ func (db *Database) GetMetric(ctx context.Context, mType, mName string) (any, bo defer db.mutex.RUnlock() row := db.DB.QueryRowContext(ctx, - `SELECT "ID", "MType", "Delta", "Value" FROM collector ` + - `WHERE "ID" = $1 AND "MType" = $2 LIMIT 1`, mName, mType) + `SELECT "ID", "MType", "Delta", "Value" FROM collector `+ + `WHERE "ID" = $1 AND "MType" = $2 LIMIT 1`, mName, mType) var ( id string @@ -102,7 +102,11 @@ func (db *Database) GetAllMetrics(ctx context.Context) []mtr.Metric { log.Error().Err(err).Msg("The request was not processed") return nil } - defer rows.Close() + defer func() { + if err := rows.Close(); err != nil { + log.Printf("failed to close rows: %v", err) + } + }() var ( delta sql.NullInt64 From b275feaf3395c9c1ca9c558a5cc2ae637e847b65 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Sun, 13 Jul 2025 00:55:15 +0300 Subject: [PATCH 085/126] empty commit, from iter10 version From d3eb5c59a16bd0aa5594001455f364df7d38e019 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Sun, 13 Jul 2025 18:28:56 +0300 Subject: [PATCH 086/126] Iter11 refactor (#14) * rewrote the storage logic for files and the database for the collector * fix tests * write a template for postgresql, but haven't implemented the logic * filled out the template, corrected the logic * fix name ms for method MemStorage --- cmd/server/cfg_server.go | 2 +- internal/storage/file.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 1491d36..c532815 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -152,7 +152,7 @@ func ChoiceCollector(ctx context.Context, opts *config.Options) (col.Collector, case opts.DataBaseDSN != "": collector, err = storage.NewDatabase(ctx, opts.DataBaseDSN) if err != nil { - return nil, fmt.Errorf("DB connection failed: %w", err) + return nil, fmt.Errorf("DB connection failed: %w", err) } case opts.FileStoragePath != "": collector, err = storage.NewFileStorage(ctx, &storage.FileParams{ diff --git a/internal/storage/file.go b/internal/storage/file.go index 66722a7..239fa33 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -56,7 +56,6 @@ func NewFileStorage(ctx context.Context, fp *FileParams) (col.Collector, error) } case <-ctx.Done(): log.Info().Msg("Shutting down server, saving metrics") - if err := database.SaveToDB(ctx, fs.storage, fp.FileStoragePath); err != nil { log.Error().Err(err).Msg("Failed to save metrics during shutdown") } From e990777624c458c3fc3e3ba2cb6fb68356cde4ff Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Mon, 14 Jul 2025 00:42:31 +0300 Subject: [PATCH 087/126] fix ChoiceCollector --- cmd/server/cfg_server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index c532815..e1b2dcc 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -152,8 +152,8 @@ func ChoiceCollector(ctx context.Context, opts *config.Options) (col.Collector, case opts.DataBaseDSN != "": collector, err = storage.NewDatabase(ctx, opts.DataBaseDSN) if err != nil { - return nil, fmt.Errorf("DB connection failed: %w", err) - } + return nil, fmt.Errorf("DB connection failed: %w", err) + } case opts.FileStoragePath != "": collector, err = storage.NewFileStorage(ctx, &storage.FileParams{ FileStoragePath: opts.FileStoragePath, From b71e7f91c01a888afdd6a59860d3f863ccd42838 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Mon, 14 Jul 2025 01:13:47 +0300 Subject: [PATCH 088/126] delete if len() from LoadFromDB() --- internal/storage/file.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/storage/file.go b/internal/storage/file.go index 239fa33..acd1329 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -72,6 +72,12 @@ func (fs *FileStorage) UpdateMetric(ctx context.Context, mType, mName string, mV fs.mutex.Lock() defer fs.mutex.Unlock() + log.Debug(). + Str("type", mType). + Str("name", mName). + Interface("value", mValue). + Msg("Updating metric") + if err := fs.storage.UpdateMetric(ctx, mType, mName, mValue); err != nil { log.Error().Err(err).Msg("failed update metric from file storage") return fmt.Errorf("failed update metric from file storage %w", err) From 70a71162cb89697bb52adef3fa17180c902757e1 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Mon, 14 Jul 2025 01:23:01 +0300 Subject: [PATCH 089/126] . From 23ad2484cc304909f94ebd25f7d2b72d1ce91c35 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Mon, 14 Jul 2025 15:31:36 +0300 Subject: [PATCH 090/126] - From be6d91f245885018c6675af83af12889679f59a7 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Mon, 14 Jul 2025 22:30:33 +0300 Subject: [PATCH 091/126] didn't fix --- internal/storage/file.go | 1 + pkg/data-base/file-storage.go | 1 + 2 files changed, 2 insertions(+) diff --git a/internal/storage/file.go b/internal/storage/file.go index acd1329..de28320 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -56,6 +56,7 @@ func NewFileStorage(ctx context.Context, fp *FileParams) (col.Collector, error) } case <-ctx.Done(): log.Info().Msg("Shutting down server, saving metrics") + if err := database.SaveToDB(ctx, fs.storage, fp.FileStoragePath); err != nil { log.Error().Err(err).Msg("Failed to save metrics during shutdown") } diff --git a/pkg/data-base/file-storage.go b/pkg/data-base/file-storage.go index d6b03b8..21a1d8d 100644 --- a/pkg/data-base/file-storage.go +++ b/pkg/data-base/file-storage.go @@ -95,6 +95,7 @@ func LoadFromDB(ctx context.Context, collector col.Collector, path string) error return fmt.Errorf("can't read file %s with DB %w", path, err) } + log.Debug().Str("data", string(bytes)).Msg("metrics") if len(bytes) == 0 { log.Warn().Msgf("DB file %s is empty, skipping restore", path) return nil From ec357bd1e9f346d8ae5b9bbfa3de7c992394ab5c Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Mon, 14 Jul 2025 23:41:55 +0300 Subject: [PATCH 092/126] add create file for SaveToDB() --- pkg/data-base/file-storage.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/data-base/file-storage.go b/pkg/data-base/file-storage.go index 21a1d8d..6f3a4bf 100644 --- a/pkg/data-base/file-storage.go +++ b/pkg/data-base/file-storage.go @@ -83,6 +83,13 @@ func SaveToDB(ctx context.Context, collector col.Collector, path string) error { Msg("Metrics successfully saved") return nil + // err = os.WriteFile(path, bytes, 0644) + // + // if err != nil { + // return fmt.Errorf("invalid write file %s: %w", path, err) + // } + // + // return nil } func LoadFromDB(ctx context.Context, collector col.Collector, path string) error { From 15b5949b91274d0c96b1a7bb8ef70aa3d69d7b81 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Tue, 15 Jul 2025 01:01:17 +0300 Subject: [PATCH 093/126] add logs for CI --- internal/handlers/server/server.go | 3 ++- internal/storage/file.go | 6 ------ pkg/data-base/file-storage.go | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index b4c7be4..f11f71d 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -229,7 +229,7 @@ func GetMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { return } log.Info().Msgf("Received metric request: %+v", metric) - + fmt.Printf("GET METRIC: Получение метрики типа '%s' с именем '%s'\n", metric.MType, metric.ID) if !FillMetricValueFromStorage(req.Context(), storage, &metric) { http.Error(resp, fmt.Sprintf("metric %s not found", metric.ID), http.StatusNotFound) return @@ -282,6 +282,7 @@ func UpdateMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { value = *metric.Delta } + fmt.Printf("UPDATE METRIC: Обновление метрики типа '%s', имя '%s', значение '%v'\n", metric.MType, metric.ID, value) if err := storage.UpdateMetric(req.Context(), metric.MType, metric.ID, value); err != nil { http.Error(resp, fmt.Sprintf("invalid update metric %s: %v", metric.ID, err), http.StatusBadRequest) } diff --git a/internal/storage/file.go b/internal/storage/file.go index de28320..66722a7 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -73,12 +73,6 @@ func (fs *FileStorage) UpdateMetric(ctx context.Context, mType, mName string, mV fs.mutex.Lock() defer fs.mutex.Unlock() - log.Debug(). - Str("type", mType). - Str("name", mName). - Interface("value", mValue). - Msg("Updating metric") - if err := fs.storage.UpdateMetric(ctx, mType, mName, mValue); err != nil { log.Error().Err(err).Msg("failed update metric from file storage") return fmt.Errorf("failed update metric from file storage %w", err) diff --git a/pkg/data-base/file-storage.go b/pkg/data-base/file-storage.go index 6f3a4bf..8d7ae11 100644 --- a/pkg/data-base/file-storage.go +++ b/pkg/data-base/file-storage.go @@ -102,7 +102,7 @@ func LoadFromDB(ctx context.Context, collector col.Collector, path string) error return fmt.Errorf("can't read file %s with DB %w", path, err) } - log.Debug().Str("data", string(bytes)).Msg("metrics") + fmt.Printf("load data: %s", string(bytes)) if len(bytes) == 0 { log.Warn().Msgf("DB file %s is empty, skipping restore", path) return nil From dd4a9dd91845c5b5b7af4d5f55cd1597df695e37 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Tue, 15 Jul 2025 16:03:41 +0300 Subject: [PATCH 094/126] maybe fix problem --- internal/handlers/server/server.go | 3 +-- pkg/data-base/file-storage.go | 7 ------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index f11f71d..b4c7be4 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -229,7 +229,7 @@ func GetMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { return } log.Info().Msgf("Received metric request: %+v", metric) - fmt.Printf("GET METRIC: Получение метрики типа '%s' с именем '%s'\n", metric.MType, metric.ID) + if !FillMetricValueFromStorage(req.Context(), storage, &metric) { http.Error(resp, fmt.Sprintf("metric %s not found", metric.ID), http.StatusNotFound) return @@ -282,7 +282,6 @@ func UpdateMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { value = *metric.Delta } - fmt.Printf("UPDATE METRIC: Обновление метрики типа '%s', имя '%s', значение '%v'\n", metric.MType, metric.ID, value) if err := storage.UpdateMetric(req.Context(), metric.MType, metric.ID, value); err != nil { http.Error(resp, fmt.Sprintf("invalid update metric %s: %v", metric.ID, err), http.StatusBadRequest) } diff --git a/pkg/data-base/file-storage.go b/pkg/data-base/file-storage.go index 8d7ae11..b9ef3ad 100644 --- a/pkg/data-base/file-storage.go +++ b/pkg/data-base/file-storage.go @@ -83,13 +83,6 @@ func SaveToDB(ctx context.Context, collector col.Collector, path string) error { Msg("Metrics successfully saved") return nil - // err = os.WriteFile(path, bytes, 0644) - // - // if err != nil { - // return fmt.Errorf("invalid write file %s: %w", path, err) - // } - // - // return nil } func LoadFromDB(ctx context.Context, collector col.Collector, path string) error { From 8ad75f961feb1ff3ef5e46917934968db14c8704 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Tue, 15 Jul 2025 17:03:22 +0300 Subject: [PATCH 095/126] fix save --- pkg/data-base/file-storage.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/data-base/file-storage.go b/pkg/data-base/file-storage.go index b9ef3ad..d6b03b8 100644 --- a/pkg/data-base/file-storage.go +++ b/pkg/data-base/file-storage.go @@ -95,7 +95,6 @@ func LoadFromDB(ctx context.Context, collector col.Collector, path string) error return fmt.Errorf("can't read file %s with DB %w", path, err) } - fmt.Printf("load data: %s", string(bytes)) if len(bytes) == 0 { log.Warn().Msgf("DB file %s is empty, skipping restore", path) return nil From 1861e5cb478952987f1698849ef2909aeebdefac Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Wed, 16 Jul 2025 03:12:45 +0300 Subject: [PATCH 096/126] first bad version, but worked, need to do refactor code --- internal/handlers/server/server.go | 38 +++++++++++- internal/metrics/metrics.go | 5 ++ internal/metrics/metrics_easyjson.go | 90 ++++++++++++++++++++++++---- internal/router/roter.go | 4 ++ internal/storage/postgres.go | 84 ++++++++++++++++++++++---- pkg/converter/converter.go | 22 +++++++ 6 files changed, 218 insertions(+), 25 deletions(-) diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index b4c7be4..90bfebc 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -14,6 +14,7 @@ import ( col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/converter" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -218,7 +219,7 @@ func GetMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { return func(resp http.ResponseWriter, req *http.Request) { var metric mtr.Metrics - log.Info().Msg("GetMetricsHandlerJSON called\n\n") + log.Info().Msg("GetMetricsHandlerJSON called") if req.Header.Get("Content-Type") != "application/json" { http.Error(resp, "Content-Type must be application/json", http.StatusUnsupportedMediaType) return @@ -228,12 +229,12 @@ func GetMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { http.Error(resp, fmt.Sprintf("invalid json body: %v", err), http.StatusBadRequest) return } - log.Info().Msgf("Received metric request: %+v", metric) if !FillMetricValueFromStorage(req.Context(), storage, &metric) { http.Error(resp, fmt.Sprintf("metric %s not found", metric.ID), http.StatusNotFound) return } + log.Info().Msgf("Received metric request: %+v", metric) resp.Header().Set("Content-Type", "application/json") if _, err := easyjson.MarshalToWriter(&metric, resp); err != nil { @@ -265,6 +266,7 @@ func UpdateMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { var metric mtr.Metrics + log.Info().Msg("UpdateMetricsHandlerJSON called") if req.Header.Get("Content-Type") != "application/json" { http.Error(resp, "Content-Type must be application/json", http.StatusUnsupportedMediaType) return @@ -299,6 +301,38 @@ func UpdateMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { } } +func UpdatesMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + var jsonMetrics mtr.MetricsList + + log.Info().Msg("UpdatesMetricsHandlerJSON called") + if req.Header.Get("Content-Type") != "application/json" { + http.Error(w, "Content-Type must be application/json", http.StatusUnsupportedMediaType) + return + } + + if err := easyjson.UnmarshalFromReader(req.Body, &jsonMetrics); err != nil { + http.Error(w, fmt.Sprintf("invalid json body: %v", err), http.StatusBadRequest) + return + } + + metrics, err := converter.ConvertMetrics(jsonMetrics) + if err != nil { + http.Error(w, fmt.Sprintf("invalid convert metrics: %v", err), http.StatusBadRequest) + return + } + + for _, metric := range metrics { + if err := storage.UpdateMetric(req.Context(), metric.Type(), metric.Name(), metric.Value()); err != nil { + http.Error(w, fmt.Sprintf("failed update metric %v", err), http.StatusInternalServerError) + return + } + } + + w.WriteHeader(http.StatusOK) + } +} + func PingHandler(col col.Collector) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if err := col.Ping(r.Context()); err != nil { diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 8dd6853..fa4bd5c 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -3,6 +3,7 @@ package metrics import ( "errors" ) + type Metric interface { Value() any Name() string @@ -17,6 +18,10 @@ type Metrics struct { Delta *int64 `json:"delta,omitempty"` Value *float64 `json:"value,omitempty"` } + +//easyjson:json +type MetricsList []Metrics + type MetricTable struct { Name string Type string diff --git a/internal/metrics/metrics_easyjson.go b/internal/metrics/metrics_easyjson.go index 188ce47..2a38cf5 100644 --- a/internal/metrics/metrics_easyjson.go +++ b/internal/metrics/metrics_easyjson.go @@ -17,7 +17,73 @@ var ( _ easyjson.Marshaler ) -func easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(in *jlexer.Lexer, out *Metrics) { +func easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(in *jlexer.Lexer, out *MetricsList) { + isTopLevel := in.IsStart() + if in.IsNull() { + in.Skip() + *out = nil + } else { + in.Delim('[') + if *out == nil { + if !in.IsDelim(']') { + *out = make(MetricsList, 0, 1) + } else { + *out = MetricsList{} + } + } else { + *out = (*out)[:0] + } + for !in.IsDelim(']') { + var v1 Metrics + (v1).UnmarshalEasyJSON(in) + *out = append(*out, v1) + in.WantComma() + } + in.Delim(']') + } + if isTopLevel { + in.Consumed() + } +} +func easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(out *jwriter.Writer, in MetricsList) { + if in == nil && (out.Flags&jwriter.NilSliceAsEmpty) == 0 { + out.RawString("null") + } else { + out.RawByte('[') + for v2, v3 := range in { + if v2 > 0 { + out.RawByte(',') + } + (v3).MarshalEasyJSON(out) + } + out.RawByte(']') + } +} + +// MarshalJSON supports json.Marshaler interface +func (v MetricsList) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(&w, v) + return w.Buffer.BuildBytes(), w.Error +} + +// MarshalEasyJSON supports easyjson.Marshaler interface +func (v MetricsList) MarshalEasyJSON(w *jwriter.Writer) { + easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(w, v) +} + +// UnmarshalJSON supports json.Unmarshaler interface +func (v *MetricsList) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(&r, v) + return r.Error() +} + +// UnmarshalEasyJSON supports easyjson.Unmarshaler interface +func (v *MetricsList) UnmarshalEasyJSON(l *jlexer.Lexer) { + easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(l, v) +} +func easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(in *jlexer.Lexer, out *Metrics) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -70,7 +136,7 @@ func easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInte in.Consumed() } } -func easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(out *jwriter.Writer, in Metrics) { +func easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(out *jwriter.Writer, in Metrics) { out.RawByte('{') first := true _ = first @@ -100,27 +166,27 @@ func easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInte // MarshalJSON supports json.Marshaler interface func (v Metrics) MarshalJSON() ([]byte, error) { w := jwriter.Writer{} - easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(&w, v) + easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(&w, v) return w.Buffer.BuildBytes(), w.Error } // MarshalEasyJSON supports easyjson.Marshaler interface func (v Metrics) MarshalEasyJSON(w *jwriter.Writer) { - easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(w, v) + easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(w, v) } // UnmarshalJSON supports json.Unmarshaler interface func (v *Metrics) UnmarshalJSON(data []byte) error { r := jlexer.Lexer{Data: data} - easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(&r, v) + easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(&r, v) return r.Error() } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *Metrics) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(l, v) + easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(l, v) } -func easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(in *jlexer.Lexer, out *MetricTable) { +func easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics2(in *jlexer.Lexer, out *MetricTable) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -155,7 +221,7 @@ func easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInte in.Consumed() } } -func easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(out *jwriter.Writer, in MetricTable) { +func easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics2(out *jwriter.Writer, in MetricTable) { out.RawByte('{') first := true _ = first @@ -180,23 +246,23 @@ func easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInte // MarshalJSON supports json.Marshaler interface func (v MetricTable) MarshalJSON() ([]byte, error) { w := jwriter.Writer{} - easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(&w, v) + easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics2(&w, v) return w.Buffer.BuildBytes(), w.Error } // MarshalEasyJSON supports easyjson.Marshaler interface func (v MetricTable) MarshalEasyJSON(w *jwriter.Writer) { - easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(w, v) + easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics2(w, v) } // UnmarshalJSON supports json.Unmarshaler interface func (v *MetricTable) UnmarshalJSON(data []byte) error { r := jlexer.Lexer{Data: data} - easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(&r, v) + easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics2(&r, v) return r.Error() } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *MetricTable) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(l, v) + easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics2(l, v) } diff --git a/internal/router/roter.go b/internal/router/roter.go index f9570c4..7aeaeb1 100644 --- a/internal/router/roter.go +++ b/internal/router/roter.go @@ -32,6 +32,10 @@ func NewRouter(storage col.Collector, opts *config.Options) http.Handler { r.Route("/ping", func(r chi.Router) { r.Get("/", server.PingHandler(storage)) }) + + r.Route("/updates", func(r chi.Router) { + r.Post("/", server.UpdatesMetricsHandlerJSON(storage)) + }) }) return r diff --git a/internal/storage/postgres.go b/internal/storage/postgres.go index ba369db..b8ce21b 100644 --- a/internal/storage/postgres.go +++ b/internal/storage/postgres.go @@ -48,6 +48,44 @@ func NewDatabase(ctx context.Context, dataBaseDSN string) (col.Collector, error) }, nil } +func (db *Database) getMetric(ctx context.Context, mType, mName string) (any, bool) { + row := db.DB.QueryRowContext(ctx, + `SELECT "ID", "MType", "Delta", "Value" FROM collector `+ + `WHERE "ID" = $1 AND "MType" = $2 LIMIT 1`, mName, mType) + + var ( + id string + Type string + delta sql.NullInt64 + value sql.NullFloat64 + ) + + err := row.Scan(&id, &Type, &delta, &value) + if err != nil { + log.Error().Err(err).Msg("failed scan row") + return nil, false + } + + switch { + case value.Valid: + if Type != mtr.GaugeType { + return nil, false + } + + return value.Float64, true + + case delta.Valid: + if Type != mtr.CounterType { + return nil, false + } + + return delta.Int64, true + default: + log.Error().Msg("not valid value") + return nil, false + } +} + func (db *Database) GetMetric(ctx context.Context, mType, mName string) (any, bool) { db.mutex.RLock() defer db.mutex.RUnlock() @@ -163,29 +201,53 @@ func (db *Database) UpdateMetric(ctx context.Context, mType, mName string, mValu db.mutex.Lock() defer db.mutex.Unlock() - m := mtr.Metrics{ + oldValue, ok := db.getMetric(ctx, mType, mName) + + var m mtr.Metric + + if !ok { + switch v := mValue.(type) { + case float64: + m = mtr.NewGauge(mName, v) + case int64: + m = mtr.NewCounter(mName, v) + default: + return fmt.Errorf("unsupported type for metric value: %T", mValue) + } + } else { + switch v := oldValue.(type) { + case float64: + m = mtr.NewGauge(mName, v) + case int64: + m = mtr.NewCounter(mName, v) + default: + return fmt.Errorf("unsupported type for existing metric: %T", oldValue) + } + + m.Update(mValue) + } + metric := mtr.Metrics{ ID: mName, MType: mType, } - //TODO - make method maybe for Metrics type - switch m.MType { + switch metric.MType { case mtr.GaugeType: - val, ok := mValue.(float64) + val, ok := m.Value().(float64) if !ok { - return fmt.Errorf("expected float64 for gauge") + return fmt.Errorf("expected float64 for gauge, got %T", m.Value()) } - m.Value = &val + metric.Value = &val case mtr.CounterType: - val, ok := mValue.(int64) + val, ok := m.Value().(int64) if !ok { - return fmt.Errorf("expected int64 for counter") + return fmt.Errorf("expected int64 for counter, got %T", m.Value()) } - m.Delta = &val + metric.Delta = &val default: - return fmt.Errorf("unknown metric type: %s", m.MType) + return fmt.Errorf("unknown metric type: %s", metric.MType) } _, err := db.DB.ExecContext(ctx, @@ -195,7 +257,7 @@ func (db *Database) UpdateMetric(ctx context.Context, mType, mName string, mValu SET "Delta" = EXCLUDED."Delta", "Value" = EXCLUDED."Value", "MType" = EXCLUDED."MType";`, - m.ID, m.MType, m.Delta, m.Value) + metric.ID, metric.MType, metric.Delta, metric.Value) if err != nil { log.Error().Err(err).Msg("failed update insert into collector") diff --git a/pkg/converter/converter.go b/pkg/converter/converter.go index 49ced6e..44f0355 100644 --- a/pkg/converter/converter.go +++ b/pkg/converter/converter.go @@ -29,3 +29,25 @@ func ConvertMetrics(src []mtr.Metrics) ([]mtr.Metric, error) { return converted, nil } + +func ConvertMetric(src mtr.Metrics) (mtr.Metric, error) { + var converted mtr.Metric + + switch src.MType { + case mtr.GaugeType: + if src.Value == nil { + return nil, fmt.Errorf("nil gauge value for ID: %s", src.ID) + } + converted = mtr.NewGauge(src.ID, *src.Value) + + case mtr.CounterType: + if src.Delta == nil { + return nil, fmt.Errorf("nil counter value for ID: %s", src.ID) + } + converted = mtr.NewCounter(src.ID, *src.Delta) + default: + return nil, fmt.Errorf("unsupported metric type: %s", src.MType) + } + + return converted, nil +} From 8dfb7c126735adebb28f14ef5741a3fadb02ad93 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Wed, 16 Jul 2025 03:27:22 +0300 Subject: [PATCH 097/126] add serialization dir --- internal/handlers/agent/agent.go | 5 +- internal/handlers/server/server.go | 7 +- internal/metrics/metrics.go | 11 - internal/metrics/metrics_easyjson.go | 268 -------------------- internal/storage/postgres.go | 7 +- pkg/converter/converter.go | 5 +- pkg/data-base/file-storage.go | 8 +- pkg/serialization/serialization.go | 13 + pkg/serialization/serialization_easyjson.go | 188 ++++++++++++++ 9 files changed, 220 insertions(+), 292 deletions(-) delete mode 100644 internal/metrics/metrics_easyjson.go create mode 100644 pkg/serialization/serialization.go create mode 100644 pkg/serialization/serialization_easyjson.go diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 019baa1..0ae78a0 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -17,6 +17,7 @@ import ( "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/storage" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" rt "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/runtime-stats" + serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" ) func UpdateAllMetrics(ctx context.Context, storage *storage.MemStorage) { @@ -53,7 +54,7 @@ func SendAllMetrics(ctx context.Context, client *resty.Client, storage *storage. mType := metric.Type() mName := metric.Name() - metricJSON := mtr.Metrics{ + metricJSON := serialize.Metric{ ID: mName, MType: mType, } @@ -84,7 +85,7 @@ func SendAllMetrics(ctx context.Context, client *resty.Client, storage *storage. } } -func sendMetric(client *resty.Client, metricJSON *mtr.Metrics) { +func sendMetric(client *resty.Client, metricJSON *serialize.Metric) { backoffSchedule := []time.Duration{ 100 * time.Millisecond, 500 * time.Millisecond, diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index 90bfebc..9bd1858 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -16,6 +16,7 @@ import ( mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/converter" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" + serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" ) func ConvertByType(mType, mValue string) (any, error) { @@ -192,7 +193,7 @@ func UpdateMetric(storage col.Collector) http.HandlerFunc { } } -func FillMetricValueFromStorage(ctx context.Context, storage col.Collector, metric *mtr.Metrics) bool { +func FillMetricValueFromStorage(ctx context.Context, storage col.Collector, metric *serialize.Metric) bool { value, ok := storage.GetMetric(ctx, metric.MType, metric.ID) if !ok { return false @@ -217,7 +218,7 @@ func FillMetricValueFromStorage(ctx context.Context, storage col.Collector, metr func GetMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { return func(resp http.ResponseWriter, req *http.Request) { - var metric mtr.Metrics + var metric serialize.Metric log.Info().Msg("GetMetricsHandlerJSON called") if req.Header.Get("Content-Type") != "application/json" { @@ -303,7 +304,7 @@ func UpdateMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { func UpdatesMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { - var jsonMetrics mtr.MetricsList + var jsonMetrics serialize.MetricsList log.Info().Msg("UpdatesMetricsHandlerJSON called") if req.Header.Get("Content-Type") != "application/json" { diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index fa4bd5c..0778aa1 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -11,17 +11,6 @@ type Metric interface { Update(mValue any) error } -//easyjson:json -type Metrics struct { - ID string `json:"id"` - MType string `json:"type"` - Delta *int64 `json:"delta,omitempty"` - Value *float64 `json:"value,omitempty"` -} - -//easyjson:json -type MetricsList []Metrics - type MetricTable struct { Name string Type string diff --git a/internal/metrics/metrics_easyjson.go b/internal/metrics/metrics_easyjson.go deleted file mode 100644 index 2a38cf5..0000000 --- a/internal/metrics/metrics_easyjson.go +++ /dev/null @@ -1,268 +0,0 @@ -// Code generated by easyjson for marshaling/unmarshaling. DO NOT EDIT. - -package metrics - -import ( - json "encoding/json" - easyjson "github.com/mailru/easyjson" - jlexer "github.com/mailru/easyjson/jlexer" - jwriter "github.com/mailru/easyjson/jwriter" -) - -// suppress unused package warning -var ( - _ *json.RawMessage - _ *jlexer.Lexer - _ *jwriter.Writer - _ easyjson.Marshaler -) - -func easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(in *jlexer.Lexer, out *MetricsList) { - isTopLevel := in.IsStart() - if in.IsNull() { - in.Skip() - *out = nil - } else { - in.Delim('[') - if *out == nil { - if !in.IsDelim(']') { - *out = make(MetricsList, 0, 1) - } else { - *out = MetricsList{} - } - } else { - *out = (*out)[:0] - } - for !in.IsDelim(']') { - var v1 Metrics - (v1).UnmarshalEasyJSON(in) - *out = append(*out, v1) - in.WantComma() - } - in.Delim(']') - } - if isTopLevel { - in.Consumed() - } -} -func easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(out *jwriter.Writer, in MetricsList) { - if in == nil && (out.Flags&jwriter.NilSliceAsEmpty) == 0 { - out.RawString("null") - } else { - out.RawByte('[') - for v2, v3 := range in { - if v2 > 0 { - out.RawByte(',') - } - (v3).MarshalEasyJSON(out) - } - out.RawByte(']') - } -} - -// MarshalJSON supports json.Marshaler interface -func (v MetricsList) MarshalJSON() ([]byte, error) { - w := jwriter.Writer{} - easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(&w, v) - return w.Buffer.BuildBytes(), w.Error -} - -// MarshalEasyJSON supports easyjson.Marshaler interface -func (v MetricsList) MarshalEasyJSON(w *jwriter.Writer) { - easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(w, v) -} - -// UnmarshalJSON supports json.Unmarshaler interface -func (v *MetricsList) UnmarshalJSON(data []byte) error { - r := jlexer.Lexer{Data: data} - easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(&r, v) - return r.Error() -} - -// UnmarshalEasyJSON supports easyjson.Unmarshaler interface -func (v *MetricsList) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics(l, v) -} -func easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(in *jlexer.Lexer, out *Metrics) { - isTopLevel := in.IsStart() - if in.IsNull() { - if isTopLevel { - in.Consumed() - } - in.Skip() - return - } - in.Delim('{') - for !in.IsDelim('}') { - key := in.UnsafeFieldName(false) - in.WantColon() - if in.IsNull() { - in.Skip() - in.WantComma() - continue - } - switch key { - case "id": - out.ID = string(in.String()) - case "type": - out.MType = string(in.String()) - case "delta": - if in.IsNull() { - in.Skip() - out.Delta = nil - } else { - if out.Delta == nil { - out.Delta = new(int64) - } - *out.Delta = int64(in.Int64()) - } - case "value": - if in.IsNull() { - in.Skip() - out.Value = nil - } else { - if out.Value == nil { - out.Value = new(float64) - } - *out.Value = float64(in.Float64()) - } - default: - in.SkipRecursive() - } - in.WantComma() - } - in.Delim('}') - if isTopLevel { - in.Consumed() - } -} -func easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(out *jwriter.Writer, in Metrics) { - out.RawByte('{') - first := true - _ = first - { - const prefix string = ",\"id\":" - out.RawString(prefix[1:]) - out.String(string(in.ID)) - } - { - const prefix string = ",\"type\":" - out.RawString(prefix) - out.String(string(in.MType)) - } - if in.Delta != nil { - const prefix string = ",\"delta\":" - out.RawString(prefix) - out.Int64(int64(*in.Delta)) - } - if in.Value != nil { - const prefix string = ",\"value\":" - out.RawString(prefix) - out.Float64(float64(*in.Value)) - } - out.RawByte('}') -} - -// MarshalJSON supports json.Marshaler interface -func (v Metrics) MarshalJSON() ([]byte, error) { - w := jwriter.Writer{} - easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(&w, v) - return w.Buffer.BuildBytes(), w.Error -} - -// MarshalEasyJSON supports easyjson.Marshaler interface -func (v Metrics) MarshalEasyJSON(w *jwriter.Writer) { - easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(w, v) -} - -// UnmarshalJSON supports json.Unmarshaler interface -func (v *Metrics) UnmarshalJSON(data []byte) error { - r := jlexer.Lexer{Data: data} - easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(&r, v) - return r.Error() -} - -// UnmarshalEasyJSON supports easyjson.Unmarshaler interface -func (v *Metrics) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics1(l, v) -} -func easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics2(in *jlexer.Lexer, out *MetricTable) { - isTopLevel := in.IsStart() - if in.IsNull() { - if isTopLevel { - in.Consumed() - } - in.Skip() - return - } - in.Delim('{') - for !in.IsDelim('}') { - key := in.UnsafeFieldName(false) - in.WantColon() - if in.IsNull() { - in.Skip() - in.WantComma() - continue - } - switch key { - case "Name": - out.Name = string(in.String()) - case "Type": - out.Type = string(in.String()) - case "Value": - out.Value = string(in.String()) - default: - in.SkipRecursive() - } - in.WantComma() - } - in.Delim('}') - if isTopLevel { - in.Consumed() - } -} -func easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics2(out *jwriter.Writer, in MetricTable) { - out.RawByte('{') - first := true - _ = first - { - const prefix string = ",\"Name\":" - out.RawString(prefix[1:]) - out.String(string(in.Name)) - } - { - const prefix string = ",\"Type\":" - out.RawString(prefix) - out.String(string(in.Type)) - } - { - const prefix string = ",\"Value\":" - out.RawString(prefix) - out.String(string(in.Value)) - } - out.RawByte('}') -} - -// MarshalJSON supports json.Marshaler interface -func (v MetricTable) MarshalJSON() ([]byte, error) { - w := jwriter.Writer{} - easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics2(&w, v) - return w.Buffer.BuildBytes(), w.Error -} - -// MarshalEasyJSON supports easyjson.Marshaler interface -func (v MetricTable) MarshalEasyJSON(w *jwriter.Writer) { - easyjson2220f231EncodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics2(w, v) -} - -// UnmarshalJSON supports json.Unmarshaler interface -func (v *MetricTable) UnmarshalJSON(data []byte) error { - r := jlexer.Lexer{Data: data} - easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics2(&r, v) - return r.Error() -} - -// UnmarshalEasyJSON supports easyjson.Unmarshaler interface -func (v *MetricTable) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjson2220f231DecodeGithubComRAchKaplinMiptGolangCourseMetricsServiceInternalMetrics2(l, v) -} diff --git a/internal/storage/postgres.go b/internal/storage/postgres.go index b8ce21b..ea2d6d1 100644 --- a/internal/storage/postgres.go +++ b/internal/storage/postgres.go @@ -8,6 +8,7 @@ import ( col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/converter" "github.com/rs/zerolog/log" ) @@ -131,7 +132,7 @@ func (db *Database) GetAllMetrics(ctx context.Context) []mtr.Metric { db.mutex.RLock() defer db.mutex.RUnlock() - metrics := make([]mtr.Metrics, 0) + metrics := make(serialize.MetricsList, 0) rows, err := db.DB.QueryContext(ctx, "SELECT ID, MType, Delta, Value FROM collector") @@ -152,7 +153,7 @@ func (db *Database) GetAllMetrics(ctx context.Context) []mtr.Metric { ) for rows.Next() { - var metric mtr.Metrics + var metric serialize.Metric err = rows.Scan(&metric.ID, &metric.MType, &delta, &value) if err != nil { @@ -226,7 +227,7 @@ func (db *Database) UpdateMetric(ctx context.Context, mType, mName string, mValu m.Update(mValue) } - metric := mtr.Metrics{ + metric := serialize.Metric{ ID: mName, MType: mType, } diff --git a/pkg/converter/converter.go b/pkg/converter/converter.go index 44f0355..4438631 100644 --- a/pkg/converter/converter.go +++ b/pkg/converter/converter.go @@ -4,9 +4,10 @@ import ( "fmt" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" ) -func ConvertMetrics(src []mtr.Metrics) ([]mtr.Metric, error) { +func ConvertMetrics(src serialize.MetricsList) ([]mtr.Metric, error) { converted := make([]mtr.Metric, 0, len(src)) for _, m := range src { @@ -30,7 +31,7 @@ func ConvertMetrics(src []mtr.Metrics) ([]mtr.Metric, error) { return converted, nil } -func ConvertMetric(src mtr.Metrics) (mtr.Metric, error) { +func ConvertMetric(src serialize.Metric) (mtr.Metric, error) { var converted mtr.Metric switch src.MType { diff --git a/pkg/data-base/file-storage.go b/pkg/data-base/file-storage.go index d6b03b8..63dc969 100644 --- a/pkg/data-base/file-storage.go +++ b/pkg/data-base/file-storage.go @@ -10,14 +10,15 @@ import ( col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" + serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" ) func SaveToDB(ctx context.Context, collector col.Collector, path string) error { allMetrics := collector.GetAllMetrics(ctx) - data := make([]mtr.Metrics, 0, len(allMetrics)) + data := make(serialize.MetricsList, 0, len(allMetrics)) for _, metric := range allMetrics { - var newMetric mtr.Metrics + var newMetric serialize.Metric newMetric.MType = metric.Type() newMetric.ID = metric.Name() @@ -99,7 +100,8 @@ func LoadFromDB(ctx context.Context, collector col.Collector, path string) error log.Warn().Msgf("DB file %s is empty, skipping restore", path) return nil } - var data []mtr.Metrics + + var data serialize.MetricsList err = json.Unmarshal(bytes, &data) if err != nil { return fmt.Errorf("can't parse json format from DB %w", err) diff --git a/pkg/serialization/serialization.go b/pkg/serialization/serialization.go new file mode 100644 index 0000000..3b74986 --- /dev/null +++ b/pkg/serialization/serialization.go @@ -0,0 +1,13 @@ +package serialization + + +//easyjson:json +type Metric struct { + ID string `json:"id"` + MType string `json:"type"` + Delta *int64 `json:"delta,omitempty"` + Value *float64 `json:"value,omitempty"` +} + +//easyjson:json +type MetricsList []Metric diff --git a/pkg/serialization/serialization_easyjson.go b/pkg/serialization/serialization_easyjson.go new file mode 100644 index 0000000..074c7d3 --- /dev/null +++ b/pkg/serialization/serialization_easyjson.go @@ -0,0 +1,188 @@ +// Code generated by easyjson for marshaling/unmarshaling. DO NOT EDIT. + +package serialization + +import ( + json "encoding/json" + easyjson "github.com/mailru/easyjson" + jlexer "github.com/mailru/easyjson/jlexer" + jwriter "github.com/mailru/easyjson/jwriter" +) + +// suppress unused package warning +var ( + _ *json.RawMessage + _ *jlexer.Lexer + _ *jwriter.Writer + _ easyjson.Marshaler +) + +func easyjson91d3f04aDecodeGithubComRAchKaplinMiptGolangCourseMetricsServicePkgSerialization(in *jlexer.Lexer, out *MetricsList) { + isTopLevel := in.IsStart() + if in.IsNull() { + in.Skip() + *out = nil + } else { + in.Delim('[') + if *out == nil { + if !in.IsDelim(']') { + *out = make(MetricsList, 0, 1) + } else { + *out = MetricsList{} + } + } else { + *out = (*out)[:0] + } + for !in.IsDelim(']') { + var v1 Metric + (v1).UnmarshalEasyJSON(in) + *out = append(*out, v1) + in.WantComma() + } + in.Delim(']') + } + if isTopLevel { + in.Consumed() + } +} +func easyjson91d3f04aEncodeGithubComRAchKaplinMiptGolangCourseMetricsServicePkgSerialization(out *jwriter.Writer, in MetricsList) { + if in == nil && (out.Flags&jwriter.NilSliceAsEmpty) == 0 { + out.RawString("null") + } else { + out.RawByte('[') + for v2, v3 := range in { + if v2 > 0 { + out.RawByte(',') + } + (v3).MarshalEasyJSON(out) + } + out.RawByte(']') + } +} + +// MarshalJSON supports json.Marshaler interface +func (v MetricsList) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + easyjson91d3f04aEncodeGithubComRAchKaplinMiptGolangCourseMetricsServicePkgSerialization(&w, v) + return w.Buffer.BuildBytes(), w.Error +} + +// MarshalEasyJSON supports easyjson.Marshaler interface +func (v MetricsList) MarshalEasyJSON(w *jwriter.Writer) { + easyjson91d3f04aEncodeGithubComRAchKaplinMiptGolangCourseMetricsServicePkgSerialization(w, v) +} + +// UnmarshalJSON supports json.Unmarshaler interface +func (v *MetricsList) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + easyjson91d3f04aDecodeGithubComRAchKaplinMiptGolangCourseMetricsServicePkgSerialization(&r, v) + return r.Error() +} + +// UnmarshalEasyJSON supports easyjson.Unmarshaler interface +func (v *MetricsList) UnmarshalEasyJSON(l *jlexer.Lexer) { + easyjson91d3f04aDecodeGithubComRAchKaplinMiptGolangCourseMetricsServicePkgSerialization(l, v) +} +func easyjson91d3f04aDecodeGithubComRAchKaplinMiptGolangCourseMetricsServicePkgSerialization1(in *jlexer.Lexer, out *Metric) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "id": + out.ID = string(in.String()) + case "type": + out.MType = string(in.String()) + case "delta": + if in.IsNull() { + in.Skip() + out.Delta = nil + } else { + if out.Delta == nil { + out.Delta = new(int64) + } + *out.Delta = int64(in.Int64()) + } + case "value": + if in.IsNull() { + in.Skip() + out.Value = nil + } else { + if out.Value == nil { + out.Value = new(float64) + } + *out.Value = float64(in.Float64()) + } + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjson91d3f04aEncodeGithubComRAchKaplinMiptGolangCourseMetricsServicePkgSerialization1(out *jwriter.Writer, in Metric) { + out.RawByte('{') + first := true + _ = first + { + const prefix string = ",\"id\":" + out.RawString(prefix[1:]) + out.String(string(in.ID)) + } + { + const prefix string = ",\"type\":" + out.RawString(prefix) + out.String(string(in.MType)) + } + if in.Delta != nil { + const prefix string = ",\"delta\":" + out.RawString(prefix) + out.Int64(int64(*in.Delta)) + } + if in.Value != nil { + const prefix string = ",\"value\":" + out.RawString(prefix) + out.Float64(float64(*in.Value)) + } + out.RawByte('}') +} + +// MarshalJSON supports json.Marshaler interface +func (v Metric) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + easyjson91d3f04aEncodeGithubComRAchKaplinMiptGolangCourseMetricsServicePkgSerialization1(&w, v) + return w.Buffer.BuildBytes(), w.Error +} + +// MarshalEasyJSON supports easyjson.Marshaler interface +func (v Metric) MarshalEasyJSON(w *jwriter.Writer) { + easyjson91d3f04aEncodeGithubComRAchKaplinMiptGolangCourseMetricsServicePkgSerialization1(w, v) +} + +// UnmarshalJSON supports json.Unmarshaler interface +func (v *Metric) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + easyjson91d3f04aDecodeGithubComRAchKaplinMiptGolangCourseMetricsServicePkgSerialization1(&r, v) + return r.Error() +} + +// UnmarshalEasyJSON supports easyjson.Unmarshaler interface +func (v *Metric) UnmarshalEasyJSON(l *jlexer.Lexer) { + easyjson91d3f04aDecodeGithubComRAchKaplinMiptGolangCourseMetricsServicePkgSerialization1(l, v) +} From 02d20a4402e59619debe86d7dc0f899ac63aea13 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Wed, 16 Jul 2025 04:03:28 +0300 Subject: [PATCH 098/126] change return value on GetMetric bool->error --- internal/collector/collerctor.go | 2 +- internal/handlers/server/server.go | 10 ++-- internal/storage/file.go | 10 ++-- internal/storage/memory.go | 11 ++-- internal/storage/postgres.go | 82 ++++++++++++++++-------------- 5 files changed, 60 insertions(+), 55 deletions(-) diff --git a/internal/collector/collerctor.go b/internal/collector/collerctor.go index d256d8d..99f5e0e 100644 --- a/internal/collector/collerctor.go +++ b/internal/collector/collerctor.go @@ -7,7 +7,7 @@ import ( ) type Collector interface { - GetMetric(ctx context.Context, mType, mName string) (any, bool) + GetMetric(ctx context.Context, mType, mName string) (any, error) GetAllMetrics(ctx context.Context) []mtr.Metric UpdateMetric(ctx context.Context, mType, mName string, mValue any) error diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index 9bd1858..d48ace8 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -42,8 +42,8 @@ func GetMetric(storage col.Collector) http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { mType := chi.URLParam(req, "mType") mName := chi.URLParam(req, "mName") - value, found := storage.GetMetric(req.Context(), mType, mName) - if !found { + value, err := storage.GetMetric(req.Context(), mType, mName) + if err != nil { http.Error(res, fmt.Sprintf("Metric %s was not found", mName), http.StatusNotFound) return } @@ -62,7 +62,7 @@ func GetMetric(storage col.Collector) http.HandlerFunc { res.Header().Set("Content-Type", "text/plain") res.WriteHeader(http.StatusOK) - _, err := res.Write([]byte(valueStr)) + _, err = res.Write([]byte(valueStr)) if err != nil { log.Error().Msgf("Failed to write response: %v", err) @@ -194,8 +194,8 @@ func UpdateMetric(storage col.Collector) http.HandlerFunc { } func FillMetricValueFromStorage(ctx context.Context, storage col.Collector, metric *serialize.Metric) bool { - value, ok := storage.GetMetric(ctx, metric.MType, metric.ID) - if !ok { + value, err := storage.GetMetric(ctx, metric.MType, metric.ID) + if err != nil { return false } diff --git a/internal/storage/file.go b/internal/storage/file.go index 66722a7..eb51b18 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -93,20 +93,20 @@ func (fs *FileStorage) UpdateMetric(ctx context.Context, mType, mName string, mV return nil } -func (fs *FileStorage) GetMetric(ctx context.Context, mType, mName string) (any, bool) { +func (fs *FileStorage) GetMetric(ctx context.Context, mType, mName string) (any, error) { fs.mutex.RLock() defer fs.mutex.RUnlock() - val, ok := fs.storage.GetMetric(ctx, mType, mName) - if !ok { + val, err := fs.storage.GetMetric(ctx, mType, mName) + if err != nil { log.Error(). Str("type", mType). Str("name", mName). Msg("can't get valid metric") - return nil, false + return nil, fmt.Errorf("can't get valid metric: %v", err) } - return val, true + return val, nil } func (fs *FileStorage) GetAllMetrics(ctx context.Context) []mtr.Metric { diff --git a/internal/storage/memory.go b/internal/storage/memory.go index b630127..a4bbc34 100644 --- a/internal/storage/memory.go +++ b/internal/storage/memory.go @@ -54,15 +54,16 @@ func (ms *MemStorage) UpdateMetric(_ context.Context, mType, mName string, mValu return nil } -func (ms *MemStorage) GetMetric(_ context.Context, mType, mName string) (any, bool) { +func (ms *MemStorage) GetMetric(_ context.Context, mType, mName string) (any, error) { ms.mutex.RLock() defer ms.mutex.RUnlock() - if metric, ok := ms.storage[mType][mName]; ok { - return metric.Value(), true + metric, ok := ms.storage[mType][mName] + if !ok { + return nil, mtr.ErrMetricsNotFound } - return nil, false + return metric.Value(), nil } func (ms *MemStorage) GetAllMetrics(_ context.Context) []mtr.Metric { @@ -85,7 +86,7 @@ func (ms *MemStorage) GetAllMetrics(_ context.Context) []mtr.Metric { } func (ms *MemStorage) Ping(ctx context.Context) error { - return nil + return nil } func (ms *MemStorage) Close() error { diff --git a/internal/storage/postgres.go b/internal/storage/postgres.go index ea2d6d1..98c34a7 100644 --- a/internal/storage/postgres.go +++ b/internal/storage/postgres.go @@ -3,13 +3,14 @@ package storage import ( "context" "database/sql" + "errors" "fmt" "sync" col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" - serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/converter" + serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" "github.com/rs/zerolog/log" ) @@ -30,7 +31,7 @@ func NewDatabase(ctx context.Context, dataBaseDSN string) (col.Collector, error) "CREATE TABLE IF NOT EXISTS collector ("+ "\"ID\" VARCHAR(250) PRIMARY KEY,"+ "\"MType\" TEXT,"+ - "\"Delta\" INTEGER,"+ + "\"Delta\" BIGINT,"+ "\"Value\" DOUBLE PRECISION"+ ");") @@ -49,7 +50,7 @@ func NewDatabase(ctx context.Context, dataBaseDSN string) (col.Collector, error) }, nil } -func (db *Database) getMetric(ctx context.Context, mType, mName string) (any, bool) { +func (db *Database) getMetric(ctx context.Context, mType, mName string) (any, error) { row := db.DB.QueryRowContext(ctx, `SELECT "ID", "MType", "Delta", "Value" FROM collector `+ `WHERE "ID" = $1 AND "MType" = $2 LIMIT 1`, mName, mType) @@ -62,32 +63,33 @@ func (db *Database) getMetric(ctx context.Context, mType, mName string) (any, bo ) err := row.Scan(&id, &Type, &delta, &value) - if err != nil { - log.Error().Err(err).Msg("failed scan row") - return nil, false + if errors.Is(err, sql.ErrNoRows) { + return nil, mtr.ErrMetricsNotFound + } else if err != nil { + return nil, fmt.Errorf("row.Scan can't read: %w", err) } switch { case value.Valid: if Type != mtr.GaugeType { - return nil, false + return nil, mtr.ErrInvalidMetricsType } - return value.Float64, true + return value.Float64, nil case delta.Valid: if Type != mtr.CounterType { - return nil, false + return nil, mtr.ErrInvalidMetricsType } - return delta.Int64, true + return delta.Int64, nil default: log.Error().Msg("not valid value") - return nil, false + return nil, mtr.ErrInvalidValueType } } -func (db *Database) GetMetric(ctx context.Context, mType, mName string) (any, bool) { +func (db *Database) GetMetric(ctx context.Context, mType, mName string) (any, error) { db.mutex.RLock() defer db.mutex.RUnlock() @@ -103,28 +105,29 @@ func (db *Database) GetMetric(ctx context.Context, mType, mName string) (any, bo ) err := row.Scan(&id, &Type, &delta, &value) - if err != nil { - log.Error().Err(err).Msg("failed scan row") - return nil, false + if errors.Is(err, sql.ErrNoRows) { + return nil, mtr.ErrMetricsNotFound + } else if err != nil { + return nil, fmt.Errorf("row.Scan can't read: %w", err) } switch { case value.Valid: if Type != mtr.GaugeType { - return nil, false + return nil, mtr.ErrInvalidMetricsType } - return value.Float64, true + return value.Float64, nil case delta.Valid: if Type != mtr.CounterType { - return nil, false + return nil, mtr.ErrInvalidMetricsType } - return delta.Int64, true + return delta.Int64, nil default: log.Error().Msg("not valid value") - return nil, false + return nil, mtr.ErrInvalidValueType } } @@ -202,31 +205,32 @@ func (db *Database) UpdateMetric(ctx context.Context, mType, mName string, mValu db.mutex.Lock() defer db.mutex.Unlock() - oldValue, ok := db.getMetric(ctx, mType, mName) + oldValue, err := db.getMetric(ctx, mType, mName) var m mtr.Metric - if !ok { - switch v := mValue.(type) { - case float64: - m = mtr.NewGauge(mName, v) - case int64: - m = mtr.NewCounter(mName, v) - default: - return fmt.Errorf("unsupported type for metric value: %T", mValue) + switch newVal := mValue.(type) { + case float64: + if mType != mtr.GaugeType { + return mtr.ErrInvalidMetricsType } - } else { - switch v := oldValue.(type) { - case float64: - m = mtr.NewGauge(mName, v) - case int64: - m = mtr.NewCounter(mName, v) - default: - return fmt.Errorf("unsupported type for existing metric: %T", oldValue) + m = mtr.NewGauge(mName, newVal) + + case int64: + if mType != mtr.CounterType { + return mtr.ErrInvalidMetricsType + } + if errors.Is(err, mtr.ErrMetricsNotFound) { + m = mtr.NewCounter(mName, newVal) + } else { + m = mtr.NewCounter(mName, oldValue.(int64)) + m.Update(newVal) } - m.Update(mValue) + default: + return fmt.Errorf("unsupported metric type %T", mValue) } + metric := serialize.Metric{ ID: mName, MType: mType, @@ -251,7 +255,7 @@ func (db *Database) UpdateMetric(ctx context.Context, mType, mName string, mValu return fmt.Errorf("unknown metric type: %s", metric.MType) } - _, err := db.DB.ExecContext(ctx, + _, err = db.DB.ExecContext(ctx, `INSERT INTO collector ("ID", "MType", "Delta", "Value") VALUES ($1, $2, $3, $4) ON CONFLICT ("ID") DO UPDATE From 816eeb3c1b0c06f06122e25e1631f40b08bcd3a4 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Wed, 16 Jul 2025 04:17:13 +0300 Subject: [PATCH 099/126] add transaction --- internal/storage/postgres.go | 47 +++++++----------------------------- 1 file changed, 9 insertions(+), 38 deletions(-) diff --git a/internal/storage/postgres.go b/internal/storage/postgres.go index 98c34a7..48898fd 100644 --- a/internal/storage/postgres.go +++ b/internal/storage/postgres.go @@ -93,42 +93,7 @@ func (db *Database) GetMetric(ctx context.Context, mType, mName string) (any, er db.mutex.RLock() defer db.mutex.RUnlock() - row := db.DB.QueryRowContext(ctx, - `SELECT "ID", "MType", "Delta", "Value" FROM collector `+ - `WHERE "ID" = $1 AND "MType" = $2 LIMIT 1`, mName, mType) - - var ( - id string - Type string - delta sql.NullInt64 - value sql.NullFloat64 - ) - - err := row.Scan(&id, &Type, &delta, &value) - if errors.Is(err, sql.ErrNoRows) { - return nil, mtr.ErrMetricsNotFound - } else if err != nil { - return nil, fmt.Errorf("row.Scan can't read: %w", err) - } - - switch { - case value.Valid: - if Type != mtr.GaugeType { - return nil, mtr.ErrInvalidMetricsType - } - - return value.Float64, nil - - case delta.Valid: - if Type != mtr.CounterType { - return nil, mtr.ErrInvalidMetricsType - } - - return delta.Int64, nil - default: - log.Error().Msg("not valid value") - return nil, mtr.ErrInvalidValueType - } + return db.getMetric(ctx, mType, mName) } func (db *Database) GetAllMetrics(ctx context.Context) []mtr.Metric { @@ -255,7 +220,12 @@ func (db *Database) UpdateMetric(ctx context.Context, mType, mName string, mValu return fmt.Errorf("unknown metric type: %s", metric.MType) } - _, err = db.DB.ExecContext(ctx, + tx, err := db.DB.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("failed begin transaction") + } + + _, err = tx.ExecContext(ctx, `INSERT INTO collector ("ID", "MType", "Delta", "Value") VALUES ($1, $2, $3, $4) ON CONFLICT ("ID") DO UPDATE @@ -266,10 +236,11 @@ func (db *Database) UpdateMetric(ctx context.Context, mType, mName string, mValu if err != nil { log.Error().Err(err).Msg("failed update insert into collector") + tx.Rollback() return fmt.Errorf("failed update insert into collector: %w", err) } - return nil + return tx.Commit() } func (db *Database) Ping(ctx context.Context) error { From 58437f76f9fe73cc15afc0afac2e4d6d2bad4f4a Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Wed, 16 Jul 2025 04:49:05 +0300 Subject: [PATCH 100/126] fix lint --- internal/storage/postgres.go | 10 ++++++++-- internal/storage/storage_test.go | 19 +++++++++++-------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/internal/storage/postgres.go b/internal/storage/postgres.go index 48898fd..f9e421f 100644 --- a/internal/storage/postgres.go +++ b/internal/storage/postgres.go @@ -189,7 +189,9 @@ func (db *Database) UpdateMetric(ctx context.Context, mType, mName string, mValu m = mtr.NewCounter(mName, newVal) } else { m = mtr.NewCounter(mName, oldValue.(int64)) - m.Update(newVal) + if err := m.Update(newVal); err != nil { + return fmt.Errorf("failed update metric %v", err) + } } default: @@ -236,7 +238,11 @@ func (db *Database) UpdateMetric(ctx context.Context, mType, mName string, mValu if err != nil { log.Error().Err(err).Msg("failed update insert into collector") - tx.Rollback() + + if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { + log.Printf("failed to rollback transaction: %v", err) + } + return fmt.Errorf("failed update insert into collector: %w", err) } diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go index aa69a5d..43aa56a 100644 --- a/internal/storage/storage_test.go +++ b/internal/storage/storage_test.go @@ -106,8 +106,8 @@ func TestMemStorage_UpdateMetric(t *testing.T) { } if !tt.wantErr { - val, ok := ms.GetMetric(ctx, tt.args.mType, tt.args.mName) - if !ok { + val, err := ms.GetMetric(ctx, tt.args.mType, tt.args.mName) + if err != nil { t.Errorf("metric not found") return } @@ -121,7 +121,7 @@ func TestMemStorage_UpdateMetric(t *testing.T) { func TestMemStorage_GetMetric(t *testing.T) { ctx := context.Background() - + counter := mtr.NewCounter("requests", 42) gauge := mtr.NewGauge("temperature", 36.6) @@ -179,12 +179,15 @@ func TestMemStorage_GetMetric(t *testing.T) { mutex: sync.RWMutex{}, storage: tt.fields, } - gotVal, gotOk := ms.GetMetric(ctx, tt.mType, tt.mName) - if gotVal != tt.wantVal { - t.Errorf("GetMetric() gotVal = %v, want %v", gotVal, tt.wantVal) + gotVal, err := ms.GetMetric(ctx, tt.mType, tt.mName) + + if (err == nil) != tt.wantOk { + t.Errorf("GetMetric() error = %v, wantOk = %v", err, tt.wantOk) + return } - if gotOk != tt.wantOk { - t.Errorf("GetMetric() gotOk = %v, want %v", gotOk, tt.wantOk) + + if tt.wantOk && gotVal != tt.wantVal { + t.Errorf("GetMetric() gotVal = %v, want %v", gotVal, tt.wantVal) } }) } From 8ae36765c472f35c6cef3009d1514291ca54bfcf Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Wed, 16 Jul 2025 21:32:33 +0300 Subject: [PATCH 101/126] after rebase from iter11 --- internal/handlers/agent/agent.go | 2 +- internal/handlers/server/server.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 0ae78a0..18d882c 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -118,7 +118,7 @@ func sendMetric(client *resty.Client, metricJSON *serialize.Metric) { } } -func ConvertToGzipData(metricJSON *mtr.Metrics) (*bytes.Buffer, bool, error) { +func ConvertToGzipData(metricJSON *serialize.Metric) (*bytes.Buffer, bool, error) { jsonData, err := easyjson.Marshal(*metricJSON) if err != nil { log.Error().Err(err).Msg("Failed to marshal metricJSON") diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index d48ace8..3b919f2 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -265,7 +265,7 @@ func UpdateMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { reader = req.Body } - var metric mtr.Metrics + var metric serialize.Metric log.Info().Msg("UpdateMetricsHandlerJSON called") if req.Header.Get("Content-Type") != "application/json" { From 271d6445ed3b1628b3cbaf0ae101d02df50c1fd8 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Wed, 16 Jul 2025 23:42:38 +0300 Subject: [PATCH 102/126] change design for CRUD app --- internal/handlers/server/server.go | 23 ++++++++----- pkg/converter/converter.go | 21 ++++-------- pkg/serialization/serialization.go | 52 ++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 22 deletions(-) diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index 3b919f2..646ea1e 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -231,11 +231,18 @@ func GetMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { return } - if !FillMetricValueFromStorage(req.Context(), storage, &metric) { - http.Error(resp, fmt.Sprintf("metric %s not found", metric.ID), http.StatusNotFound) + value, err := storage.GetMetric(req.Context(), metric.MType, metric.ID) + if err != nil { + log.Error().Err(err).Msg("can't get valid metric") + http.Error(resp, "can't get valid metric", http.StatusNotFound) + return + } + + if err := metric.SetValue(value); err != nil { + log.Error().Err(err).Msg("can't set new value") + http.Error(resp, "can't set new value", http.StatusInternalServerError) return } - log.Info().Msgf("Received metric request: %+v", metric) resp.Header().Set("Content-Type", "application/json") if _, err := easyjson.MarshalToWriter(&metric, resp); err != nil { @@ -278,11 +285,11 @@ func UpdateMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { return } - var value any - if metric.MType == mtr.GaugeType { - value = *metric.Value - } else { - value = *metric.Delta + value, err := metric.GetValue() + if err != nil { + log.Error().Err(err).Msg("can't get value") + http.Error(resp, "can't get value", http.StatusBadRequest) + return } if err := storage.UpdateMetric(req.Context(), metric.MType, metric.ID, value); err != nil { diff --git a/pkg/converter/converter.go b/pkg/converter/converter.go index 4438631..8a8491d 100644 --- a/pkg/converter/converter.go +++ b/pkg/converter/converter.go @@ -5,27 +5,20 @@ import ( mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" + "github.com/rs/zerolog/log" ) func ConvertMetrics(src serialize.MetricsList) ([]mtr.Metric, error) { converted := make([]mtr.Metric, 0, len(src)) for _, m := range src { - switch m.MType { - case mtr.GaugeType: - if m.Value == nil { - return nil, fmt.Errorf("nil gauge value for ID: %s", m.ID) - } - converted = append(converted, mtr.NewGauge(m.ID, *m.Value)) - - case mtr.CounterType: - if m.Delta == nil { - return nil, fmt.Errorf("nil counter value for ID: %s", m.ID) - } - converted = append(converted, mtr.NewCounter(m.ID, *m.Delta)) - default: - return nil, fmt.Errorf("unsupported metric type: %s", m.MType) + metric, err := ConvertMetric(m) + if err != nil { + log.Error().Err(err).Msg("failed convert metric") + return nil, fmt.Errorf("metric failed convert %+v", m) } + + converted = append(converted, metric) } return converted, nil diff --git a/pkg/serialization/serialization.go b/pkg/serialization/serialization.go index 3b74986..edaef03 100644 --- a/pkg/serialization/serialization.go +++ b/pkg/serialization/serialization.go @@ -1,5 +1,9 @@ package serialization +import ( + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" +) //easyjson:json type Metric struct { @@ -11,3 +15,51 @@ type Metric struct { //easyjson:json type MetricsList []Metric + +func (mtr *Metric) SetValue(value any) error { + switch mtr.MType { + case metrics.GaugeType: + val, ok := value.(float64) + if !ok { + log.Error().Msg("not value type of value") + return metrics.ErrInvalidValueType + } + + mtr.Value = &val + return nil + + case metrics.CounterType: + val, ok := value.(int64) + if !ok { + log.Error().Msg("not value type of value") + return metrics.ErrInvalidValueType + } + + mtr.Delta = &val + return nil + + default: + return metrics.ErrInvalidMetricsType + } +} + +func (mtr *Metric) GetValue() (any, error) { + switch mtr.MType { + case metrics.GaugeType: + if mtr.Value == nil { + return nil, metrics.ErrMetricsNotFound + } + + return *mtr.Value, nil + + case metrics.CounterType: + if mtr.Delta == nil { + return nil, metrics.ErrMetricsNotFound + } + + return *mtr.Delta, nil + + default: + return nil, metrics.ErrInvalidMetricsType + } +} From dffa07d624026e8c51f324fae79a8535439b2a69 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Thu, 17 Jul 2025 00:31:58 +0300 Subject: [PATCH 103/126] fix mutex on files database --- internal/storage/file.go | 26 +++++++++++++++++--------- pkg/data-base/file-storage.go | 9 ++------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/internal/storage/file.go b/internal/storage/file.go index eb51b18..9af5bb7 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -27,14 +27,27 @@ type FileParams struct { StoreInterval int } +func (fs *FileStorage) save(ctx context.Context) { + fs.mutex.Lock() + defer fs.mutex.Unlock() + + if err := database.SaveToDB(ctx, fs.storage, fs.filePath); err != nil { + log.Error().Err(err).Msg("failed to save DB") + } +} + func NewFileStorage(ctx context.Context, fp *FileParams) (col.Collector, error) { fs := &FileStorage{ filePath: fp.FileStoragePath, storage: NewMemStorage(), - SyncRecord: false, + SyncRecord: fp.StoreInterval == 0, } if fp.RestoreOnStart { - if err := database.LoadFromDB(ctx, fs.storage, fp.FileStoragePath); err != nil && !errors.Is(err, os.ErrNotExist) { + fs.mutex.Lock() + err := database.LoadFromDB(ctx, fs.storage, fp.FileStoragePath) + fs.mutex.Unlock() + + if err != nil && !errors.Is(err, os.ErrNotExist) { return nil, fmt.Errorf("LoadFromDB error %w", err) } } @@ -51,15 +64,10 @@ func NewFileStorage(ctx context.Context, fp *FileParams) (col.Collector, error) for { select { case <-ticker.C: - if err := database.SaveToDB(ctx, fs.storage, fp.FileStoragePath); err != nil { - log.Error().Err(err).Msg("failed to save DB") - } + fs.save(ctx) case <-ctx.Done(): log.Info().Msg("Shutting down server, saving metrics") - - if err := database.SaveToDB(ctx, fs.storage, fp.FileStoragePath); err != nil { - log.Error().Err(err).Msg("Failed to save metrics during shutdown") - } + fs.save(ctx) return } } diff --git a/pkg/data-base/file-storage.go b/pkg/data-base/file-storage.go index 63dc969..ad4f32d 100644 --- a/pkg/data-base/file-storage.go +++ b/pkg/data-base/file-storage.go @@ -55,8 +55,7 @@ func SaveToDB(ctx context.Context, collector col.Collector, path string) error { return fmt.Errorf("failed to create directory: %w", err) } - tmpPath := path + ".tmp" - file, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return fmt.Errorf("failed to create temp file: %w", err) } @@ -74,10 +73,6 @@ func SaveToDB(ctx context.Context, collector col.Collector, path string) error { return fmt.Errorf("sync failed: %w", err) } - if err := os.Rename(tmpPath, path); err != nil { - return fmt.Errorf("rename failed: %w", err) - } - log.Info(). Str("path", path). Int("metrics_saved", len(data)). @@ -98,7 +93,7 @@ func LoadFromDB(ctx context.Context, collector col.Collector, path string) error if len(bytes) == 0 { log.Warn().Msgf("DB file %s is empty, skipping restore", path) - return nil + return os.ErrNotExist } var data serialize.MetricsList From 98bdfe50f15894847f6873aef474a5074b8634fc Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Thu, 17 Jul 2025 00:43:50 +0300 Subject: [PATCH 104/126] fix --- pkg/data-base/file-storage.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/data-base/file-storage.go b/pkg/data-base/file-storage.go index ad4f32d..f8500ae 100644 --- a/pkg/data-base/file-storage.go +++ b/pkg/data-base/file-storage.go @@ -55,7 +55,8 @@ func SaveToDB(ctx context.Context, collector col.Collector, path string) error { return fmt.Errorf("failed to create directory: %w", err) } - file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + tmpPath := path + ".tmp" + file, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return fmt.Errorf("failed to create temp file: %w", err) } @@ -73,6 +74,10 @@ func SaveToDB(ctx context.Context, collector col.Collector, path string) error { return fmt.Errorf("sync failed: %w", err) } + if err := os.Rename(tmpPath, path); err != nil { + return fmt.Errorf("rename failed: %w", err) + } + log.Info(). Str("path", path). Int("metrics_saved", len(data)). From f05c74abda7d5f433697658556fdfd3f3ee2a1b5 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Thu, 17 Jul 2025 17:37:25 +0300 Subject: [PATCH 105/126] add retriable-errors for postgres --- go.mod | 5 ++ go.sum | 13 ++++++ internal/storage/postgres.go | 28 ++++++++---- pkg/errors-handlers/handle-errors.go | 68 ++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 8 deletions(-) create mode 100644 pkg/errors-handlers/handle-errors.go diff --git a/go.mod b/go.mod index 48383dc..18e2f57 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,12 @@ require ( require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgconn v1.14.3 // indirect + github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 // indirect + github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.3.3 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect diff --git a/go.sum b/go.sum index 06a96de..adca236 100644 --- a/go.sum +++ b/go.sum @@ -12,8 +12,19 @@ github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= +github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= +github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 h1:Dj0L5fhJ9F82ZJyVOmBx6msDp/kfd1t9GRfny/mfJA0= +github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= +github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.7.5 h1:JHGfMnQY+IEtGM63d+NGMjoRpysB2JBwDr5fsngwmJs= @@ -44,6 +55,7 @@ github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -63,6 +75,7 @@ golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/storage/postgres.go b/internal/storage/postgres.go index f9e421f..6b586d6 100644 --- a/internal/storage/postgres.go +++ b/internal/storage/postgres.go @@ -10,6 +10,7 @@ import ( col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/converter" + errH "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/errors-handlers" serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" "github.com/rs/zerolog/log" ) @@ -51,10 +52,6 @@ func NewDatabase(ctx context.Context, dataBaseDSN string) (col.Collector, error) } func (db *Database) getMetric(ctx context.Context, mType, mName string) (any, error) { - row := db.DB.QueryRowContext(ctx, - `SELECT "ID", "MType", "Delta", "Value" FROM collector `+ - `WHERE "ID" = $1 AND "MType" = $2 LIMIT 1`, mName, mType) - var ( id string Type string @@ -62,7 +59,16 @@ func (db *Database) getMetric(ctx context.Context, mType, mName string) (any, er value sql.NullFloat64 ) - err := row.Scan(&id, &Type, &delta, &value) + getMtr := func() error { + row := db.DB.QueryRowContext(ctx, + `SELECT "ID", "MType", "Delta", "Value" FROM collector `+ + `WHERE "ID" = $1 AND "MType" = $2 LIMIT 1`, mName, mType) + + return row.Scan(&id, &Type, &delta, &value) + } + + err := errH.WithRetry(getMtr, errH.IsPostgresRetriableError) + if errors.Is(err, sql.ErrNoRows) { return nil, mtr.ErrMetricsNotFound } else if err != nil { @@ -227,14 +233,20 @@ func (db *Database) UpdateMetric(ctx context.Context, mType, mName string, mValu return fmt.Errorf("failed begin transaction") } - _, err = tx.ExecContext(ctx, - `INSERT INTO collector ("ID", "MType", "Delta", "Value") + execContext := func() error { + _, err = tx.ExecContext(ctx, + `INSERT INTO collector ("ID", "MType", "Delta", "Value") VALUES ($1, $2, $3, $4) ON CONFLICT ("ID") DO UPDATE SET "Delta" = EXCLUDED."Delta", "Value" = EXCLUDED."Value", "MType" = EXCLUDED."MType";`, - metric.ID, metric.MType, metric.Delta, metric.Value) + metric.ID, metric.MType, metric.Delta, metric.Value) + + return err + } + + err = errH.WithRetry(execContext, errH.IsPostgresRetriableError) if err != nil { log.Error().Err(err).Msg("failed update insert into collector") diff --git a/pkg/errors-handlers/handle-errors.go b/pkg/errors-handlers/handle-errors.go new file mode 100644 index 0000000..9a2c98b --- /dev/null +++ b/pkg/errors-handlers/handle-errors.go @@ -0,0 +1,68 @@ +package errors + +import ( + "errors" + "time" + + "github.com/jackc/pgconn" + "github.com/jackc/pgerrcode" +) + +func WithRetry(f func() error, searchError func(error) bool) error { + backoffSchedule := []time.Duration{ + 1 * time.Second, + 3 * time.Second, + 5 * time.Second, + } + + var err error + + for _, backoff := range backoffSchedule { + err = f() + if err != nil { + return err + } + + if !searchError(err) { + return err + } + + time.Sleep(backoff) + } + + return err +} + +func IsPostgresRetriableError(err error) bool { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + switch pgErr.Code { + case + // Class 08 — Connection Exception + pgerrcode.ConnectionException, + pgerrcode.ConnectionDoesNotExist, + pgerrcode.ConnectionFailure, + pgerrcode.SQLClientUnableToEstablishSQLConnection, + pgerrcode.SQLServerRejectedEstablishmentOfSQLConnection, + pgerrcode.TransactionResolutionUnknown, + pgerrcode.ProtocolViolation, + + // Class 40 — Transaction Rollback + pgerrcode.TransactionRollback, + pgerrcode.SerializationFailure, + pgerrcode.DeadlockDetected, + + // Class 53 — Insufficient Resources + pgerrcode.TooManyConnections, + + // Class 57 — Operator Intervention + pgerrcode.AdminShutdown, + pgerrcode.CrashShutdown, + pgerrcode.CannotConnectNow: + + return true + } + } + + return false +} From 37a2edca1a88722bb53d55e7b509e0eb262c4495 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Sun, 20 Jul 2025 17:50:38 +0300 Subject: [PATCH 106/126] after code review --- .gitignore | 2 +- cmd/server/cfg_server.go | 47 +++++------------------ internal/collector/config/config.go | 49 ++++++++++++++++++++++++ internal/handlers/server/server.go | 1 + internal/router/{roter.go => router.go} | 0 internal/storage/file.go | 29 +++++++------- internal/storage/postgres.go | 29 +++----------- pkg/{data-base => files}/file-storage.go | 0 8 files changed, 80 insertions(+), 77 deletions(-) create mode 100644 internal/collector/config/config.go rename internal/router/{roter.go => router.go} (100%) rename pkg/{data-base => files}/file-storage.go (100%) diff --git a/.gitignore b/.gitignore index 6cc5864..4aa7bde 100644 --- a/.gitignore +++ b/.gitignore @@ -36,5 +36,5 @@ logFileAgent.log cmd/server/server cmd/agent/agent -tmp +/tmp log.txt diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index e1b2dcc..9df5ae7 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -12,10 +12,9 @@ import ( _ "github.com/jackc/pgx/v5/stdlib" "github.com/spf13/cobra" - col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" + colcfg "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector/config" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/router" - storage "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/storage" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -33,8 +32,8 @@ var rootCmd = &cobra.Command{ Short: "MetricService", Long: "MetricService", Args: cobra.NoArgs, - PreRunE: PreRunE, - RunE: RunE, + PreRunE: preRunE, + RunE: runE, } func init() { @@ -45,7 +44,7 @@ func init() { rootCmd.Flags().StringVarP(&dataBaseDSN, "d", "d", config.DefaultDataBaseDSN, "database dsn") } -func PreRunE(cmd *cobra.Command, args []string) error { +func preRunE(cmd *cobra.Command, args []string) error { var err error opts, err = config.ParseOptionsFromCmdAndEnvs(cmd, &config.Options{ EndPointAddr: endPointAddr, @@ -65,7 +64,7 @@ func PreRunE(cmd *cobra.Command, args []string) error { return err } -func RunE(cmd *cobra.Command, args []string) error { +func runE(cmd *cobra.Command, args []string) error { logFile, err := log.InitLogger("logFileServer.log") if err != nil { return fmt.Errorf("logger init error: %w", err) @@ -104,7 +103,10 @@ func startServer(ctx context.Context, opts *config.Options) error { Str("address", opts.EndPointAddr). Msg("Server configuration") - collector, err := ChoiceCollector(ctx, opts) + collector, err := colcfg.NewCollector(&colcfg.Params{ + Ctx: ctx, + Opts: opts, + }) if err != nil { return fmt.Errorf("failed to create collector: %w", err) } @@ -141,34 +143,3 @@ func startServer(ctx context.Context, opts *config.Options) error { return nil } - -func ChoiceCollector(ctx context.Context, opts *config.Options) (col.Collector, error) { - var ( - collector col.Collector - err error - ) - - switch { - case opts.DataBaseDSN != "": - collector, err = storage.NewDatabase(ctx, opts.DataBaseDSN) - if err != nil { - return nil, fmt.Errorf("DB connection failed: %w", err) - } - case opts.FileStoragePath != "": - collector, err = storage.NewFileStorage(ctx, &storage.FileParams{ - FileStoragePath: opts.FileStoragePath, - RestoreOnStart: opts.RestoreOnStart, - StoreInterval: opts.StoreInterval}) - - log.Debug().Msg("chose file storage") - default: - collector = storage.NewMemStorage() - } - - if err != nil { - log.Error().Err(err).Msg("failed create storage") - return nil, err - } - - return collector, nil -} diff --git a/internal/collector/config/config.go b/internal/collector/config/config.go new file mode 100644 index 0000000..a6b112d --- /dev/null +++ b/internal/collector/config/config.go @@ -0,0 +1,49 @@ +package config + +import ( + "context" + "fmt" + + col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/storage" + log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" +) + +type Params struct { + Ctx context.Context + Opts *config.Options +} + +func NewCollector(params *Params) (col.Collector, error) { + var ( + collector col.Collector + err error + ) + + switch { + case params.Opts.DataBaseDSN != "": + collector, err = storage.NewDatabase(params.Ctx, params.Opts.DataBaseDSN) + if err != nil { + return nil, fmt.Errorf("DB connection failed: %w", err) + } + + case params.Opts.FileStoragePath != "": + collector, err = storage.NewFileStorage(params.Ctx, &storage.FileParams{ + FileStoragePath: params.Opts.FileStoragePath, + RestoreOnStart: params.Opts.RestoreOnStart, + StoreInterval: params.Opts.StoreInterval}) + + log.Debug().Msg("chose file storage") + + default: + collector = storage.NewMemStorage() + } + + if err != nil { + log.Error().Err(err).Msg("failed create storage") + return nil, err + } + + return collector, nil +} diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index 646ea1e..a359052 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -344,6 +344,7 @@ func UpdatesMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { func PingHandler(col col.Collector) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if err := col.Ping(r.Context()); err != nil { + log.Error().Err(err).Msg("failed ping") http.Error(w, "can't ping DB", http.StatusInternalServerError) return } diff --git a/internal/router/roter.go b/internal/router/router.go similarity index 100% rename from internal/router/roter.go rename to internal/router/router.go diff --git a/internal/storage/file.go b/internal/storage/file.go index 9af5bb7..aad3368 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -10,12 +10,13 @@ import ( col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" - database "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/data-base" + files "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/files" "github.com/rs/zerolog/log" ) type FileStorage struct { mutex sync.RWMutex + wg *sync.WaitGroup filePath string storage col.Collector SyncRecord bool @@ -31,33 +32,31 @@ func (fs *FileStorage) save(ctx context.Context) { fs.mutex.Lock() defer fs.mutex.Unlock() - if err := database.SaveToDB(ctx, fs.storage, fs.filePath); err != nil { + if err := files.SaveToDB(ctx, fs.storage, fs.filePath); err != nil { log.Error().Err(err).Msg("failed to save DB") } } func NewFileStorage(ctx context.Context, fp *FileParams) (col.Collector, error) { fs := &FileStorage{ + wg: &sync.WaitGroup{}, filePath: fp.FileStoragePath, storage: NewMemStorage(), SyncRecord: fp.StoreInterval == 0, } if fp.RestoreOnStart { - fs.mutex.Lock() - err := database.LoadFromDB(ctx, fs.storage, fp.FileStoragePath) - fs.mutex.Unlock() + err := files.LoadFromDB(ctx, fs.storage, fp.FileStoragePath) if err != nil && !errors.Is(err, os.ErrNotExist) { return nil, fmt.Errorf("LoadFromDB error %w", err) } } - if fp.StoreInterval == 0 { - fs.SyncRecord = true - } - if fp.StoreInterval > 0 { + fs.wg.Add(1) + go func() { + defer fs.wg.Done() ticker := time.NewTicker(time.Duration(fp.StoreInterval) * time.Second) defer ticker.Stop() @@ -72,22 +71,24 @@ func NewFileStorage(ctx context.Context, fp *FileParams) (col.Collector, error) } } }() + + fs.wg.Wait() } return fs, nil } func (fs *FileStorage) UpdateMetric(ctx context.Context, mType, mName string, mValue any) error { - fs.mutex.Lock() - defer fs.mutex.Unlock() - if err := fs.storage.UpdateMetric(ctx, mType, mName, mValue); err != nil { log.Error().Err(err).Msg("failed update metric from file storage") return fmt.Errorf("failed update metric from file storage %w", err) } if fs.SyncRecord { - if err := database.SaveToDB(ctx, fs.storage, fs.filePath); err != nil { + fs.mutex.Lock() + defer fs.mutex.Unlock() + + if err := files.SaveToDB(ctx, fs.storage, fs.filePath); err != nil { log.Error().Err(err).Msg("failed save storage") return fmt.Errorf("failed save storage %w", err) } @@ -124,7 +125,7 @@ func (fs *FileStorage) GetAllMetrics(ctx context.Context) []mtr.Metric { return fs.storage.GetAllMetrics(ctx) } -func (fs *FileStorage) Ping(ctx context.Context) error { +func (fs *FileStorage) Ping(_ context.Context) error { return nil } diff --git a/internal/storage/postgres.go b/internal/storage/postgres.go index 6b586d6..3f4991a 100644 --- a/internal/storage/postgres.go +++ b/internal/storage/postgres.go @@ -5,7 +5,6 @@ import ( "database/sql" "errors" "fmt" - "sync" col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" @@ -16,8 +15,7 @@ import ( ) type Database struct { - mutex sync.RWMutex - DB *sql.DB + DB *sql.DB } func NewDatabase(ctx context.Context, dataBaseDSN string) (col.Collector, error) { @@ -46,12 +44,11 @@ func NewDatabase(ctx context.Context, dataBaseDSN string) (col.Collector, error) } return &Database{ - mutex: sync.RWMutex{}, - DB: db, + DB: db, }, nil } -func (db *Database) getMetric(ctx context.Context, mType, mName string) (any, error) { +func (db *Database) GetMetric(ctx context.Context, mType, mName string) (any, error) { var ( id string Type string @@ -95,17 +92,7 @@ func (db *Database) getMetric(ctx context.Context, mType, mName string) (any, er } } -func (db *Database) GetMetric(ctx context.Context, mType, mName string) (any, error) { - db.mutex.RLock() - defer db.mutex.RUnlock() - - return db.getMetric(ctx, mType, mName) -} - func (db *Database) GetAllMetrics(ctx context.Context) []mtr.Metric { - db.mutex.RLock() - defer db.mutex.RUnlock() - metrics := make(serialize.MetricsList, 0) rows, err := db.DB.QueryContext(ctx, @@ -173,10 +160,7 @@ func (db *Database) GetAllMetrics(ctx context.Context) []mtr.Metric { } func (db *Database) UpdateMetric(ctx context.Context, mType, mName string, mValue any) error { - db.mutex.Lock() - defer db.mutex.Unlock() - - oldValue, err := db.getMetric(ctx, mType, mName) + oldValue, err := db.GetMetric(ctx, mType, mName) var m mtr.Metric @@ -262,13 +246,10 @@ func (db *Database) UpdateMetric(ctx context.Context, mType, mName string, mValu } func (db *Database) Ping(ctx context.Context) error { - return db.DB.PingContext(ctx) + return fmt.Errorf("failed ping database: %w", db.DB.PingContext(ctx)) } func (db *Database) Close() error { - db.mutex.Lock() - defer db.mutex.Unlock() - if db.DB != nil { if err := db.DB.Close(); err != nil { log.Error().Err(err).Msg("failed to close database") diff --git a/pkg/data-base/file-storage.go b/pkg/files/file-storage.go similarity index 100% rename from pkg/data-base/file-storage.go rename to pkg/files/file-storage.go From 65a2e9549ba4bdaab5958750fa38333080cdda33 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Mon, 21 Jul 2025 00:18:12 +0300 Subject: [PATCH 107/126] fix files --- internal/config/server-config.go | 2 +- internal/storage/file.go | 162 +++++++++++++++++++++++++++---- 2 files changed, 145 insertions(+), 19 deletions(-) diff --git a/internal/config/server-config.go b/internal/config/server-config.go index facbb83..4c78d2a 100644 --- a/internal/config/server-config.go +++ b/internal/config/server-config.go @@ -12,7 +12,7 @@ import ( const ( DefaultEndpoint = "localhost:8080" DefaultStoreInterval = 300 - DefaultFileStoragePath = "/tmp/metrics-db.json" + DefaultFileStoragePath = "" DefaultRestoreOnStart = true DefaultDataBaseDSN = "" ) diff --git a/internal/storage/file.go b/internal/storage/file.go index aad3368..e83e9e5 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -1,3 +1,138 @@ +// package storage +// +// import ( +// "context" +// "errors" +// "fmt" +// "os" +// "sync" +// "time" +// +// col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" +// mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" +// files "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/files" +// "github.com/rs/zerolog/log" +// ) +// +// type FileStorage struct { +// mutex sync.RWMutex +// wg *sync.WaitGroup +// filePath string +// storage col.Collector +// SyncRecord bool +// } +// +// type FileParams struct { +// FileStoragePath string +// RestoreOnStart bool +// StoreInterval int +// } +// +// func (fs *FileStorage) save(ctx context.Context) { +// fs.mutex.Lock() +// defer fs.mutex.Unlock() +// +// if err := files.SaveToDB(ctx, fs.storage, fs.filePath); err != nil { +// log.Error().Err(err).Msg("failed to save DB") +// } +// } +// +// func NewFileStorage(ctx context.Context, fp *FileParams) (col.Collector, error) { +// fs := &FileStorage{ +// wg: &sync.WaitGroup{}, +// filePath: fp.FileStoragePath, +// storage: NewMemStorage(), +// SyncRecord: fp.StoreInterval == 0, +// } +// if fp.RestoreOnStart { +// err := files.LoadFromDB(ctx, fs.storage, fp.FileStoragePath) +// +// if err != nil && !errors.Is(err, os.ErrNotExist) { +// return nil, fmt.Errorf("LoadFromDB error %w", err) +// } +// } +// +// if fp.StoreInterval > 0 { +// fs.wg.Add(1) +// +// go func() { +// defer fs.wg.Done() +// ticker := time.NewTicker(time.Duration(fp.StoreInterval) * time.Second) +// defer ticker.Stop() +// +// for { +// select { +// case <-ticker.C: +// fs.save(ctx) +// case <-ctx.Done(): +// log.Info().Msg("Shutting down server, saving metrics") +// fs.save(ctx) +// return +// } +// } +// }() +// +// fs.wg.Wait() +// } +// +// return fs, nil +// } +// +// func (fs *FileStorage) UpdateMetric(ctx context.Context, mType, mName string, mValue any) error { +// if err := fs.storage.UpdateMetric(ctx, mType, mName, mValue); err != nil { +// log.Error().Err(err).Msg("failed update metric from file storage") +// return fmt.Errorf("failed update metric from file storage %w", err) +// } +// +// if fs.SyncRecord { +// fs.mutex.Lock() +// defer fs.mutex.Unlock() +// +// if err := files.SaveToDB(ctx, fs.storage, fs.filePath); err != nil { +// log.Error().Err(err).Msg("failed save storage") +// return fmt.Errorf("failed save storage %w", err) +// } +// } +// +// log.Info(). +// Str("type", mType). +// Str("name", mName). +// Msg("Metric updated successfully") +// +// return nil +// } +// +// func (fs *FileStorage) GetMetric(ctx context.Context, mType, mName string) (any, error) { +// fs.mutex.RLock() +// defer fs.mutex.RUnlock() +// +// val, err := fs.storage.GetMetric(ctx, mType, mName) +// if err != nil { +// log.Error(). +// Str("type", mType). +// Str("name", mName). +// Msg("can't get valid metric") +// return nil, fmt.Errorf("can't get valid metric: %v", err) +// } +// +// return val, nil +// } +// +// func (fs *FileStorage) GetAllMetrics(ctx context.Context) []mtr.Metric { +// fs.mutex.RLock() +// defer fs.mutex.RUnlock() +// +// return fs.storage.GetAllMetrics(ctx) +// } +// +// func (fs *FileStorage) Ping(_ context.Context) error { +// return nil +// } +// +// func (fs *FileStorage) Close() error { +// return nil +// } + package storage import ( @@ -10,13 +145,13 @@ import ( col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" - files "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/files" + database "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/files" "github.com/rs/zerolog/log" ) type FileStorage struct { mutex sync.RWMutex - wg *sync.WaitGroup + wg sync.WaitGroup filePath string storage col.Collector SyncRecord bool @@ -32,20 +167,20 @@ func (fs *FileStorage) save(ctx context.Context) { fs.mutex.Lock() defer fs.mutex.Unlock() - if err := files.SaveToDB(ctx, fs.storage, fs.filePath); err != nil { + if err := database.SaveToDB(ctx, fs.storage, fs.filePath); err != nil { log.Error().Err(err).Msg("failed to save DB") } } func NewFileStorage(ctx context.Context, fp *FileParams) (col.Collector, error) { fs := &FileStorage{ - wg: &sync.WaitGroup{}, + wg: sync.WaitGroup{}, filePath: fp.FileStoragePath, storage: NewMemStorage(), SyncRecord: fp.StoreInterval == 0, } if fp.RestoreOnStart { - err := files.LoadFromDB(ctx, fs.storage, fp.FileStoragePath) + err := database.LoadFromDB(ctx, fs.storage, fp.FileStoragePath) if err != nil && !errors.Is(err, os.ErrNotExist) { return nil, fmt.Errorf("LoadFromDB error %w", err) @@ -57,6 +192,7 @@ func NewFileStorage(ctx context.Context, fp *FileParams) (col.Collector, error) go func() { defer fs.wg.Done() + ticker := time.NewTicker(time.Duration(fp.StoreInterval) * time.Second) defer ticker.Stop() @@ -71,8 +207,6 @@ func NewFileStorage(ctx context.Context, fp *FileParams) (col.Collector, error) } } }() - - fs.wg.Wait() } return fs, nil @@ -85,10 +219,7 @@ func (fs *FileStorage) UpdateMetric(ctx context.Context, mType, mName string, mV } if fs.SyncRecord { - fs.mutex.Lock() - defer fs.mutex.Unlock() - - if err := files.SaveToDB(ctx, fs.storage, fs.filePath); err != nil { + if err := database.SaveToDB(ctx, fs.storage, fs.filePath); err != nil { log.Error().Err(err).Msg("failed save storage") return fmt.Errorf("failed save storage %w", err) } @@ -103,9 +234,6 @@ func (fs *FileStorage) UpdateMetric(ctx context.Context, mType, mName string, mV } func (fs *FileStorage) GetMetric(ctx context.Context, mType, mName string) (any, error) { - fs.mutex.RLock() - defer fs.mutex.RUnlock() - val, err := fs.storage.GetMetric(ctx, mType, mName) if err != nil { log.Error(). @@ -119,16 +247,14 @@ func (fs *FileStorage) GetMetric(ctx context.Context, mType, mName string) (any, } func (fs *FileStorage) GetAllMetrics(ctx context.Context) []mtr.Metric { - fs.mutex.RLock() - defer fs.mutex.RUnlock() - return fs.storage.GetAllMetrics(ctx) } -func (fs *FileStorage) Ping(_ context.Context) error { +func (fs *FileStorage) Ping(ctx context.Context) error { return nil } func (fs *FileStorage) Close() error { + fs.wg.Wait() return nil } From d735b5d9bfdfe15375ced2b1e923af6923fb347c Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Mon, 21 Jul 2025 00:34:43 +0300 Subject: [PATCH 108/126] fix psql --- internal/storage/file.go | 135 ----------------------------------- internal/storage/postgres.go | 5 +- 2 files changed, 4 insertions(+), 136 deletions(-) diff --git a/internal/storage/file.go b/internal/storage/file.go index e83e9e5..1577683 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -1,138 +1,3 @@ -// package storage -// -// import ( -// "context" -// "errors" -// "fmt" -// "os" -// "sync" -// "time" -// -// col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" -// mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" -// files "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/files" -// "github.com/rs/zerolog/log" -// ) -// -// type FileStorage struct { -// mutex sync.RWMutex -// wg *sync.WaitGroup -// filePath string -// storage col.Collector -// SyncRecord bool -// } -// -// type FileParams struct { -// FileStoragePath string -// RestoreOnStart bool -// StoreInterval int -// } -// -// func (fs *FileStorage) save(ctx context.Context) { -// fs.mutex.Lock() -// defer fs.mutex.Unlock() -// -// if err := files.SaveToDB(ctx, fs.storage, fs.filePath); err != nil { -// log.Error().Err(err).Msg("failed to save DB") -// } -// } -// -// func NewFileStorage(ctx context.Context, fp *FileParams) (col.Collector, error) { -// fs := &FileStorage{ -// wg: &sync.WaitGroup{}, -// filePath: fp.FileStoragePath, -// storage: NewMemStorage(), -// SyncRecord: fp.StoreInterval == 0, -// } -// if fp.RestoreOnStart { -// err := files.LoadFromDB(ctx, fs.storage, fp.FileStoragePath) -// -// if err != nil && !errors.Is(err, os.ErrNotExist) { -// return nil, fmt.Errorf("LoadFromDB error %w", err) -// } -// } -// -// if fp.StoreInterval > 0 { -// fs.wg.Add(1) -// -// go func() { -// defer fs.wg.Done() -// ticker := time.NewTicker(time.Duration(fp.StoreInterval) * time.Second) -// defer ticker.Stop() -// -// for { -// select { -// case <-ticker.C: -// fs.save(ctx) -// case <-ctx.Done(): -// log.Info().Msg("Shutting down server, saving metrics") -// fs.save(ctx) -// return -// } -// } -// }() -// -// fs.wg.Wait() -// } -// -// return fs, nil -// } -// -// func (fs *FileStorage) UpdateMetric(ctx context.Context, mType, mName string, mValue any) error { -// if err := fs.storage.UpdateMetric(ctx, mType, mName, mValue); err != nil { -// log.Error().Err(err).Msg("failed update metric from file storage") -// return fmt.Errorf("failed update metric from file storage %w", err) -// } -// -// if fs.SyncRecord { -// fs.mutex.Lock() -// defer fs.mutex.Unlock() -// -// if err := files.SaveToDB(ctx, fs.storage, fs.filePath); err != nil { -// log.Error().Err(err).Msg("failed save storage") -// return fmt.Errorf("failed save storage %w", err) -// } -// } -// -// log.Info(). -// Str("type", mType). -// Str("name", mName). -// Msg("Metric updated successfully") -// -// return nil -// } -// -// func (fs *FileStorage) GetMetric(ctx context.Context, mType, mName string) (any, error) { -// fs.mutex.RLock() -// defer fs.mutex.RUnlock() -// -// val, err := fs.storage.GetMetric(ctx, mType, mName) -// if err != nil { -// log.Error(). -// Str("type", mType). -// Str("name", mName). -// Msg("can't get valid metric") -// return nil, fmt.Errorf("can't get valid metric: %v", err) -// } -// -// return val, nil -// } -// -// func (fs *FileStorage) GetAllMetrics(ctx context.Context) []mtr.Metric { -// fs.mutex.RLock() -// defer fs.mutex.RUnlock() -// -// return fs.storage.GetAllMetrics(ctx) -// } -// -// func (fs *FileStorage) Ping(_ context.Context) error { -// return nil -// } -// -// func (fs *FileStorage) Close() error { -// return nil -// } - package storage import ( diff --git a/internal/storage/postgres.go b/internal/storage/postgres.go index 3f4991a..290976f 100644 --- a/internal/storage/postgres.go +++ b/internal/storage/postgres.go @@ -246,7 +246,10 @@ func (db *Database) UpdateMetric(ctx context.Context, mType, mName string, mValu } func (db *Database) Ping(ctx context.Context) error { - return fmt.Errorf("failed ping database: %w", db.DB.PingContext(ctx)) + if err := db.DB.PingContext(ctx); err != nil { + return fmt.Errorf("failed ping database: %w", err) + } + return nil } func (db *Database) Close() error { From daf73bafba54c90b139753307726894b50bf6f3c Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Mon, 21 Jul 2025 04:41:58 +0300 Subject: [PATCH 109/126] some correct --- cmd/server/cfg_server.go | 3 +- internal/handlers/agent/agent.go | 3 +- internal/handlers/server/server.go | 58 +++++++++++-------------- internal/handlers/server/server_test.go | 5 ++- internal/router/router.go | 18 ++++---- internal/storage/postgres.go | 48 ++++++++++++-------- pkg/converter/converter.go | 20 +++++++++ 7 files changed, 90 insertions(+), 65 deletions(-) diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 9df5ae7..1709dc2 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -14,6 +14,7 @@ import ( colcfg "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector/config" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/router" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -116,7 +117,7 @@ func startServer(ctx context.Context, opts *config.Options) error { } }() - r := router.NewRouter(collector, opts) + r := router.NewRouter(server.NewServer(collector, opts)) srv := &http.Server{ Addr: opts.EndPointAddr, diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 18d882c..24df116 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -13,6 +13,7 @@ import ( "github.com/go-resty/resty/v2" "github.com/mailru/easyjson" + col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/storage" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" @@ -20,7 +21,7 @@ import ( serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" ) -func UpdateAllMetrics(ctx context.Context, storage *storage.MemStorage) { +func UpdateAllMetrics(ctx context.Context, storage col.Collector) { var memStats runtime.MemStats runtime.ReadMemStats(&memStats) diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index a359052..96f2a42 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -13,36 +13,30 @@ import ( "github.com/mailru/easyjson" col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/converter" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" ) -func ConvertByType(mType, mValue string) (any, error) { - switch mType { - case mtr.GaugeType: - if val, err := strconv.ParseFloat(mValue, 64); err != nil { - return nil, fmt.Errorf("convert gauge value %s: %w", mValue, err) - } else { - return val, nil - } - case mtr.CounterType: - if val, err := strconv.ParseInt(mValue, 10, 64); err != nil { - return nil, fmt.Errorf("convert counter value %s: %w", mValue, err) - } else { - return val, nil - } - default: - return nil, fmt.Errorf("unknown metric type: %s", mType) +type Server struct { + Storage col.Collector + Opts *config.Options +} + +func NewServer(col col.Collector, opts *config.Options) *Server { + return &Server{ + Storage: col, + Opts: opts, } } -func GetMetric(storage col.Collector) http.HandlerFunc { +func (srv *Server) GetMetric() http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { mType := chi.URLParam(req, "mType") mName := chi.URLParam(req, "mName") - value, err := storage.GetMetric(req.Context(), mType, mName) + value, err := srv.Storage.GetMetric(req.Context(), mType, mName) if err != nil { http.Error(res, fmt.Sprintf("Metric %s was not found", mName), http.StatusNotFound) return @@ -70,9 +64,9 @@ func GetMetric(storage col.Collector) http.HandlerFunc { } } -func GetAllMetrics(storage col.Collector) http.HandlerFunc { +func (srv *Server) GetAllMetrics() http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { - allMetrics := storage.GetAllMetrics(req.Context()) + allMetrics := srv.Storage.GetAllMetrics(req.Context()) var metricsToTable []mtr.MetricTable @@ -153,7 +147,7 @@ func GetAllMetrics(storage col.Collector) http.HandlerFunc { } } -func UpdateMetric(storage col.Collector) http.HandlerFunc { +func (srv *Server) UpdateMetric() http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { mType := chi.URLParam(req, "mType") mName := chi.URLParam(req, "mName") @@ -168,7 +162,7 @@ func UpdateMetric(storage col.Collector) http.HandlerFunc { return } - val, err := ConvertByType(mType, mValue) + val, err := converter.ConvertByType(mType, mValue) if err != nil { log.Error(). Err(err). @@ -180,7 +174,7 @@ func UpdateMetric(storage col.Collector) http.HandlerFunc { return } - if err := storage.UpdateMetric(req.Context(), mType, mName, val); err != nil { + if err := srv.Storage.UpdateMetric(req.Context(), mType, mName, val); err != nil { http.Error(res, err.Error(), http.StatusBadRequest) return } @@ -216,7 +210,7 @@ func FillMetricValueFromStorage(ctx context.Context, storage col.Collector, metr return true } -func GetMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { +func (srv *Server) GetMetricsHandlerJSON() http.HandlerFunc { return func(resp http.ResponseWriter, req *http.Request) { var metric serialize.Metric @@ -231,7 +225,7 @@ func GetMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { return } - value, err := storage.GetMetric(req.Context(), metric.MType, metric.ID) + value, err := srv.Storage.GetMetric(req.Context(), metric.MType, metric.ID) if err != nil { log.Error().Err(err).Msg("can't get valid metric") http.Error(resp, "can't get valid metric", http.StatusNotFound) @@ -253,7 +247,7 @@ func GetMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { } } -func UpdateMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { +func (srv *Server) UpdateMetricsHandlerJSON() http.HandlerFunc { return func(resp http.ResponseWriter, req *http.Request) { var reader io.Reader if req.Header.Get("Content-Encoding") == "gzip" { @@ -292,11 +286,11 @@ func UpdateMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { return } - if err := storage.UpdateMetric(req.Context(), metric.MType, metric.ID, value); err != nil { + if err := srv.Storage.UpdateMetric(req.Context(), metric.MType, metric.ID, value); err != nil { http.Error(resp, fmt.Sprintf("invalid update metric %s: %v", metric.ID, err), http.StatusBadRequest) } - if !FillMetricValueFromStorage(req.Context(), storage, &metric) { + if !FillMetricValueFromStorage(req.Context(), srv.Storage, &metric) { http.Error(resp, fmt.Sprintf("metric %s not found", metric.ID), http.StatusNotFound) return } @@ -309,7 +303,7 @@ func UpdateMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { } } -func UpdatesMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { +func (srv *Server) UpdatesMetricsHandlerJSON() http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { var jsonMetrics serialize.MetricsList @@ -331,7 +325,7 @@ func UpdatesMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { } for _, metric := range metrics { - if err := storage.UpdateMetric(req.Context(), metric.Type(), metric.Name(), metric.Value()); err != nil { + if err := srv.Storage.UpdateMetric(req.Context(), metric.Type(), metric.Name(), metric.Value()); err != nil { http.Error(w, fmt.Sprintf("failed update metric %v", err), http.StatusInternalServerError) return } @@ -341,9 +335,9 @@ func UpdatesMetricsHandlerJSON(storage col.Collector) http.HandlerFunc { } } -func PingHandler(col col.Collector) http.HandlerFunc { +func (srv *Server) PingHandler() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - if err := col.Ping(r.Context()); err != nil { + if err := srv.Storage.Ping(r.Context()); err != nil { log.Error().Err(err).Msg("failed ping") http.Error(w, "can't ping DB", http.StatusInternalServerError) return diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index 99a4189..b63592a 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -11,6 +11,7 @@ import ( _ "github.com/jackc/pgx/v5/stdlib" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/router" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/storage" @@ -41,7 +42,7 @@ func TestUpdateMetric(t *testing.T) { }() storage := storage.NewMemStorage() - router := router.NewRouter(storage, opts) + router := router.NewRouter(server.NewServer(storage, opts)) tests := []struct { name string @@ -142,7 +143,7 @@ func TestGetMetric(t *testing.T) { log.Error().Msgf("Failed to update metric requests_total: %v", err) } - router := router.NewRouter(storage, opts) + router := router.NewRouter(server.NewServer(storage, opts)) tests := []struct { name string diff --git a/internal/router/router.go b/internal/router/router.go index 7aeaeb1..34166f4 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -5,36 +5,34 @@ import ( "github.com/go-chi/chi/v5" - col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" ) -func NewRouter(storage col.Collector, opts *config.Options) http.Handler { +func NewRouter(srv *server.Server) http.Handler { r := chi.NewRouter() r.Use(server.WithLogging) r.Use(server.WithGzipCompress) r.Route("/", func(r chi.Router) { - r.Get("/", server.GetAllMetrics(storage)) + r.Get("/", srv.GetAllMetrics()) r.Route("/update", func(r chi.Router) { - r.Post("/", server.UpdateMetricsHandlerJSON(storage)) - r.Post("/{mType}/{mName}/{mValue}", server.UpdateMetric(storage)) + r.Post("/", srv.UpdateMetricsHandlerJSON()) + r.Post("/{mType}/{mName}/{mValue}", srv.UpdateMetric()) }) r.Route("/value", func(r chi.Router) { - r.Post("/", server.GetMetricsHandlerJSON(storage)) - r.Get("/{mType}/{mName}", server.GetMetric(storage)) + r.Post("/", srv.GetMetricsHandlerJSON()) + r.Get("/{mType}/{mName}", srv.GetMetric()) }) r.Route("/ping", func(r chi.Router) { - r.Get("/", server.PingHandler(storage)) + r.Get("/", srv.PingHandler()) }) r.Route("/updates", func(r chi.Router) { - r.Post("/", server.UpdatesMetricsHandlerJSON(storage)) + r.Post("/", srv.UpdatesMetricsHandlerJSON()) }) }) diff --git a/internal/storage/postgres.go b/internal/storage/postgres.go index 290976f..61c658f 100644 --- a/internal/storage/postgres.go +++ b/internal/storage/postgres.go @@ -92,6 +92,30 @@ func (db *Database) GetMetric(ctx context.Context, mType, mName string) (any, er } } +func fillMetricValues(metric *serialize.Metric, delta sql.NullInt64, value sql.NullFloat64) error { + switch { + case value.Valid: + if metric.MType != mtr.GaugeType { + return mtr.ErrInvalidMetricsType + } + + metric.Value = &value.Float64 + + case delta.Valid: + if metric.MType != mtr.CounterType { + return mtr.ErrInvalidMetricsType + } + + metric.Delta = &delta.Int64 + + default: + log.Error().Msg("not valid value") + return nil + } + + return nil +} + func (db *Database) GetAllMetrics(ctx context.Context) []mtr.Metric { metrics := make(serialize.MetricsList, 0) @@ -123,22 +147,8 @@ func (db *Database) GetAllMetrics(ctx context.Context) []mtr.Metric { return nil } - switch { - case value.Valid: - if metric.MType != mtr.GaugeType { - return nil - } - - metric.Value = &value.Float64 - - case delta.Valid: - if metric.MType != mtr.CounterType { - return nil - } - - metric.Delta = &delta.Int64 - default: - log.Error().Msg("not valid value") + if err := fillMetricValues(&metric, delta, value); err != nil { + log.Error().Err(err).Msg("can't set values") return nil } @@ -247,9 +257,9 @@ func (db *Database) UpdateMetric(ctx context.Context, mType, mName string, mValu func (db *Database) Ping(ctx context.Context) error { if err := db.DB.PingContext(ctx); err != nil { - return fmt.Errorf("failed ping database: %w", err) - } - return nil + return fmt.Errorf("failed ping database: %w", err) + } + return nil } func (db *Database) Close() error { diff --git a/pkg/converter/converter.go b/pkg/converter/converter.go index 8a8491d..179996f 100644 --- a/pkg/converter/converter.go +++ b/pkg/converter/converter.go @@ -2,12 +2,32 @@ package converter import ( "fmt" + "strconv" mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" "github.com/rs/zerolog/log" ) +func ConvertByType(mType, mValue string) (any, error) { + switch mType { + case mtr.GaugeType: + if val, err := strconv.ParseFloat(mValue, 64); err != nil { + return nil, fmt.Errorf("convert gauge value %s: %w", mValue, err) + } else { + return val, nil + } + case mtr.CounterType: + if val, err := strconv.ParseInt(mValue, 10, 64); err != nil { + return nil, fmt.Errorf("convert counter value %s: %w", mValue, err) + } else { + return val, nil + } + default: + return nil, fmt.Errorf("unknown metric type: %s", mType) + } +} + func ConvertMetrics(src serialize.MetricsList) ([]mtr.Metric, error) { converted := make([]mtr.Metric, 0, len(src)) From 08b08fbc77cebf9bbc45d0616decbf277d712a04 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Mon, 21 Jul 2025 22:51:20 +0300 Subject: [PATCH 110/126] rename dirs --- cmd/agent/cfg_agent.go | 4 +- go.mod | 4 +- go.sum | 2 + internal/collector/collerctor.go | 4 +- internal/collector/config/config.go | 8 +-- internal/handlers/agent/agent.go | 14 ++--- internal/handlers/server/server.go | 10 ++-- internal/handlers/server/server_test.go | 12 ++-- internal/{metrics => models}/counter.go | 2 +- internal/{metrics => models}/counter_test.go | 2 +- internal/{metrics => models}/gauge.go | 2 +- internal/{metrics => models}/gauge_test.go | 2 +- internal/{metrics => models}/metrics.go | 2 +- internal/{storage => repository}/file.go | 14 ++--- internal/{storage => repository}/memory.go | 32 +++++------ internal/{storage => repository}/postgres.go | 48 ++++++++-------- .../{storage => repository}/storage_test.go | 54 +++++++++--------- pkg/converter/converter.go | 22 ++++---- pkg/files/file-storage.go | 12 ++-- pkg/runtime-stats/runtime-stats.go | 56 +++++++++---------- pkg/serialization/serialization.go | 22 ++++---- 21 files changed, 165 insertions(+), 163 deletions(-) rename internal/{metrics => models}/counter.go (97%) rename internal/{metrics => models}/counter_test.go (98%) rename internal/{metrics => models}/gauge.go (97%) rename internal/{metrics => models}/gauge_test.go (98%) rename internal/{metrics => models}/metrics.go (96%) rename internal/{storage => repository}/file.go (83%) rename internal/{storage => repository}/memory.go (63%) rename internal/{storage => repository}/postgres.go (85%) rename internal/{storage => repository}/storage_test.go (80%) diff --git a/cmd/agent/cfg_agent.go b/cmd/agent/cfg_agent.go index c696ebb..a474b70 100644 --- a/cmd/agent/cfg_agent.go +++ b/cmd/agent/cfg_agent.go @@ -14,7 +14,7 @@ import ( "github.com/spf13/cobra" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/agent" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/storage" + repo "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/repository" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -108,7 +108,7 @@ func init() { } func startAgent(ctx context.Context) { - storage := storage.NewMemStorage() + storage := repo.NewMemStorage() client := resty.New(). SetTimeout(5 * time.Second). diff --git a/go.mod b/go.mod index 18e2f57..79ed0b2 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,8 @@ require ( github.com/caarlos0/env/v6 v6.10.1 github.com/go-chi/chi/v5 v5.2.2 github.com/go-resty/resty/v2 v2.16.5 + github.com/jackc/pgconn v1.14.3 + github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 github.com/jackc/pgx/v5 v5.7.5 github.com/mailru/easyjson v0.9.0 github.com/rs/zerolog v1.34.0 @@ -15,8 +17,6 @@ require ( require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect - github.com/jackc/pgconn v1.14.3 // indirect - github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgproto3/v2 v2.3.3 // indirect diff --git a/go.sum b/go.sum index adca236..c19ceca 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,8 @@ github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 h1:Dj0L5fhJ9F82ZJy github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= diff --git a/internal/collector/collerctor.go b/internal/collector/collerctor.go index 99f5e0e..bc8043a 100644 --- a/internal/collector/collerctor.go +++ b/internal/collector/collerctor.go @@ -3,12 +3,12 @@ package collector import ( "context" - mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" ) type Collector interface { GetMetric(ctx context.Context, mType, mName string) (any, error) - GetAllMetrics(ctx context.Context) []mtr.Metric + GetAllMetrics(ctx context.Context) []models.Metric UpdateMetric(ctx context.Context, mType, mName string, mValue any) error Ping(ctx context.Context) error diff --git a/internal/collector/config/config.go b/internal/collector/config/config.go index a6b112d..fd1ff72 100644 --- a/internal/collector/config/config.go +++ b/internal/collector/config/config.go @@ -6,7 +6,7 @@ import ( col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/storage" + repo "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/repository" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -23,13 +23,13 @@ func NewCollector(params *Params) (col.Collector, error) { switch { case params.Opts.DataBaseDSN != "": - collector, err = storage.NewDatabase(params.Ctx, params.Opts.DataBaseDSN) + collector, err = repo.NewDatabase(params.Ctx, params.Opts.DataBaseDSN) if err != nil { return nil, fmt.Errorf("DB connection failed: %w", err) } case params.Opts.FileStoragePath != "": - collector, err = storage.NewFileStorage(params.Ctx, &storage.FileParams{ + collector, err = repo.NewFileStorage(params.Ctx, &repo.FileParams{ FileStoragePath: params.Opts.FileStoragePath, RestoreOnStart: params.Opts.RestoreOnStart, StoreInterval: params.Opts.StoreInterval}) @@ -37,7 +37,7 @@ func NewCollector(params *Params) (col.Collector, error) { log.Debug().Msg("chose file storage") default: - collector = storage.NewMemStorage() + collector = repo.NewMemStorage() } if err != nil { diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 24df116..add40af 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -14,8 +14,8 @@ import ( "github.com/mailru/easyjson" col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" - mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/storage" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" + repo "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/repository" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" rt "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/runtime-stats" serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" @@ -39,16 +39,16 @@ func UpdateAllMetrics(ctx context.Context, storage col.Collector) { log.Debug().Msgf("update metric %s", stat.Name) } - if err := storage.UpdateMetric(ctx, mtr.CounterType, "PollCount", int64(1)); err != nil { + if err := storage.UpdateMetric(ctx, models.CounterType, "PollCount", int64(1)); err != nil { log.Error().Msgf("Failed to update PollCount metric: %v", err) } - if err := storage.UpdateMetric(ctx, mtr.GaugeType, "RandomValue", rand.Float64()); err != nil { + if err := storage.UpdateMetric(ctx, models.GaugeType, "RandomValue", rand.Float64()); err != nil { log.Error().Msgf("Failed to update RandomValue metric: %v", err) } } -func SendAllMetrics(ctx context.Context, client *resty.Client, storage *storage.MemStorage) { +func SendAllMetrics(ctx context.Context, client *resty.Client, storage *repo.MemStorage) { allMetrics := storage.GetAllMetrics(ctx) for _, metric := range allMetrics { @@ -61,7 +61,7 @@ func SendAllMetrics(ctx context.Context, client *resty.Client, storage *storage. } switch mType { - case mtr.GaugeType: + case models.GaugeType: val, ok := metric.Value().(float64) if !ok { log.Error().Str("metric_name", mName).Str("metric_type", mType). @@ -72,7 +72,7 @@ func SendAllMetrics(ctx context.Context, client *resty.Client, storage *storage. metricJSON.Value = &val sendMetric(client, &metricJSON) - case mtr.CounterType: + case models.CounterType: val, ok := metric.Value().(int64) if !ok { log.Error().Str("metric_name", mName).Str("metric_type", mType). diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index 96f2a42..ccc0b49 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -14,7 +14,7 @@ import ( col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" - mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + models "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/converter" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" @@ -68,7 +68,7 @@ func (srv *Server) GetAllMetrics() http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { allMetrics := srv.Storage.GetAllMetrics(req.Context()) - var metricsToTable []mtr.MetricTable + var metricsToTable []models.MetricTable for _, metric := range allMetrics { var valStr string @@ -76,7 +76,7 @@ func (srv *Server) GetAllMetrics() http.HandlerFunc { mType := metric.Type() switch mType { - case mtr.GaugeType: + case models.GaugeType: val, ok := metric.Value().(float64) if !ok { log.Error().Str("metric_name", mName).Str("metric_type", mType). @@ -85,7 +85,7 @@ func (srv *Server) GetAllMetrics() http.HandlerFunc { } valStr = strconv.FormatFloat(val, 'f', -1, 64) - case mtr.CounterType: + case models.CounterType: val, ok := metric.Value().(int64) if !ok { log.Error().Str("metric_name", mName).Str("metric_type", mType). @@ -95,7 +95,7 @@ func (srv *Server) GetAllMetrics() http.HandlerFunc { valStr = strconv.FormatInt(val, 10) } - metricsToTable = append(metricsToTable, mtr.MetricTable{ + metricsToTable = append(metricsToTable, models.MetricTable{ Name: mName, Type: mType, Value: valStr, diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index b63592a..7a49086 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -12,9 +12,9 @@ import ( "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" - mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/router" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/storage" + repo "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/repository" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -41,7 +41,7 @@ func TestUpdateMetric(t *testing.T) { } }() - storage := storage.NewMemStorage() + storage := repo.NewMemStorage() router := router.NewRouter(server.NewServer(storage, opts)) tests := []struct { @@ -133,13 +133,13 @@ func TestGetMetric(t *testing.T) { //FIXME - maybe need mocks ctx := context.Background() - storage := storage.NewMemStorage() + storage := repo.NewMemStorage() - if err := storage.UpdateMetric(ctx, mtr.GaugeType, "cpu_usage", 75.5); err != nil { + if err := storage.UpdateMetric(ctx, models.GaugeType, "cpu_usage", 75.5); err != nil { log.Error().Msgf("Failed to update metric cpu_usage: %v", err) } - if err := storage.UpdateMetric(ctx, mtr.CounterType, "requests_total", int64(100)); err != nil { + if err := storage.UpdateMetric(ctx, models.CounterType, "requests_total", int64(100)); err != nil { log.Error().Msgf("Failed to update metric requests_total: %v", err) } diff --git a/internal/metrics/counter.go b/internal/models/counter.go similarity index 97% rename from internal/metrics/counter.go rename to internal/models/counter.go index 8c1a546..92bff0e 100644 --- a/internal/metrics/counter.go +++ b/internal/models/counter.go @@ -1,4 +1,4 @@ -package metrics +package models type counter struct { name string diff --git a/internal/metrics/counter_test.go b/internal/models/counter_test.go similarity index 98% rename from internal/metrics/counter_test.go rename to internal/models/counter_test.go index 60536d0..c94dc7c 100644 --- a/internal/metrics/counter_test.go +++ b/internal/models/counter_test.go @@ -1,4 +1,4 @@ -package metrics +package models import "testing" diff --git a/internal/metrics/gauge.go b/internal/models/gauge.go similarity index 97% rename from internal/metrics/gauge.go rename to internal/models/gauge.go index 6349490..f916919 100644 --- a/internal/metrics/gauge.go +++ b/internal/models/gauge.go @@ -1,4 +1,4 @@ -package metrics +package models type gauge struct { name string diff --git a/internal/metrics/gauge_test.go b/internal/models/gauge_test.go similarity index 98% rename from internal/metrics/gauge_test.go rename to internal/models/gauge_test.go index 9b6dc19..a6f85bf 100644 --- a/internal/metrics/gauge_test.go +++ b/internal/models/gauge_test.go @@ -1,4 +1,4 @@ -package metrics +package models import "testing" diff --git a/internal/metrics/metrics.go b/internal/models/metrics.go similarity index 96% rename from internal/metrics/metrics.go rename to internal/models/metrics.go index 0778aa1..9c5f22c 100644 --- a/internal/metrics/metrics.go +++ b/internal/models/metrics.go @@ -1,4 +1,4 @@ -package metrics +package models import ( "errors" diff --git a/internal/storage/file.go b/internal/repository/file.go similarity index 83% rename from internal/storage/file.go rename to internal/repository/file.go index 1577683..08f6096 100644 --- a/internal/storage/file.go +++ b/internal/repository/file.go @@ -1,4 +1,4 @@ -package storage +package repository import ( "context" @@ -9,8 +9,8 @@ import ( "time" col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" - mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" - database "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/files" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/files" "github.com/rs/zerolog/log" ) @@ -32,7 +32,7 @@ func (fs *FileStorage) save(ctx context.Context) { fs.mutex.Lock() defer fs.mutex.Unlock() - if err := database.SaveToDB(ctx, fs.storage, fs.filePath); err != nil { + if err := files.SaveToDB(ctx, fs.storage, fs.filePath); err != nil { log.Error().Err(err).Msg("failed to save DB") } } @@ -45,7 +45,7 @@ func NewFileStorage(ctx context.Context, fp *FileParams) (col.Collector, error) SyncRecord: fp.StoreInterval == 0, } if fp.RestoreOnStart { - err := database.LoadFromDB(ctx, fs.storage, fp.FileStoragePath) + err := files.LoadFromDB(ctx, fs.storage, fp.FileStoragePath) if err != nil && !errors.Is(err, os.ErrNotExist) { return nil, fmt.Errorf("LoadFromDB error %w", err) @@ -84,7 +84,7 @@ func (fs *FileStorage) UpdateMetric(ctx context.Context, mType, mName string, mV } if fs.SyncRecord { - if err := database.SaveToDB(ctx, fs.storage, fs.filePath); err != nil { + if err := files.SaveToDB(ctx, fs.storage, fs.filePath); err != nil { log.Error().Err(err).Msg("failed save storage") return fmt.Errorf("failed save storage %w", err) } @@ -111,7 +111,7 @@ func (fs *FileStorage) GetMetric(ctx context.Context, mType, mName string) (any, return val, nil } -func (fs *FileStorage) GetAllMetrics(ctx context.Context) []mtr.Metric { +func (fs *FileStorage) GetAllMetrics(ctx context.Context) []models.Metric { return fs.storage.GetAllMetrics(ctx) } diff --git a/internal/storage/memory.go b/internal/repository/memory.go similarity index 63% rename from internal/storage/memory.go rename to internal/repository/memory.go index a4bbc34..0fdd92e 100644 --- a/internal/storage/memory.go +++ b/internal/repository/memory.go @@ -1,20 +1,20 @@ -package storage +package repository import ( "context" "sync" - mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" ) type MemStorage struct { mutex sync.RWMutex - storage map[string]map[string]mtr.Metric + storage map[string]map[string]models.Metric } func NewMemStorage() *MemStorage { return &MemStorage{ - storage: make(map[string]map[string]mtr.Metric), + storage: make(map[string]map[string]models.Metric), } } @@ -23,30 +23,30 @@ func (ms *MemStorage) UpdateMetric(_ context.Context, mType, mName string, mValu defer ms.mutex.Unlock() if _, ok := ms.storage[mType]; !ok { - ms.storage[mType] = make(map[string]mtr.Metric) + ms.storage[mType] = make(map[string]models.Metric) } if oldMetric, ok := ms.storage[mType][mName]; ok { return oldMetric.Update(mValue) } - var newMetric mtr.Metric + var newMetric models.Metric switch mType { - case mtr.GaugeType: + case models.GaugeType: value, ok := mValue.(float64) if !ok { - return mtr.ErrInvalidValueType + return models.ErrInvalidValueType } - newMetric = mtr.NewGauge(mName, value) - case mtr.CounterType: + newMetric = models.NewGauge(mName, value) + case models.CounterType: value, ok := mValue.(int64) if !ok { - return mtr.ErrInvalidValueType + return models.ErrInvalidValueType } - newMetric = mtr.NewCounter(mName, value) + newMetric = models.NewCounter(mName, value) default: - return mtr.ErrInvalidMetricsType + return models.ErrInvalidMetricsType } ms.storage[mType][mName] = newMetric @@ -60,13 +60,13 @@ func (ms *MemStorage) GetMetric(_ context.Context, mType, mName string) (any, er metric, ok := ms.storage[mType][mName] if !ok { - return nil, mtr.ErrMetricsNotFound + return nil, models.ErrMetricsNotFound } return metric.Value(), nil } -func (ms *MemStorage) GetAllMetrics(_ context.Context) []mtr.Metric { +func (ms *MemStorage) GetAllMetrics(_ context.Context) []models.Metric { ms.mutex.RLock() defer ms.mutex.RUnlock() @@ -74,7 +74,7 @@ func (ms *MemStorage) GetAllMetrics(_ context.Context) []mtr.Metric { for _, innerMap := range ms.storage { total += len(innerMap) } - result := make([]mtr.Metric, 0, total) + result := make([]models.Metric, 0, total) for _, innerMap := range ms.storage { for _, metric := range innerMap { diff --git a/internal/storage/postgres.go b/internal/repository/postgres.go similarity index 85% rename from internal/storage/postgres.go rename to internal/repository/postgres.go index 61c658f..2916200 100644 --- a/internal/storage/postgres.go +++ b/internal/repository/postgres.go @@ -1,4 +1,4 @@ -package storage +package repository import ( "context" @@ -7,7 +7,7 @@ import ( "fmt" col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" - mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/converter" errH "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/errors-handlers" serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" @@ -67,43 +67,43 @@ func (db *Database) GetMetric(ctx context.Context, mType, mName string) (any, er err := errH.WithRetry(getMtr, errH.IsPostgresRetriableError) if errors.Is(err, sql.ErrNoRows) { - return nil, mtr.ErrMetricsNotFound + return nil, models.ErrMetricsNotFound } else if err != nil { return nil, fmt.Errorf("row.Scan can't read: %w", err) } switch { case value.Valid: - if Type != mtr.GaugeType { - return nil, mtr.ErrInvalidMetricsType + if Type != models.GaugeType { + return nil, models.ErrInvalidMetricsType } return value.Float64, nil case delta.Valid: - if Type != mtr.CounterType { - return nil, mtr.ErrInvalidMetricsType + if Type != models.CounterType { + return nil, models.ErrInvalidMetricsType } return delta.Int64, nil default: log.Error().Msg("not valid value") - return nil, mtr.ErrInvalidValueType + return nil, models.ErrInvalidValueType } } func fillMetricValues(metric *serialize.Metric, delta sql.NullInt64, value sql.NullFloat64) error { switch { case value.Valid: - if metric.MType != mtr.GaugeType { - return mtr.ErrInvalidMetricsType + if metric.MType != models.GaugeType { + return models.ErrInvalidMetricsType } metric.Value = &value.Float64 case delta.Valid: - if metric.MType != mtr.CounterType { - return mtr.ErrInvalidMetricsType + if metric.MType != models.CounterType { + return models.ErrInvalidMetricsType } metric.Delta = &delta.Int64 @@ -116,7 +116,7 @@ func fillMetricValues(metric *serialize.Metric, delta sql.NullInt64, value sql.N return nil } -func (db *Database) GetAllMetrics(ctx context.Context) []mtr.Metric { +func (db *Database) GetAllMetrics(ctx context.Context) []models.Metric { metrics := make(serialize.MetricsList, 0) rows, err := db.DB.QueryContext(ctx, @@ -172,23 +172,23 @@ func (db *Database) GetAllMetrics(ctx context.Context) []mtr.Metric { func (db *Database) UpdateMetric(ctx context.Context, mType, mName string, mValue any) error { oldValue, err := db.GetMetric(ctx, mType, mName) - var m mtr.Metric + var m models.Metric switch newVal := mValue.(type) { case float64: - if mType != mtr.GaugeType { - return mtr.ErrInvalidMetricsType + if mType != models.GaugeType { + return models.ErrInvalidMetricsType } - m = mtr.NewGauge(mName, newVal) + m = models.NewGauge(mName, newVal) case int64: - if mType != mtr.CounterType { - return mtr.ErrInvalidMetricsType + if mType != models.CounterType { + return models.ErrInvalidMetricsType } - if errors.Is(err, mtr.ErrMetricsNotFound) { - m = mtr.NewCounter(mName, newVal) + if errors.Is(err, models.ErrMetricsNotFound) { + m = models.NewCounter(mName, newVal) } else { - m = mtr.NewCounter(mName, oldValue.(int64)) + m = models.NewCounter(mName, oldValue.(int64)) if err := m.Update(newVal); err != nil { return fmt.Errorf("failed update metric %v", err) } @@ -204,14 +204,14 @@ func (db *Database) UpdateMetric(ctx context.Context, mType, mName string, mValu } //TODO - make method maybe for Metrics type switch metric.MType { - case mtr.GaugeType: + case models.GaugeType: val, ok := m.Value().(float64) if !ok { return fmt.Errorf("expected float64 for gauge, got %T", m.Value()) } metric.Value = &val - case mtr.CounterType: + case models.CounterType: val, ok := m.Value().(int64) if !ok { return fmt.Errorf("expected int64 for counter, got %T", m.Value()) diff --git a/internal/storage/storage_test.go b/internal/repository/storage_test.go similarity index 80% rename from internal/storage/storage_test.go rename to internal/repository/storage_test.go index 43aa56a..5c5a367 100644 --- a/internal/storage/storage_test.go +++ b/internal/repository/storage_test.go @@ -1,18 +1,18 @@ -package storage +package repository import ( "context" "sync" "testing" - mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" ) func TestMemStorage_UpdateMetric(t *testing.T) { ctx := context.Background() type fields struct { - storage map[string]map[string]mtr.Metric + storage map[string]map[string]models.Metric } type args struct { mType string @@ -29,10 +29,10 @@ func TestMemStorage_UpdateMetric(t *testing.T) { { name: "Add new gauge", fields: fields{ - storage: make(map[string]map[string]mtr.Metric), + storage: make(map[string]map[string]models.Metric), }, args: args{ - mType: mtr.GaugeType, + mType: models.GaugeType, mName: "g1", mValue: 3.14, }, @@ -42,10 +42,10 @@ func TestMemStorage_UpdateMetric(t *testing.T) { { name: "Add new counter", fields: fields{ - storage: make(map[string]map[string]mtr.Metric), + storage: make(map[string]map[string]models.Metric), }, args: args{ - mType: mtr.CounterType, + mType: models.CounterType, mName: "c1", mValue: int64(10), }, @@ -55,10 +55,10 @@ func TestMemStorage_UpdateMetric(t *testing.T) { { name: "Invalid gauge value", fields: fields{ - storage: make(map[string]map[string]mtr.Metric), + storage: make(map[string]map[string]models.Metric), }, args: args{ - mType: mtr.GaugeType, + mType: models.GaugeType, mName: "g2", mValue: "not a float", }, @@ -67,14 +67,14 @@ func TestMemStorage_UpdateMetric(t *testing.T) { { name: "Update existing counter", fields: fields{ - storage: map[string]map[string]mtr.Metric{ - mtr.CounterType: { - "c2": mtr.NewCounter("c2", 5), + storage: map[string]map[string]models.Metric{ + models.CounterType: { + "c2": models.NewCounter("c2", 5), }, }, }, args: args{ - mType: mtr.CounterType, + mType: models.CounterType, mName: "c2", mValue: int64(7), }, @@ -84,7 +84,7 @@ func TestMemStorage_UpdateMetric(t *testing.T) { { name: "Unknown metric type", fields: fields{ - storage: make(map[string]map[string]mtr.Metric), + storage: make(map[string]map[string]models.Metric), }, args: args{ mType: "unknown", @@ -122,12 +122,12 @@ func TestMemStorage_UpdateMetric(t *testing.T) { func TestMemStorage_GetMetric(t *testing.T) { ctx := context.Background() - counter := mtr.NewCounter("requests", 42) - gauge := mtr.NewGauge("temperature", 36.6) + counter := models.NewCounter("requests", 42) + gauge := models.NewGauge("temperature", 36.6) tests := []struct { name string - fields map[string]map[string]mtr.Metric + fields map[string]map[string]models.Metric mType string mName string wantVal any @@ -135,37 +135,37 @@ func TestMemStorage_GetMetric(t *testing.T) { }{ { name: "existing counter", - fields: map[string]map[string]mtr.Metric{ - mtr.CounterType: {"requests": counter}, + fields: map[string]map[string]models.Metric{ + models.CounterType: {"requests": counter}, }, - mType: mtr.CounterType, + mType: models.CounterType, mName: "requests", wantVal: int64(42), wantOk: true, }, { name: "existing gauge", - fields: map[string]map[string]mtr.Metric{ - mtr.GaugeType: {"temperature": gauge}, + fields: map[string]map[string]models.Metric{ + models.GaugeType: {"temperature": gauge}, }, - mType: mtr.GaugeType, + mType: models.GaugeType, mName: "temperature", wantVal: float64(36.6), wantOk: true, }, { name: "missing metric name", - fields: map[string]map[string]mtr.Metric{ - mtr.CounterType: {"requests": counter}, + fields: map[string]map[string]models.Metric{ + models.CounterType: {"requests": counter}, }, - mType: mtr.CounterType, + mType: models.CounterType, mName: "nonexistent", wantVal: nil, wantOk: false, }, { name: "missing metric type", - fields: map[string]map[string]mtr.Metric{}, + fields: map[string]map[string]models.Metric{}, mType: "unknown_type", mName: "anything", wantVal: nil, diff --git a/pkg/converter/converter.go b/pkg/converter/converter.go index 179996f..01a815a 100644 --- a/pkg/converter/converter.go +++ b/pkg/converter/converter.go @@ -4,20 +4,20 @@ import ( "fmt" "strconv" - mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" "github.com/rs/zerolog/log" ) func ConvertByType(mType, mValue string) (any, error) { switch mType { - case mtr.GaugeType: + case models.GaugeType: if val, err := strconv.ParseFloat(mValue, 64); err != nil { return nil, fmt.Errorf("convert gauge value %s: %w", mValue, err) } else { return val, nil } - case mtr.CounterType: + case models.CounterType: if val, err := strconv.ParseInt(mValue, 10, 64); err != nil { return nil, fmt.Errorf("convert counter value %s: %w", mValue, err) } else { @@ -28,8 +28,8 @@ func ConvertByType(mType, mValue string) (any, error) { } } -func ConvertMetrics(src serialize.MetricsList) ([]mtr.Metric, error) { - converted := make([]mtr.Metric, 0, len(src)) +func ConvertMetrics(src serialize.MetricsList) ([]models.Metric, error) { + converted := make([]models.Metric, 0, len(src)) for _, m := range src { metric, err := ConvertMetric(m) @@ -44,21 +44,21 @@ func ConvertMetrics(src serialize.MetricsList) ([]mtr.Metric, error) { return converted, nil } -func ConvertMetric(src serialize.Metric) (mtr.Metric, error) { - var converted mtr.Metric +func ConvertMetric(src serialize.Metric) (models.Metric, error) { + var converted models.Metric switch src.MType { - case mtr.GaugeType: + case models.GaugeType: if src.Value == nil { return nil, fmt.Errorf("nil gauge value for ID: %s", src.ID) } - converted = mtr.NewGauge(src.ID, *src.Value) + converted = models.NewGauge(src.ID, *src.Value) - case mtr.CounterType: + case models.CounterType: if src.Delta == nil { return nil, fmt.Errorf("nil counter value for ID: %s", src.ID) } - converted = mtr.NewCounter(src.ID, *src.Delta) + converted = models.NewCounter(src.ID, *src.Delta) default: return nil, fmt.Errorf("unsupported metric type: %s", src.MType) } diff --git a/pkg/files/file-storage.go b/pkg/files/file-storage.go index f8500ae..f0bc098 100644 --- a/pkg/files/file-storage.go +++ b/pkg/files/file-storage.go @@ -1,4 +1,4 @@ -package database +package files import ( "context" @@ -8,7 +8,7 @@ import ( "path/filepath" col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" - mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" ) @@ -23,13 +23,13 @@ func SaveToDB(ctx context.Context, collector col.Collector, path string) error { newMetric.ID = metric.Name() switch metric.Type() { - case mtr.GaugeType: + case models.GaugeType: val, ok := metric.Value().(float64) if !ok { return fmt.Errorf("invalid gauge metric value") } newMetric.Value = &val - case mtr.CounterType: + case models.CounterType: val, ok := metric.Value().(int64) if !ok { return fmt.Errorf("invalid counter metric value") @@ -109,12 +109,12 @@ func LoadFromDB(ctx context.Context, collector col.Collector, path string) error for _, metric := range data { switch metric.MType { - case mtr.GaugeType: + case models.GaugeType: if err := collector.UpdateMetric(ctx, metric.MType, metric.ID, *metric.Value); err != nil { log.Error().Err(err).Msg("update metric error") return fmt.Errorf("update metric error %w", err) } - case mtr.CounterType: + case models.CounterType: if err := collector.UpdateMetric(ctx, metric.MType, metric.ID, *metric.Delta); err != nil { log.Error().Err(err).Msg("update metric error") return fmt.Errorf("update metric error %w", err) diff --git a/pkg/runtime-stats/runtime-stats.go b/pkg/runtime-stats/runtime-stats.go index 5c024d7..4cbe07c 100644 --- a/pkg/runtime-stats/runtime-stats.go +++ b/pkg/runtime-stats/runtime-stats.go @@ -3,7 +3,7 @@ package runtimestats import ( "runtime" - mtr "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" ) type MemRuntimeStat struct { @@ -15,137 +15,137 @@ type MemRuntimeStat struct { var MemRuntimeStats []MemRuntimeStat = []MemRuntimeStat{ { Name: "Alloc", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.Alloc) }, }, { Name: "BuckHashSys", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.BuckHashSys) }, }, { Name: "Frees", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.Frees) }, }, { Name: "GCCPUFraction", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.GCCPUFraction) }, }, { Name: "GCSys", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.GCSys) }, }, { Name: "HeapAlloc", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.HeapAlloc) }, }, { Name: "HeapIdle", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.HeapIdle) }, }, { Name: "HeapInuse", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.HeapInuse) }, }, { Name: "HeapObjects", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.HeapObjects) }, }, { Name: "HeapReleased", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.HeapReleased) }, }, { Name: "HeapSys", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.HeapSys) }, }, { Name: "LastGC", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.LastGC) }, }, { Name: "Lookups", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.Lookups) }, }, { Name: "MCacheInuse", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.MCacheInuse) }, }, { Name: "MCacheSys", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.MCacheSys) }, }, { Name: "MSpanInuse", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.MSpanInuse) }, }, { Name: "MSpanSys", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.MSpanSys) }, }, { Name: "Mallocs", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.Mallocs) }, }, { Name: "NextGC", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.NextGC) }, }, { Name: "NumForcedGC", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.NumForcedGC) }, }, { Name: "NumGC", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.NumGC) }, }, { Name: "OtherSys", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.OtherSys) }, }, { Name: "PauseTotalNs", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.PauseTotalNs) }, }, { Name: "StackInuse", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.StackInuse) }, }, { Name: "StackSys", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.StackSys) }, }, { Name: "Sys", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.Sys) }, }, { Name: "TotalAlloc", - Type: mtr.GaugeType, + Type: models.GaugeType, Get: func(m *runtime.MemStats) any { return float64(m.TotalAlloc) }, }, } diff --git a/pkg/serialization/serialization.go b/pkg/serialization/serialization.go index edaef03..c10eae5 100644 --- a/pkg/serialization/serialization.go +++ b/pkg/serialization/serialization.go @@ -1,7 +1,7 @@ package serialization import ( - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/metrics" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -18,48 +18,48 @@ type MetricsList []Metric func (mtr *Metric) SetValue(value any) error { switch mtr.MType { - case metrics.GaugeType: + case models.GaugeType: val, ok := value.(float64) if !ok { log.Error().Msg("not value type of value") - return metrics.ErrInvalidValueType + return models.ErrInvalidValueType } mtr.Value = &val return nil - case metrics.CounterType: + case models.CounterType: val, ok := value.(int64) if !ok { log.Error().Msg("not value type of value") - return metrics.ErrInvalidValueType + return models.ErrInvalidValueType } mtr.Delta = &val return nil default: - return metrics.ErrInvalidMetricsType + return models.ErrInvalidMetricsType } } func (mtr *Metric) GetValue() (any, error) { switch mtr.MType { - case metrics.GaugeType: + case models.GaugeType: if mtr.Value == nil { - return nil, metrics.ErrMetricsNotFound + return nil, models.ErrMetricsNotFound } return *mtr.Value, nil - case metrics.CounterType: + case models.CounterType: if mtr.Delta == nil { - return nil, metrics.ErrMetricsNotFound + return nil, models.ErrMetricsNotFound } return *mtr.Delta, nil default: - return nil, metrics.ErrInvalidMetricsType + return nil, models.ErrInvalidMetricsType } } From e5185cd5ac464df1156e0777c395d32cd0bb9bd3 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Tue, 22 Jul 2025 02:07:13 +0300 Subject: [PATCH 111/126] change sign on collector, test + --- internal/collector/collerctor.go | 4 +- internal/handlers/agent/agent.go | 5 +- internal/handlers/server/server.go | 69 +++++------ internal/repository/file.go | 16 ++- internal/repository/memory.go | 8 +- internal/repository/postgres.go | 168 +++++++++----------------- internal/repository/storage_test.go | 178 ++++++++++++++++------------ pkg/files/file-storage.go | 6 +- 8 files changed, 207 insertions(+), 247 deletions(-) diff --git a/internal/collector/collerctor.go b/internal/collector/collerctor.go index bc8043a..7d1ccea 100644 --- a/internal/collector/collerctor.go +++ b/internal/collector/collerctor.go @@ -7,8 +7,8 @@ import ( ) type Collector interface { - GetMetric(ctx context.Context, mType, mName string) (any, error) - GetAllMetrics(ctx context.Context) []models.Metric + GetMetric(ctx context.Context, mType, mName string) (models.Metric, error) + GetAllMetrics(ctx context.Context) ([]models.Metric, error) UpdateMetric(ctx context.Context, mType, mName string, mValue any) error Ping(ctx context.Context) error diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index add40af..824fd65 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -49,7 +49,10 @@ func UpdateAllMetrics(ctx context.Context, storage col.Collector) { } func SendAllMetrics(ctx context.Context, client *resty.Client, storage *repo.MemStorage) { - allMetrics := storage.GetAllMetrics(ctx) + allMetrics, err := storage.GetAllMetrics(ctx) + if err != nil { + log.Error().Err(err).Msg("failed to Get metrics") + } for _, metric := range allMetrics { mType := metric.Type() diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index ccc0b49..2f9802f 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -2,7 +2,6 @@ package server import ( "compress/gzip" - "context" "fmt" "html/template" "io" @@ -36,14 +35,14 @@ func (srv *Server) GetMetric() http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { mType := chi.URLParam(req, "mType") mName := chi.URLParam(req, "mName") - value, err := srv.Storage.GetMetric(req.Context(), mType, mName) + metric, err := srv.Storage.GetMetric(req.Context(), mType, mName) if err != nil { http.Error(res, fmt.Sprintf("Metric %s was not found", mName), http.StatusNotFound) return } var valueStr string - switch v := value.(type) { + switch v := metric.Value().(type) { case float64: valueStr = strconv.FormatFloat(v, 'f', -1, 64) case int64: @@ -66,7 +65,10 @@ func (srv *Server) GetMetric() http.HandlerFunc { func (srv *Server) GetAllMetrics() http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { - allMetrics := srv.Storage.GetAllMetrics(req.Context()) + allMetrics, err := srv.Storage.GetAllMetrics(req.Context()) + if err != nil { + log.Error().Err(err).Msg("failed to Get metrics") + } var metricsToTable []models.MetricTable @@ -187,32 +189,9 @@ func (srv *Server) UpdateMetric() http.HandlerFunc { } } -func FillMetricValueFromStorage(ctx context.Context, storage col.Collector, metric *serialize.Metric) bool { - value, err := storage.GetMetric(ctx, metric.MType, metric.ID) - if err != nil { - return false - } - - switch v := value.(type) { - case float64: - metric.Value = &v - case int64: - metric.Delta = &v - default: - log.Error(). - Str("metricType", metric.MType). - Str("metricName", metric.ID). - Msg("unsupported metric type") - - return false - } - - return true -} - func (srv *Server) GetMetricsHandlerJSON() http.HandlerFunc { return func(resp http.ResponseWriter, req *http.Request) { - var metric serialize.Metric + var jsonMetric serialize.Metric log.Info().Msg("GetMetricsHandlerJSON called") if req.Header.Get("Content-Type") != "application/json" { @@ -220,26 +199,27 @@ func (srv *Server) GetMetricsHandlerJSON() http.HandlerFunc { return } - if err := easyjson.UnmarshalFromReader(req.Body, &metric); err != nil { + if err := easyjson.UnmarshalFromReader(req.Body, &jsonMetric); err != nil { http.Error(resp, fmt.Sprintf("invalid json body: %v", err), http.StatusBadRequest) return } - value, err := srv.Storage.GetMetric(req.Context(), metric.MType, metric.ID) + log.Debug().Str("type", jsonMetric.MType).Str("name", jsonMetric.ID).Msg("") + metric, err := srv.Storage.GetMetric(req.Context(), jsonMetric.MType, jsonMetric.ID) if err != nil { log.Error().Err(err).Msg("can't get valid metric") http.Error(resp, "can't get valid metric", http.StatusNotFound) return } - if err := metric.SetValue(value); err != nil { + if err := jsonMetric.SetValue(metric.Value()); err != nil { log.Error().Err(err).Msg("can't set new value") http.Error(resp, "can't set new value", http.StatusInternalServerError) return } resp.Header().Set("Content-Type", "application/json") - if _, err := easyjson.MarshalToWriter(&metric, resp); err != nil { + if _, err := easyjson.MarshalToWriter(&jsonMetric, resp); err != nil { http.Error(resp, fmt.Sprintf("failed to encode json: %v", err), http.StatusInternalServerError) return } @@ -266,7 +246,7 @@ func (srv *Server) UpdateMetricsHandlerJSON() http.HandlerFunc { reader = req.Body } - var metric serialize.Metric + var jsonMetric serialize.Metric log.Info().Msg("UpdateMetricsHandlerJSON called") if req.Header.Get("Content-Type") != "application/json" { @@ -274,29 +254,38 @@ func (srv *Server) UpdateMetricsHandlerJSON() http.HandlerFunc { return } - if err := easyjson.UnmarshalFromReader(reader, &metric); err != nil { + if err := easyjson.UnmarshalFromReader(reader, &jsonMetric); err != nil { http.Error(resp, fmt.Sprintf("invalid json body: %v", err), http.StatusBadRequest) return } - value, err := metric.GetValue() + value, err := jsonMetric.GetValue() if err != nil { log.Error().Err(err).Msg("can't get value") http.Error(resp, "can't get value", http.StatusBadRequest) return } - if err := srv.Storage.UpdateMetric(req.Context(), metric.MType, metric.ID, value); err != nil { - http.Error(resp, fmt.Sprintf("invalid update metric %s: %v", metric.ID, err), http.StatusBadRequest) + if err := srv.Storage.UpdateMetric(req.Context(), jsonMetric.MType, jsonMetric.ID, value); err != nil { + http.Error(resp, fmt.Sprintf("invalid update metric %s: %v", jsonMetric.ID, err), http.StatusBadRequest) + return } - if !FillMetricValueFromStorage(req.Context(), srv.Storage, &metric) { - http.Error(resp, fmt.Sprintf("metric %s not found", metric.ID), http.StatusNotFound) + newMetric, err := srv.Storage.GetMetric(req.Context(), jsonMetric.MType, jsonMetric.ID) + if err != nil { + log.Error().Err(err).Msg("can't get new value metric") + http.Error(resp, "can't get new value metric", http.StatusNotFound) + return + } + + err = jsonMetric.SetValue(newMetric.Value()) + if err != nil { + http.Error(resp, fmt.Sprintf("metric %s not found", jsonMetric.ID), http.StatusNotFound) return } resp.Header().Set("Content-Type", "application/json") - if _, err := easyjson.MarshalToWriter(&metric, resp); err != nil { + if _, err := easyjson.MarshalToWriter(&jsonMetric, resp); err != nil { http.Error(resp, fmt.Sprintf("failed to encode json: %v", err), http.StatusInternalServerError) return } diff --git a/internal/repository/file.go b/internal/repository/file.go index 08f6096..cf9cea4 100644 --- a/internal/repository/file.go +++ b/internal/repository/file.go @@ -98,8 +98,8 @@ func (fs *FileStorage) UpdateMetric(ctx context.Context, mType, mName string, mV return nil } -func (fs *FileStorage) GetMetric(ctx context.Context, mType, mName string) (any, error) { - val, err := fs.storage.GetMetric(ctx, mType, mName) +func (fs *FileStorage) GetMetric(ctx context.Context, mType, mName string) (models.Metric, error) { + metric, err := fs.storage.GetMetric(ctx, mType, mName) if err != nil { log.Error(). Str("type", mType). @@ -108,11 +108,17 @@ func (fs *FileStorage) GetMetric(ctx context.Context, mType, mName string) (any, return nil, fmt.Errorf("can't get valid metric: %v", err) } - return val, nil + return metric, nil } -func (fs *FileStorage) GetAllMetrics(ctx context.Context) []models.Metric { - return fs.storage.GetAllMetrics(ctx) +func (fs *FileStorage) GetAllMetrics(ctx context.Context) ([]models.Metric, error) { + metrics, err := fs.storage.GetAllMetrics(ctx) + if err != nil { + log.Error().Err(err).Msg("failed get all metrics") + return nil, fmt.Errorf("failed GetAllMetrics %v", err) + } + + return metrics, nil } func (fs *FileStorage) Ping(ctx context.Context) error { diff --git a/internal/repository/memory.go b/internal/repository/memory.go index 0fdd92e..00fc362 100644 --- a/internal/repository/memory.go +++ b/internal/repository/memory.go @@ -54,7 +54,7 @@ func (ms *MemStorage) UpdateMetric(_ context.Context, mType, mName string, mValu return nil } -func (ms *MemStorage) GetMetric(_ context.Context, mType, mName string) (any, error) { +func (ms *MemStorage) GetMetric(_ context.Context, mType, mName string) (models.Metric, error) { ms.mutex.RLock() defer ms.mutex.RUnlock() @@ -63,10 +63,10 @@ func (ms *MemStorage) GetMetric(_ context.Context, mType, mName string) (any, er return nil, models.ErrMetricsNotFound } - return metric.Value(), nil + return metric, nil } -func (ms *MemStorage) GetAllMetrics(_ context.Context) []models.Metric { +func (ms *MemStorage) GetAllMetrics(_ context.Context) ([]models.Metric, error) { ms.mutex.RLock() defer ms.mutex.RUnlock() @@ -82,7 +82,7 @@ func (ms *MemStorage) GetAllMetrics(_ context.Context) []models.Metric { } } - return result + return result, nil } func (ms *MemStorage) Ping(ctx context.Context) error { diff --git a/internal/repository/postgres.go b/internal/repository/postgres.go index 2916200..277cc03 100644 --- a/internal/repository/postgres.go +++ b/internal/repository/postgres.go @@ -8,9 +8,7 @@ import ( col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/converter" errH "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/errors-handlers" - serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" "github.com/rs/zerolog/log" ) @@ -48,7 +46,7 @@ func NewDatabase(ctx context.Context, dataBaseDSN string) (col.Collector, error) }, nil } -func (db *Database) GetMetric(ctx context.Context, mType, mName string) (any, error) { +func (db *Database) GetMetric(ctx context.Context, mType, mName string) (models.Metric, error) { var ( id string Type string @@ -72,59 +70,37 @@ func (db *Database) GetMetric(ctx context.Context, mType, mName string) (any, er return nil, fmt.Errorf("row.Scan can't read: %w", err) } + if Type != mType { + return nil, models.ErrInvalidMetricsType + } + switch { case value.Valid: if Type != models.GaugeType { return nil, models.ErrInvalidMetricsType } - return value.Float64, nil + return models.NewGauge(mName, value.Float64), nil case delta.Valid: if Type != models.CounterType { return nil, models.ErrInvalidMetricsType } - return delta.Int64, nil + return models.NewCounter(mName, delta.Int64), nil default: log.Error().Msg("not valid value") return nil, models.ErrInvalidValueType } } -func fillMetricValues(metric *serialize.Metric, delta sql.NullInt64, value sql.NullFloat64) error { - switch { - case value.Valid: - if metric.MType != models.GaugeType { - return models.ErrInvalidMetricsType - } - - metric.Value = &value.Float64 - - case delta.Valid: - if metric.MType != models.CounterType { - return models.ErrInvalidMetricsType - } - - metric.Delta = &delta.Int64 - - default: - log.Error().Msg("not valid value") - return nil - } - - return nil -} - -func (db *Database) GetAllMetrics(ctx context.Context) []models.Metric { - metrics := make(serialize.MetricsList, 0) - +func (db *Database) GetAllMetrics(ctx context.Context) ([]models.Metric, error) { rows, err := db.DB.QueryContext(ctx, "SELECT ID, MType, Delta, Value FROM collector") if err != nil { log.Error().Err(err).Msg("The request was not processed") - return nil + return nil, fmt.Errorf("failed to query all metrics: %v", err) } defer func() { if err := rows.Close(); err != nil { @@ -132,124 +108,86 @@ func (db *Database) GetAllMetrics(ctx context.Context) []models.Metric { } }() + metrics := make([]models.Metric, 0) + var ( + id string + mType string delta sql.NullInt64 value sql.NullFloat64 ) for rows.Next() { - var metric serialize.Metric - - err = rows.Scan(&metric.ID, &metric.MType, &delta, &value) + err = rows.Scan(&id, &mType, &delta, &value) if err != nil { - log.Error().Err(err).Msgf("failed scan row: ID = %s, MType = %s", - metric.ID, metric.MType) - return nil + log.Error().Err(err).Msgf("failed scan row: ID = %s, MType = %s", id, mType) + return nil, fmt.Errorf("failed to scan metric row: %v", err) } - if err := fillMetricValues(&metric, delta, value); err != nil { - log.Error().Err(err).Msg("can't set values") - return nil - } + switch mType { + case models.GaugeType: + if value.Valid { + metrics = append(metrics, models.NewGauge(id, value.Float64)) + } - metrics = append(metrics, metric) - } + case models.CounterType: + if delta.Valid { + metrics = append(metrics, models.NewCounter(id, delta.Int64)) + } - if rows.Err() != nil { - log.Error().Err(rows.Err()).Msg("have rows error") - return nil - } + default: + return nil, fmt.Errorf("incorrectly metric type %v", models.ErrInvalidMetricsType) + } - convertedMetrics, err := converter.ConvertMetrics(metrics) - if err != nil { - log.Error().Err(err).Msg("failed to convert metrics") - return nil } - return convertedMetrics + return metrics, nil } func (db *Database) UpdateMetric(ctx context.Context, mType, mName string, mValue any) error { - oldValue, err := db.GetMetric(ctx, mType, mName) - - var m models.Metric + var delta *int64 + var value *float64 - switch newVal := mValue.(type) { + switch v := mValue.(type) { case float64: if mType != models.GaugeType { - return models.ErrInvalidMetricsType + return fmt.Errorf("metric type mismatch: got float64 with type %q", mType) } - m = models.NewGauge(mName, newVal) + value = &v case int64: if mType != models.CounterType { - return models.ErrInvalidMetricsType - } - if errors.Is(err, models.ErrMetricsNotFound) { - m = models.NewCounter(mName, newVal) - } else { - m = models.NewCounter(mName, oldValue.(int64)) - if err := m.Update(newVal); err != nil { - return fmt.Errorf("failed update metric %v", err) - } + return fmt.Errorf("metric type mismatch: got int64 with type %q", mType) } + delta = &v default: - return fmt.Errorf("unsupported metric type %T", mValue) - } - - metric := serialize.Metric{ - ID: mName, - MType: mType, - } - //TODO - make method maybe for Metrics type - switch metric.MType { - case models.GaugeType: - val, ok := m.Value().(float64) - if !ok { - return fmt.Errorf("expected float64 for gauge, got %T", m.Value()) - } - metric.Value = &val - - case models.CounterType: - val, ok := m.Value().(int64) - if !ok { - return fmt.Errorf("expected int64 for counter, got %T", m.Value()) - } - metric.Delta = &val - - default: - return fmt.Errorf("unknown metric type: %s", metric.MType) + return fmt.Errorf("unsupported metric value type: %T", v) } tx, err := db.DB.BeginTx(ctx, nil) if err != nil { - return fmt.Errorf("failed begin transaction") + return fmt.Errorf("begin transaction: %w", err) } + defer func() { + _ = tx.Rollback() + }() - execContext := func() error { - _, err = tx.ExecContext(ctx, - `INSERT INTO collector ("ID", "MType", "Delta", "Value") - VALUES ($1, $2, $3, $4) - ON CONFLICT ("ID") DO UPDATE - SET "Delta" = EXCLUDED."Delta", - "Value" = EXCLUDED."Value", - "MType" = EXCLUDED."MType";`, - metric.ID, metric.MType, metric.Delta, metric.Value) - + exec := func() error { + _, err := tx.ExecContext(ctx, ` + INSERT INTO collector ("ID", "MType", "Delta", "Value") + VALUES ($1, $2, $3, $4) + ON CONFLICT ("ID") DO UPDATE + SET "Delta" = collector."Delta" + EXCLUDED."Delta", + "Value" = EXCLUDED."Value", + "MType" = EXCLUDED."MType"`, + mName, mType, delta, value) return err } - err = errH.WithRetry(execContext, errH.IsPostgresRetriableError) - - if err != nil { - log.Error().Err(err).Msg("failed update insert into collector") - - if err := tx.Rollback(); err != nil && err != sql.ErrTxDone { - log.Printf("failed to rollback transaction: %v", err) - } - - return fmt.Errorf("failed update insert into collector: %w", err) + if err := errH.WithRetry(exec, errH.IsPostgresRetriableError); err != nil { + log.Error().Err(err).Msg("failed to insert/update metric") + return fmt.Errorf("update metric: %w", err) } return tx.Commit() diff --git a/internal/repository/storage_test.go b/internal/repository/storage_test.go index 5c5a367..c9e3696 100644 --- a/internal/repository/storage_test.go +++ b/internal/repository/storage_test.go @@ -106,13 +106,13 @@ func TestMemStorage_UpdateMetric(t *testing.T) { } if !tt.wantErr { - val, err := ms.GetMetric(ctx, tt.args.mType, tt.args.mName) + metric, err := ms.GetMetric(ctx, tt.args.mType, tt.args.mName) if err != nil { t.Errorf("metric not found") return } - if val != tt.wantResult { - t.Errorf("got = %v, want = %v", val, tt.wantResult) + if metric.Value() != tt.wantResult { + t.Errorf("got = %v, want = %v", metric.Value(), tt.wantResult) } } }) @@ -179,90 +179,110 @@ func TestMemStorage_GetMetric(t *testing.T) { mutex: sync.RWMutex{}, storage: tt.fields, } - gotVal, err := ms.GetMetric(ctx, tt.mType, tt.mName) + metric, err := ms.GetMetric(ctx, tt.mType, tt.mName) if (err == nil) != tt.wantOk { t.Errorf("GetMetric() error = %v, wantOk = %v", err, tt.wantOk) return } - if tt.wantOk && gotVal != tt.wantVal { - t.Errorf("GetMetric() gotVal = %v, want %v", gotVal, tt.wantVal) + if tt.wantOk && metric.Value() != tt.wantVal { + t.Errorf("GetMetric() gotVal = %v, want %v", metric.Value(), tt.wantVal) + } + }) + } +} + +func TestMemStorage_GetAllMetrics(t *testing.T) { + ctx := context.Background() + + counter := models.NewCounter("requests", 100) + gauge := models.NewGauge("temperature", 25.5) + + tests := []struct { + name string + storage map[string]map[string]models.Metric + want map[string]models.Metric + }{ + { + name: "empty storage", + storage: map[string]map[string]models.Metric{}, + want: map[string]models.Metric{}, + }, + { + name: "storage with one counter", + storage: map[string]map[string]models.Metric{ + models.CounterType: { + "requests": counter, + }, + }, + want: map[string]models.Metric{ + "requests": counter, + }, + }, + { + name: "storage with gauge and counter", + storage: map[string]map[string]models.Metric{ + models.CounterType: { + "requests": counter, + }, + models.GaugeType: { + "temperature": gauge, + }, + }, + want: map[string]models.Metric{ + "requests": counter, + "temperature": gauge, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ms := &MemStorage{ + mutex: sync.RWMutex{}, + storage: tt.storage, + } + + got, err := ms.GetAllMetrics(ctx) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + gotMap := make(map[string]models.Metric) + for _, m := range got { + gotMap[m.Name()] = m + } + + if len(gotMap) != len(tt.want) { + t.Errorf("wrong number of metrics: got %d, want %d", len(gotMap), len(tt.want)) + } + + for name, wantMetric := range tt.want { + gotMetric, ok := gotMap[name] + if !ok { + t.Errorf("missing metric: %s", name) + continue + } + + if gotMetric.Type() != wantMetric.Type() { + t.Errorf("metric %s type mismatch: got %s, want %s", name, gotMetric.Type(), wantMetric.Type()) + } + + switch wantMetric.Type() { + case models.CounterType: + if gotMetric.Value() == nil { + t.Errorf("metric %s: expected counter delta, got nil", name) + } + case models.GaugeType: + if gotMetric.Value() == nil { + t.Errorf("metric %s: expected gauge value, got nil", name) + } + default: + t.Errorf("metric %s: unknown type %s", name, wantMetric.Type()) + } } }) } } -// func TestMemStorage_GetAllMetrics(t *testing.T) { -// counter := mtr.NewCounter("requests", 100) -// gauge := mtr.NewGauge("temperature", 25.5) -// type fields struct { -// mutex sync.RWMutex -// storage map[string]map[string]mtr.Metric -// } -// tests := []struct { -// name string -// fields fields -// want map[string]map[string]mtr.Metric -// }{ -// { -// name: "empty storage", -// fields: fields{ -// mutex: sync.RWMutex{}, -// storage: map[string]map[string]mtr.Metric{}, -// }, -// want: map[string]map[string]mtr.Metric{}, -// }, -// { -// name: "storage with one counter", -// fields: fields{ -// mutex: sync.RWMutex{}, -// storage: map[string]map[string]mtr.Metric{ -// mtr.CounterType: { -// "requests": counter, -// }, -// }, -// }, -// want: map[string]map[string]mtr.Metric{ -// mtr.CounterType: { -// "requests": counter, -// }, -// }, -// }, -// { -// name: "storage with gauge and counter", -// fields: fields{ -// mutex: sync.RWMutex{}, -// storage: map[string]map[string]mtr.Metric{ -// mtr.CounterType: { -// "requests": counter, -// }, -// mtr.GaugeType: { -// "temperature": gauge, -// }, -// }, -// }, -// want: map[string]map[string]mtr.Metric{ -// mtr.CounterType: { -// "requests": counter, -// }, -// mtr.GaugeType: { -// "temperature": gauge, -// }, -// }, -// }, -// } -// -// for i := range tests { -// tt := &tests[i] -// t.Run(tt.name, func(t *testing.T) { -// ms := &MemStorage{ -// mutex: sync.RWMutex{}, -// storage: tt.fields.storage, -// } -// if got := ms.GetAllMetrics(); !reflect.DeepEqual(got, tt.want) { -// t.Errorf("MemStorage.GetAllMetrics() = %v, want %v", got, tt.want) -// } -// }) -// } -// } diff --git a/pkg/files/file-storage.go b/pkg/files/file-storage.go index f0bc098..e314ccc 100644 --- a/pkg/files/file-storage.go +++ b/pkg/files/file-storage.go @@ -14,7 +14,11 @@ import ( ) func SaveToDB(ctx context.Context, collector col.Collector, path string) error { - allMetrics := collector.GetAllMetrics(ctx) + allMetrics, err := collector.GetAllMetrics(ctx) + if err != nil { + log.Error().Err(err).Msg("failed to Get metrics") + } + data := make(serialize.MetricsList, 0, len(allMetrics)) for _, metric := range allMetrics { From 6d484e26569105c96a550a0a8dd07fd8c720258a Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Tue, 22 Jul 2025 04:48:59 +0300 Subject: [PATCH 112/126] add usecases --- cmd/agent/cfg_agent.go | 7 +- cmd/server/cfg_server.go | 5 +- internal/handlers/agent/agent.go | 40 ++------- internal/handlers/agent/usecase.go | 60 ++++++++++++++ internal/handlers/server/server.go | 74 ++++------------- internal/handlers/server/server_test.go | 35 ++------ internal/usecase/usecases.go | 106 ++++++++++++++++++++++++ 7 files changed, 203 insertions(+), 124 deletions(-) create mode 100644 internal/handlers/agent/usecase.go create mode 100644 internal/usecase/usecases.go diff --git a/cmd/agent/cfg_agent.go b/cmd/agent/cfg_agent.go index a474b70..5abda25 100644 --- a/cmd/agent/cfg_agent.go +++ b/cmd/agent/cfg_agent.go @@ -108,7 +108,8 @@ func init() { } func startAgent(ctx context.Context) { - storage := repo.NewMemStorage() + metricStorage := repo.NewMemStorage() + agentUsecase := agent.NewAgentUsecase(metricStorage) client := resty.New(). SetTimeout(5 * time.Second). @@ -128,9 +129,9 @@ func startAgent(ctx context.Context) { return case <-pollTimer.C: - agent.UpdateAllMetrics(ctx, storage) + agent.UpdateAllMetrics(ctx, metricStorage) case <-reportTimer.C: - agent.SendAllMetrics(ctx, client, storage) + agentUsecase.SendAllMetrics(ctx, client) } } } diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 1709dc2..f05d4d7 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -16,6 +16,7 @@ import ( "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/router" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecase" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -117,7 +118,9 @@ func startServer(ctx context.Context, opts *config.Options) error { } }() - r := router.NewRouter(server.NewServer(collector, opts)) + metricUsecase := usecase.NewMetricUsecase(collector) + + r := router.NewRouter(server.NewServer(metricUsecase, opts)) srv := &http.Server{ Addr: opts.EndPointAddr, diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 824fd65..ce01123 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -15,7 +15,6 @@ import ( col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" - repo "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/repository" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" rt "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/runtime-stats" serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" @@ -48,44 +47,20 @@ func UpdateAllMetrics(ctx context.Context, storage col.Collector) { } } -func SendAllMetrics(ctx context.Context, client *resty.Client, storage *repo.MemStorage) { - allMetrics, err := storage.GetAllMetrics(ctx) +func (uc *AgentUsecase) SendAllMetrics(ctx context.Context, client *resty.Client) { + allMetrics, err := uc.GetAllMetrics(ctx) if err != nil { log.Error().Err(err).Msg("failed to Get metrics") } + log.Debug().Msgf("send all metrics: got %d metrics", len(allMetrics)) for _, metric := range allMetrics { - mType := metric.Type() - mName := metric.Name() - - metricJSON := serialize.Metric{ - ID: mName, - MType: mType, + jsonMetric, err := uc.GetMetricJSON(ctx, metric) + if err != nil { + log.Error().Err(err).Msg("failed to get json metric") } - switch mType { - case models.GaugeType: - val, ok := metric.Value().(float64) - if !ok { - log.Error().Str("metric_name", mName).Str("metric_type", mType). - Msg("Invalid metric value type") - continue - } - - metricJSON.Value = &val - sendMetric(client, &metricJSON) - - case models.CounterType: - val, ok := metric.Value().(int64) - if !ok { - log.Error().Str("metric_name", mName).Str("metric_type", mType). - Msg("Invalid metric value type") - continue - } - - metricJSON.Delta = &val - sendMetric(client, &metricJSON) - } + sendMetric(client, jsonMetric) } } @@ -96,6 +71,7 @@ func sendMetric(client *resty.Client, metricJSON *serialize.Metric) { 1 * time.Second, } + log.Debug().Msg("sendMetric") buf, ok, err := ConvertToGzipData(metricJSON) if err != nil { log.Error().Err(err).Msg("Failed to convert metric to gzip") diff --git a/internal/handlers/agent/usecase.go b/internal/handlers/agent/usecase.go new file mode 100644 index 0000000..de20531 --- /dev/null +++ b/internal/handlers/agent/usecase.go @@ -0,0 +1,60 @@ +package agent + +import ( + "context" + + col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" + log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" + serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" +) + +type AgentUsecase struct { + Storage col.Collector +} + +func NewAgentUsecase(storage col.Collector) *AgentUsecase { + return &AgentUsecase{ + Storage: storage, + } +} + +func (uc *AgentUsecase) GetAllMetrics(ctx context.Context) ([]models.Metric, error) { + return uc.Storage.GetAllMetrics(ctx) +} + +func (uc *AgentUsecase) GetMetricJSON(ctx context.Context, metric models.Metric) (*serialize.Metric, error) { + mType := metric.Type() + mName := metric.Name() + + metricJSON := &serialize.Metric{ + ID: mName, + MType: mType, + } + + switch mType { + case models.GaugeType: + val, ok := metric.Value().(float64) + if !ok { + log.Error().Str("metric_name", mName).Str("metric_type", mType). + Msg("Invalid metric value type") + + return nil, models.ErrInvalidValueType + } + + metricJSON.Value = &val + + case models.CounterType: + val, ok := metric.Value().(int64) + if !ok { + log.Error().Str("metric_name", mName).Str("metric_type", mType). + Msg("Invalid metric value type") + + return nil, models.ErrInvalidValueType + } + + metricJSON.Delta = &val + } + + return metricJSON, nil +} diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index 2f9802f..c3d37c5 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -6,27 +6,25 @@ import ( "html/template" "io" "net/http" - "strconv" "github.com/go-chi/chi/v5" "github.com/mailru/easyjson" - col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" - models "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecase" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/converter" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" ) type Server struct { - Storage col.Collector + Usecase *usecase.MetricUsecase Opts *config.Options } -func NewServer(col col.Collector, opts *config.Options) *Server { +func NewServer(uc *usecase.MetricUsecase, opts *config.Options) *Server { return &Server{ - Storage: col, + Usecase: uc, Opts: opts, } } @@ -35,23 +33,13 @@ func (srv *Server) GetMetric() http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { mType := chi.URLParam(req, "mType") mName := chi.URLParam(req, "mName") - metric, err := srv.Storage.GetMetric(req.Context(), mType, mName) + + valueStr, err := srv.Usecase.GetMetricStr(req.Context(), mType, mName) if err != nil { http.Error(res, fmt.Sprintf("Metric %s was not found", mName), http.StatusNotFound) return } - var valueStr string - switch v := metric.Value().(type) { - case float64: - valueStr = strconv.FormatFloat(v, 'f', -1, 64) - case int64: - valueStr = strconv.FormatInt(v, 10) - default: - http.Error(res, "an unexpected type of metric", http.StatusInternalServerError) - return - } - res.Header().Set("Content-Type", "text/plain") res.WriteHeader(http.StatusOK) @@ -65,43 +53,11 @@ func (srv *Server) GetMetric() http.HandlerFunc { func (srv *Server) GetAllMetrics() http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { - allMetrics, err := srv.Storage.GetAllMetrics(req.Context()) + metricsToTable, err := srv.Usecase.GetAllMetrics(req.Context()) if err != nil { log.Error().Err(err).Msg("failed to Get metrics") - } - - var metricsToTable []models.MetricTable - - for _, metric := range allMetrics { - var valStr string - mName := metric.Name() - mType := metric.Type() - - switch mType { - case models.GaugeType: - val, ok := metric.Value().(float64) - if !ok { - log.Error().Str("metric_name", mName).Str("metric_type", mType). - Msg("Invalid metric value type") - continue - } - valStr = strconv.FormatFloat(val, 'f', -1, 64) - - case models.CounterType: - val, ok := metric.Value().(int64) - if !ok { - log.Error().Str("metric_name", mName).Str("metric_type", mType). - Msg("Invalid metric value type") - continue - } - valStr = strconv.FormatInt(val, 10) - } - - metricsToTable = append(metricsToTable, models.MetricTable{ - Name: mName, - Type: mType, - Value: valStr, - }) + http.Error(res, "failed to get metrics", http.StatusNotFound) + return } const htmlTemplate = ` @@ -176,7 +132,7 @@ func (srv *Server) UpdateMetric() http.HandlerFunc { return } - if err := srv.Storage.UpdateMetric(req.Context(), mType, mName, val); err != nil { + if err := srv.Usecase.UpdateMetric(req.Context(), mType, mName, val); err != nil { http.Error(res, err.Error(), http.StatusBadRequest) return } @@ -205,7 +161,7 @@ func (srv *Server) GetMetricsHandlerJSON() http.HandlerFunc { } log.Debug().Str("type", jsonMetric.MType).Str("name", jsonMetric.ID).Msg("") - metric, err := srv.Storage.GetMetric(req.Context(), jsonMetric.MType, jsonMetric.ID) + metric, err := srv.Usecase.GetMetric(req.Context(), jsonMetric.MType, jsonMetric.ID) if err != nil { log.Error().Err(err).Msg("can't get valid metric") http.Error(resp, "can't get valid metric", http.StatusNotFound) @@ -266,12 +222,12 @@ func (srv *Server) UpdateMetricsHandlerJSON() http.HandlerFunc { return } - if err := srv.Storage.UpdateMetric(req.Context(), jsonMetric.MType, jsonMetric.ID, value); err != nil { + if err := srv.Usecase.UpdateMetric(req.Context(), jsonMetric.MType, jsonMetric.ID, value); err != nil { http.Error(resp, fmt.Sprintf("invalid update metric %s: %v", jsonMetric.ID, err), http.StatusBadRequest) return } - newMetric, err := srv.Storage.GetMetric(req.Context(), jsonMetric.MType, jsonMetric.ID) + newMetric, err := srv.Usecase.GetMetric(req.Context(), jsonMetric.MType, jsonMetric.ID) if err != nil { log.Error().Err(err).Msg("can't get new value metric") http.Error(resp, "can't get new value metric", http.StatusNotFound) @@ -314,7 +270,7 @@ func (srv *Server) UpdatesMetricsHandlerJSON() http.HandlerFunc { } for _, metric := range metrics { - if err := srv.Storage.UpdateMetric(req.Context(), metric.Type(), metric.Name(), metric.Value()); err != nil { + if err := srv.Usecase.UpdateMetric(req.Context(), metric.Type(), metric.Name(), metric.Value()); err != nil { http.Error(w, fmt.Sprintf("failed update metric %v", err), http.StatusInternalServerError) return } @@ -326,7 +282,7 @@ func (srv *Server) UpdatesMetricsHandlerJSON() http.HandlerFunc { func (srv *Server) PingHandler() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - if err := srv.Storage.Ping(r.Context()); err != nil { + if err := srv.Usecase.Ping(r.Context()); err != nil { log.Error().Err(err).Msg("failed ping") http.Error(w, "can't ping DB", http.StatusInternalServerError) return diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index 7a49086..7e54546 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -2,7 +2,6 @@ package server_test import ( "context" - "database/sql" "io" "net/http" "net/http/httptest" @@ -13,8 +12,9 @@ import ( "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/router" repo "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/repository" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/router" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecase" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -29,20 +29,8 @@ func TestUpdateMetric(t *testing.T) { opt(opts) } - //FIXME - db, err := sql.Open("pgx", opts.DataBaseDSN) - if err != nil { - log.Error().Err(err).Msg("sql.Open error") - panic(err) - } - defer func() { - if err := db.Close(); err != nil { - log.Error().Err(err).Msg("Failed to db.Close") - } - }() - - storage := repo.NewMemStorage() - router := router.NewRouter(server.NewServer(storage, opts)) + metricUsecase := usecase.NewMetricUsecase(repo.NewMemStorage()) + router := router.NewRouter(server.NewServer(metricUsecase, opts)) tests := []struct { name string @@ -119,18 +107,6 @@ func TestGetMetric(t *testing.T) { opt(opts) } - //FIXME - db, err := sql.Open("pgx", opts.DataBaseDSN) - if err != nil { - log.Error().Err(err).Msg("sql.Open error") - panic(err) - } - defer func() { - if err := db.Close(); err != nil { - log.Error().Err(err).Msg("Failed to db.Close") - } - }() - //FIXME - maybe need mocks ctx := context.Background() storage := repo.NewMemStorage() @@ -143,7 +119,8 @@ func TestGetMetric(t *testing.T) { log.Error().Msgf("Failed to update metric requests_total: %v", err) } - router := router.NewRouter(server.NewServer(storage, opts)) + metricUsecase := usecase.NewMetricUsecase(storage) + router := router.NewRouter(server.NewServer(metricUsecase, opts)) tests := []struct { name string diff --git a/internal/usecase/usecases.go b/internal/usecase/usecases.go new file mode 100644 index 0000000..852c81e --- /dev/null +++ b/internal/usecase/usecases.go @@ -0,0 +1,106 @@ +package usecase + +import ( + "context" + "fmt" + "strconv" + + col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" + "github.com/rs/zerolog/log" +) + +type MetricUsecase struct { + repo col.Collector +} + +func NewMetricUsecase(repo col.Collector) *MetricUsecase { + return &MetricUsecase{repo: repo} +} + +func (uc *MetricUsecase) GetMetric(ctx context.Context, mType, mName string) (models.Metric, error) { + metric, err := uc.repo.GetMetric(ctx, mType, mName) + if err != nil { + return nil, fmt.Errorf("metric not found: %w", err) + } + + return metric, nil +} + +func (uc *MetricUsecase) GetMetricStr(ctx context.Context, mType, mName string) (string, error) { + metric, err := uc.repo.GetMetric(ctx, mType, mName) + if err != nil { + return "", fmt.Errorf("metric not found: %w", err) + } + + var valueStr string + switch v := metric.Value().(type) { + case float64: + valueStr = strconv.FormatFloat(v, 'f', -1, 64) + case int64: + valueStr = strconv.FormatInt(v, 10) + default: + return "", models.ErrInvalidValueType + } + + return valueStr, nil +} + +func (uc *MetricUsecase) GetAllMetrics(ctx context.Context) ([]models.MetricTable, error) { + allMetrics, err := uc.repo.GetAllMetrics(ctx) + if err != nil { + log.Error().Err(err).Msg("failed to Get metrics") + return nil, fmt.Errorf("failed to get metrics: %v", err) + } + + var metricsToTable []models.MetricTable + + for _, metric := range allMetrics { + var valStr string + mName := metric.Name() + mType := metric.Type() + + switch mType { + case models.GaugeType: + val, ok := metric.Value().(float64) + if !ok { + log.Error().Str("metric_name", mName).Str("metric_type", mType). + Msg("Invalid metric value type") + continue + } + valStr = strconv.FormatFloat(val, 'f', -1, 64) + + case models.CounterType: + val, ok := metric.Value().(int64) + if !ok { + log.Error().Str("metric_name", mName).Str("metric_type", mType). + Msg("Invalid metric value type") + continue + } + valStr = strconv.FormatInt(val, 10) + } + + metricsToTable = append(metricsToTable, models.MetricTable{ + Name: mName, + Type: mType, + Value: valStr, + }) + } + + return metricsToTable, nil +} + +func (uc *MetricUsecase) UpdateMetric(ctx context.Context, mType, mName string, value any) error { + if err := uc.repo.UpdateMetric(ctx, mType, mName, value); err != nil { + return fmt.Errorf("failed to update metric: %w", err) + } + + return nil +} + +func (uc *MetricUsecase) Ping(ctx context.Context) error { + if err := uc.repo.Ping(ctx); err != nil { + return fmt.Errorf("failed ping database: %w", err) + } + return nil +} From f7ebdaf61ea79d14991005b8540e447090ecd195 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Tue, 22 Jul 2025 04:55:17 +0300 Subject: [PATCH 113/126] fix lint --- internal/repository/postgres.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/repository/postgres.go b/internal/repository/postgres.go index 277cc03..92dae6e 100644 --- a/internal/repository/postgres.go +++ b/internal/repository/postgres.go @@ -73,7 +73,7 @@ func (db *Database) GetMetric(ctx context.Context, mType, mName string) (models. if Type != mType { return nil, models.ErrInvalidMetricsType } - + switch { case value.Valid: if Type != models.GaugeType { @@ -141,6 +141,11 @@ func (db *Database) GetAllMetrics(ctx context.Context) ([]models.Metric, error) } + if rows.Err() != nil { + log.Error().Err(rows.Err()).Msg("have rows error") + return nil, rows.Err() + } + return metrics, nil } From 875d543d2aa5d1088c1b69cf9c17a97090e2d09a Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Wed, 23 Jul 2025 15:38:12 +0300 Subject: [PATCH 114/126] fix tests --- .gitignore | 1 - go.mod | 6 ++ go.sum | 13 ++- .../collector/{collerctor.go => collector.go} | 0 internal/handlers/server/server_test.go | 13 +-- internal/models/counter_test.go | 25 +++--- internal/models/gauge_test.go | 25 +++--- internal/repository/memory.go | 7 +- .../{storage_test.go => repository_test.go} | 88 +++++++++---------- pkg/converter/converter.go | 1 + 10 files changed, 99 insertions(+), 80 deletions(-) rename internal/collector/{collerctor.go => collector.go} (100%) rename internal/repository/{storage_test.go => repository_test.go} (68%) diff --git a/.gitignore b/.gitignore index 4aa7bde..c16c13f 100644 --- a/.gitignore +++ b/.gitignore @@ -36,5 +36,4 @@ logFileAgent.log cmd/server/server cmd/agent/agent -/tmp log.txt diff --git a/go.mod b/go.mod index 79ed0b2..e34887f 100644 --- a/go.mod +++ b/go.mod @@ -12,9 +12,11 @@ require ( github.com/mailru/easyjson v0.9.0 github.com/rs/zerolog v1.34.0 github.com/spf13/cobra v1.9.1 + github.com/stretchr/testify v1.10.0 ) require ( + github.com/davecgh/go-spew v1.1.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgio v1.0.0 // indirect @@ -23,12 +25,16 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/kr/text v0.2.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/spf13/pflag v1.0.6 // indirect golang.org/x/crypto v0.37.0 // indirect golang.org/x/net v0.33.0 // indirect golang.org/x/sync v0.13.0 // indirect golang.org/x/sys v0.33.0 // indirect golang.org/x/text v0.24.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index c19ceca..9122e6e 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,7 @@ github.com/caarlos0/env/v6 v6.10.1 h1:t1mPSxNpei6M5yAeu1qtRdPAK29Nbcf/n3G7x+b3/I github.com/caarlos0/env/v6 v6.10.1/go.mod h1:hvp/ryKXKipEkcuYjs9mI4bBCg+UI0Yhgm5Zu0ddvwc= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -35,6 +36,10 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= @@ -47,6 +52,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= @@ -59,8 +66,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= @@ -77,6 +84,8 @@ golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/collector/collerctor.go b/internal/collector/collector.go similarity index 100% rename from internal/collector/collerctor.go rename to internal/collector/collector.go diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index 7e54546..df838db 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -16,6 +16,7 @@ import ( "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/router" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecase" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" + "github.com/stretchr/testify/assert" ) func TestUpdateMetric(t *testing.T) { @@ -89,9 +90,7 @@ func TestUpdateMetric(t *testing.T) { router.ServeHTTP(rr, req) - if rr.Code != tt.wantStatus { - t.Errorf("Test %s: expected status %d, got %d", tt.name, tt.wantStatus, rr.Code) - } + assert.Equal(t, tt.wantStatus, rr.Code, "Test %s: expected status %d, got %d", tt.name, tt.wantStatus, rr.Code) }) } } @@ -167,13 +166,9 @@ func TestGetMetric(t *testing.T) { router.ServeHTTP(rr, req) - if rr.Code != tt.wantStatus { - t.Errorf("Test %s: expected status %d, got %d", tt.name, tt.wantStatus, rr.Code) - } + assert.Equal(t, tt.wantStatus, rr.Code, "Test %s: expected status %d, got %d", tt.name, tt.wantStatus, rr.Code) body, _ := io.ReadAll(rr.Body) - if string(body) != tt.wantBody { - t.Errorf("Test %s: expected body %q, got %q", tt.name, tt.wantBody, string(body)) - } + assert.Equal(t, tt.wantBody, string(body), "Test %s: expected body %q, got %q", tt.name, tt.wantBody, string(body)) }) } } diff --git a/internal/models/counter_test.go b/internal/models/counter_test.go index c94dc7c..969a7ec 100644 --- a/internal/models/counter_test.go +++ b/internal/models/counter_test.go @@ -1,6 +1,12 @@ -package models +package models_test -import "testing" +import ( + "testing" + + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) func TestCounterUpdate(t *testing.T) { type fields struct { @@ -68,17 +74,14 @@ func TestCounterUpdate(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &counter{ - name: tt.fields.name, - value: tt.fields.value, - } + c := models.NewCounter(tt.fields.name, tt.fields.value) err := c.Update(tt.args.mValue) - if (err != nil) != tt.wantErr { - t.Errorf("Update() error = %v, wantErr = %v", err, tt.wantErr) - } - if c.value != tt.wantValue { - t.Errorf("Update() value = %v, wantValue = %v", c.value, tt.wantValue) + if tt.wantErr { + require.Error(t, err, "Update() error = %v, wantErr = %v", err, tt.wantErr) + } else { + require.NoError(t, err, "Update() error = %v, wantErr = %v", err, tt.wantErr) } + assert.Equal(t, tt.wantValue, c.Value(), "Update() value = %v, wantValue = %v", c.Value(), tt.wantValue) }) } } diff --git a/internal/models/gauge_test.go b/internal/models/gauge_test.go index a6f85bf..7ac0cf5 100644 --- a/internal/models/gauge_test.go +++ b/internal/models/gauge_test.go @@ -1,6 +1,12 @@ -package models +package models_test -import "testing" +import ( + "testing" + + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) func TestGaugeUpdate(t *testing.T) { type fields struct { @@ -44,17 +50,14 @@ func TestGaugeUpdate(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - g := &gauge{ - name: tt.fields.name, - value: tt.fields.value, - } + g := models.NewGauge(tt.fields.name, tt.fields.value) err := g.Update(tt.args.mValue) - if (err != nil) != tt.wantErr { - t.Errorf("gauge.Update() error = %v, wantErr %v", err, tt.wantErr) - } - if !tt.wantErr && g.value != tt.want { - t.Errorf("gauge.value = %v, want %v", g.value, tt.want) + if tt.wantErr { + require.Error(t, err, "gauge.Update() error = %v, wantErr = %v", err, tt.wantErr) + } else { + require.NoError(t, err, "gauge.Update() error = %v, wantErr = %v", err, tt.wantErr) } + assert.Equal(t, tt.want, g.Value(), "gauge.value = %v, want %v", g.Value(), tt.want) }) } } diff --git a/internal/repository/memory.go b/internal/repository/memory.go index 00fc362..fc0304d 100644 --- a/internal/repository/memory.go +++ b/internal/repository/memory.go @@ -58,7 +58,12 @@ func (ms *MemStorage) GetMetric(_ context.Context, mType, mName string) (models. ms.mutex.RLock() defer ms.mutex.RUnlock() - metric, ok := ms.storage[mType][mName] + typedMetric, ok := ms.storage[mType] + if !ok { + return nil, models.ErrInvalidMetricsType + } + + metric, ok := typedMetric[mName] if !ok { return nil, models.ErrMetricsNotFound } diff --git a/internal/repository/storage_test.go b/internal/repository/repository_test.go similarity index 68% rename from internal/repository/storage_test.go rename to internal/repository/repository_test.go index c9e3696..b4acec6 100644 --- a/internal/repository/storage_test.go +++ b/internal/repository/repository_test.go @@ -1,11 +1,13 @@ -package repository +package repository_test import ( "context" - "sync" "testing" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/repository" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestMemStorage_UpdateMetric(t *testing.T) { @@ -97,23 +99,25 @@ func TestMemStorage_UpdateMetric(t *testing.T) { for i := range tests { tt := &tests[i] t.Run(tt.name, func(t *testing.T) { - ms := &MemStorage{ - mutex: sync.RWMutex{}, - storage: tt.fields.storage, + ms := repository.NewMemStorage() + + for metricType, metrics := range tt.fields.storage { + for name, metric := range metrics { + _ = ms.UpdateMetric(ctx, metricType, name, metric.Value()) + } } - if err := ms.UpdateMetric(ctx, tt.args.mType, tt.args.mName, tt.args.mValue); (err != nil) != tt.wantErr { - t.Errorf("MemStorage.UpdateMetric() error = %v, wantErr %v", err, tt.wantErr) + + err := ms.UpdateMetric(ctx, tt.args.mType, tt.args.mName, tt.args.mValue) + if tt.wantErr { + require.Error(t, err, "MemStorage.UpdateMetric() error = %v, wantErr = %v", err, tt.wantErr) + } else { + require.NoError(t, err, "MemStorage.UpdateMetric() error = %v, wantErr = %v", err, tt.wantErr) } if !tt.wantErr { metric, err := ms.GetMetric(ctx, tt.args.mType, tt.args.mName) - if err != nil { - t.Errorf("metric not found") - return - } - if metric.Value() != tt.wantResult { - t.Errorf("got = %v, want = %v", metric.Value(), tt.wantResult) - } + require.NoError(t, err, "metric not found") + assert.Equal(t, tt.wantResult, metric.Value(), "got = %v, want = %v", metric.Value(), tt.wantResult) } }) } @@ -175,19 +179,21 @@ func TestMemStorage_GetMetric(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ms := &MemStorage{ - mutex: sync.RWMutex{}, - storage: tt.fields, - } - metric, err := ms.GetMetric(ctx, tt.mType, tt.mName) + ms := repository.NewMemStorage() - if (err == nil) != tt.wantOk { - t.Errorf("GetMetric() error = %v, wantOk = %v", err, tt.wantOk) - return + for metricType, metrics := range tt.fields { + for name, metric := range metrics { + _ = ms.UpdateMetric(ctx, metricType, name, metric.Value()) + } } - if tt.wantOk && metric.Value() != tt.wantVal { - t.Errorf("GetMetric() gotVal = %v, want %v", metric.Value(), tt.wantVal) + metric, err := ms.GetMetric(ctx, tt.mType, tt.mName) + + if tt.wantOk { + require.NoError(t, err, "GetMetric() error = %v, wantOk = %v", err, tt.wantOk) + assert.Equal(t, tt.wantVal, metric.Value(), "GetMetric() gotVal = %v, want %v", metric.Value(), tt.wantVal) + } else { + require.Error(t, err, "GetMetric() error = %v, wantOk = %v", err, tt.wantOk) } }) } @@ -239,50 +245,42 @@ func TestMemStorage_GetAllMetrics(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ms := &MemStorage{ - mutex: sync.RWMutex{}, - storage: tt.storage, + ms := repository.NewMemStorage() + + for metricType, metrics := range tt.storage { + for name, metric := range metrics { + _ = ms.UpdateMetric(ctx, metricType, name, metric.Value()) + } } got, err := ms.GetAllMetrics(ctx) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + require.NoError(t, err, "unexpected error: %v", err) gotMap := make(map[string]models.Metric) for _, m := range got { gotMap[m.Name()] = m } - if len(gotMap) != len(tt.want) { - t.Errorf("wrong number of metrics: got %d, want %d", len(gotMap), len(tt.want)) - } + assert.Equal(t, len(tt.want), len(gotMap), "wrong number of metrics: got %d, want %d", len(gotMap), len(tt.want)) for name, wantMetric := range tt.want { gotMetric, ok := gotMap[name] + assert.True(t, ok, "missing metric: %s", name) if !ok { - t.Errorf("missing metric: %s", name) continue } - if gotMetric.Type() != wantMetric.Type() { - t.Errorf("metric %s type mismatch: got %s, want %s", name, gotMetric.Type(), wantMetric.Type()) - } + assert.Equal(t, wantMetric.Type(), gotMetric.Type(), "metric %s type mismatch: got %s, want %s", name, gotMetric.Type(), wantMetric.Type()) switch wantMetric.Type() { case models.CounterType: - if gotMetric.Value() == nil { - t.Errorf("metric %s: expected counter delta, got nil", name) - } + assert.NotNil(t, gotMetric.Value(), "metric %s: expected counter delta, got nil", name) case models.GaugeType: - if gotMetric.Value() == nil { - t.Errorf("metric %s: expected gauge value, got nil", name) - } + assert.NotNil(t, gotMetric.Value(), "metric %s: expected gauge value, got nil", name) default: - t.Errorf("metric %s: unknown type %s", name, wantMetric.Type()) + assert.Fail(t, "unknown metric type", "metric %s: unknown type %s", name, wantMetric.Type()) } } }) } } - diff --git a/pkg/converter/converter.go b/pkg/converter/converter.go index 01a815a..bc67735 100644 --- a/pkg/converter/converter.go +++ b/pkg/converter/converter.go @@ -65,3 +65,4 @@ func ConvertMetric(src serialize.Metric) (models.Metric, error) { return converted, nil } + From f13315037a1f3326d40460bfb26cda867512c32e Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Wed, 23 Jul 2025 16:54:21 +0300 Subject: [PATCH 115/126] simplified file storage --- internal/models/counter.go | 8 ---- internal/models/gauge.go | 8 ---- internal/repository/memory.go | 30 ++++++------- pkg/converter/converter.go | 52 ++++++++++++++++++++++- pkg/files/file-storage.go | 79 +++++++++++++++-------------------- 5 files changed, 99 insertions(+), 78 deletions(-) diff --git a/internal/models/counter.go b/internal/models/counter.go index 92bff0e..9b44a79 100644 --- a/internal/models/counter.go +++ b/internal/models/counter.go @@ -33,11 +33,3 @@ func (c *counter) Update(mValue any) error { return nil } -func (c *counter) SetValue(v any) error { - val, ok := v.(int64) - if !ok { - return ErrInvalidValueType - } - c.value = val - return nil -} diff --git a/internal/models/gauge.go b/internal/models/gauge.go index f916919..9733350 100644 --- a/internal/models/gauge.go +++ b/internal/models/gauge.go @@ -33,11 +33,3 @@ func (g *gauge) Update(mValue any) error { return nil } -func (g *gauge) SetValue(v any) error { - val, ok := v.(float64) - if !ok { - return ErrInvalidValueType - } - g.value = val - return nil -} diff --git a/internal/repository/memory.go b/internal/repository/memory.go index fc0304d..959dde1 100644 --- a/internal/repository/memory.go +++ b/internal/repository/memory.go @@ -14,43 +14,43 @@ type MemStorage struct { func NewMemStorage() *MemStorage { return &MemStorage{ - storage: make(map[string]map[string]models.Metric), + storage: map[string]map[string]models.Metric{ + models.GaugeType: make(map[string]models.Metric), + models.CounterType: make(map[string]models.Metric), + }, } } func (ms *MemStorage) UpdateMetric(_ context.Context, mType, mName string, mValue any) error { ms.mutex.Lock() defer ms.mutex.Unlock() + if mName == "" { + return models.ErrMetricsNotFound + } if _, ok := ms.storage[mType]; !ok { - ms.storage[mType] = make(map[string]models.Metric) + return models.ErrInvalidMetricsType } - if oldMetric, ok := ms.storage[mType][mName]; ok { - return oldMetric.Update(mValue) + if metric, ok := ms.storage[mType][mName]; ok { + return metric.Update(mValue) } var newMetric models.Metric switch mType { case models.GaugeType: - value, ok := mValue.(float64) - if !ok { - return models.ErrInvalidValueType - } - newMetric = models.NewGauge(mName, value) + newMetric = models.NewGauge(mName, 0) case models.CounterType: - value, ok := mValue.(int64) - if !ok { - return models.ErrInvalidValueType - } - newMetric = models.NewCounter(mName, value) + newMetric = models.NewCounter(mName, 0) default: return models.ErrInvalidMetricsType } + if err := newMetric.Update(mValue); err != nil { + return err + } ms.storage[mType][mName] = newMetric - return nil } diff --git a/pkg/converter/converter.go b/pkg/converter/converter.go index bc67735..639296c 100644 --- a/pkg/converter/converter.go +++ b/pkg/converter/converter.go @@ -32,7 +32,7 @@ func ConvertMetrics(src serialize.MetricsList) ([]models.Metric, error) { converted := make([]models.Metric, 0, len(src)) for _, m := range src { - metric, err := ConvertMetric(m) + metric, err := сonvertMetric(m) if err != nil { log.Error().Err(err).Msg("failed convert metric") return nil, fmt.Errorf("metric failed convert %+v", m) @@ -44,7 +44,7 @@ func ConvertMetrics(src serialize.MetricsList) ([]models.Metric, error) { return converted, nil } -func ConvertMetric(src serialize.Metric) (models.Metric, error) { +func сonvertMetric(src serialize.Metric) (models.Metric, error) { var converted models.Metric switch src.MType { @@ -66,3 +66,51 @@ func ConvertMetric(src serialize.Metric) (models.Metric, error) { return converted, nil } +func сonvertToSerialization(src models.Metric) (serialize.Metric, error) { + var converted serialize.Metric + + switch src.Type() { + case models.GaugeType: + value, ok := src.Value().(float64) + if !ok { + return serialize.Metric{}, fmt.Errorf("invalid gauge value: %v", src.Value()) + } + converted = serialize.Metric{ + ID: src.Name(), + MType: src.Type(), + Value: &value, + } + + case models.CounterType: + delta, ok := src.Value().(int64) + if !ok { + return serialize.Metric{}, fmt.Errorf("invalid counter value: %v", src.Value()) + } + converted = serialize.Metric{ + ID: src.Name(), + MType: src.Type(), + Delta: &delta, + } + + default: + return serialize.Metric{}, fmt.Errorf("unknown metric type: %s", src.Type()) + } + + return converted, nil +} + +func ConverToSerialization(src []models.Metric) ([]serialize.Metric, error) { + converted := make([]serialize.Metric, 0, len(src)) + + for _, mtr := range src { + metric, err := сonvertToSerialization(mtr) + if err != nil { + log.Error().Err(err).Msg("failed convert metric") + return nil, fmt.Errorf("metric failed convert %+v", mtr) + } + + converted = append(converted, metric) + } + + return converted, nil +} diff --git a/pkg/files/file-storage.go b/pkg/files/file-storage.go index e314ccc..a3ac516 100644 --- a/pkg/files/file-storage.go +++ b/pkg/files/file-storage.go @@ -8,7 +8,7 @@ import ( "path/filepath" col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/converter" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" ) @@ -21,56 +21,52 @@ func SaveToDB(ctx context.Context, collector col.Collector, path string) error { data := make(serialize.MetricsList, 0, len(allMetrics)) - for _, metric := range allMetrics { - var newMetric serialize.Metric - newMetric.MType = metric.Type() - newMetric.ID = metric.Name() - - switch metric.Type() { - case models.GaugeType: - val, ok := metric.Value().(float64) - if !ok { - return fmt.Errorf("invalid gauge metric value") - } - newMetric.Value = &val - case models.CounterType: - val, ok := metric.Value().(int64) - if !ok { - return fmt.Errorf("invalid counter metric value") - } - newMetric.Delta = &val - default: - return fmt.Errorf("unknown metric type") - } - data = append(data, newMetric) + jsonMetrics, err := converter.ConverToSerialization(allMetrics) + if err != nil { + return fmt.Errorf("convert metrics error: %w", err) } - if len(data) == 0 { + if len(jsonMetrics) == 0 { log.Warn().Msg("No metrics to save, skipping file write") return nil } - bytes, err := json.MarshalIndent(data, "", " ") + bytes, err := json.MarshalIndent(jsonMetrics, "", " ") if err != nil { return fmt.Errorf("json marshal error: %w", err) } + err = WriteFileAtomic(path, bytes) + if err != nil { + return fmt.Errorf("write file atomic error: %w", err) + } + + log.Info(). + Str("path", path). + Int("metrics_saved", len(data)). + Msg("Metrics successfully saved") + + return nil +} + +func WriteFileAtomic(path string, data []byte) error { if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { return fmt.Errorf("failed to create directory: %w", err) } tmpPath := path + ".tmp" + file, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return fmt.Errorf("failed to create temp file: %w", err) } defer func() { - if err := file.Close(); err != nil { - log.Error().Err(err).Msg("failed to close file") + if cerr := file.Close(); cerr != nil { + log.Error().Err(cerr).Msg("failed to close file") } }() - if _, err := file.Write(bytes); err != nil { + if _, err := file.Write(data); err != nil { return fmt.Errorf("write failed: %w", err) } @@ -82,14 +78,10 @@ func SaveToDB(ctx context.Context, collector col.Collector, path string) error { return fmt.Errorf("rename failed: %w", err) } - log.Info(). - Str("path", path). - Int("metrics_saved", len(data)). - Msg("Metrics successfully saved") - return nil } + func LoadFromDB(ctx context.Context, collector col.Collector, path string) error { log.Info(). Str("path", path). @@ -111,18 +103,15 @@ func LoadFromDB(ctx context.Context, collector col.Collector, path string) error return fmt.Errorf("can't parse json format from DB %w", err) } - for _, metric := range data { - switch metric.MType { - case models.GaugeType: - if err := collector.UpdateMetric(ctx, metric.MType, metric.ID, *metric.Value); err != nil { - log.Error().Err(err).Msg("update metric error") - return fmt.Errorf("update metric error %w", err) - } - case models.CounterType: - if err := collector.UpdateMetric(ctx, metric.MType, metric.ID, *metric.Delta); err != nil { - log.Error().Err(err).Msg("update metric error") - return fmt.Errorf("update metric error %w", err) - } + metrics, err := converter.ConvertMetrics(data) + if err != nil { + return fmt.Errorf("convert metrics error: %w", err) + } + + for _, metric := range metrics { + if err := collector.UpdateMetric(ctx, metric.Type(), metric.Name(), metric.Value()); err != nil { + log.Error().Err(err).Msg("update metric error") + return fmt.Errorf("update metric error %w", err) } } From c45b556ebdb093ec74d9ea6e8fa657fff6487bbd Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Wed, 23 Jul 2025 17:44:21 +0300 Subject: [PATCH 116/126] add squirrel --- go.mod | 4 +++ go.sum | 7 ++++ internal/handlers/server/server.go | 26 +++++++++----- internal/repository/postgres.go | 55 +++++++++++++++++++++--------- internal/usecase/usecases.go | 29 ++++++---------- 5 files changed, 78 insertions(+), 43 deletions(-) diff --git a/go.mod b/go.mod index e34887f..fbc65a4 100644 --- a/go.mod +++ b/go.mod @@ -13,9 +13,11 @@ require ( github.com/rs/zerolog v1.34.0 github.com/spf13/cobra v1.9.1 github.com/stretchr/testify v1.10.0 + ) require ( + github.com/Masterminds/squirrel v1.5.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect @@ -26,6 +28,8 @@ require ( github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/kr/text v0.2.0 // indirect + github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect + github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index 9122e6e..6b7ca96 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= +github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/caarlos0/env/v6 v6.10.1 h1:t1mPSxNpei6M5yAeu1qtRdPAK29Nbcf/n3G7x+b3/II= github.com/caarlos0/env/v6 v6.10.1/go.mod h1:hvp/ryKXKipEkcuYjs9mI4bBCg+UI0Yhgm5Zu0ddvwc= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -40,6 +42,10 @@ github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= +github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= +github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= @@ -63,6 +69,7 @@ github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wx github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index c3d37c5..c498c98 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -6,6 +6,7 @@ import ( "html/template" "io" "net/http" + "strconv" "github.com/go-chi/chi/v5" "github.com/mailru/easyjson" @@ -19,13 +20,11 @@ import ( type Server struct { Usecase *usecase.MetricUsecase - Opts *config.Options } func NewServer(uc *usecase.MetricUsecase, opts *config.Options) *Server { return &Server{ Usecase: uc, - Opts: opts, } } @@ -34,12 +33,24 @@ func (srv *Server) GetMetric() http.HandlerFunc { mType := chi.URLParam(req, "mType") mName := chi.URLParam(req, "mName") - valueStr, err := srv.Usecase.GetMetricStr(req.Context(), mType, mName) + metric, err := srv.Usecase.GetMetric(req.Context(), mType, mName) if err != nil { + log.Error().Err(err).Msg("can't get valid metric") http.Error(res, fmt.Sprintf("Metric %s was not found", mName), http.StatusNotFound) return } + var valueStr string + switch v := metric.Value().(type) { + case float64: + valueStr = strconv.FormatFloat(v, 'f', -1, 64) + case int64: + valueStr = strconv.FormatInt(v, 10) + default: + http.Error(res, "an unexpected type of metric", http.StatusInternalServerError) + return + } + res.Header().Set("Content-Type", "text/plain") res.WriteHeader(http.StatusOK) @@ -269,11 +280,10 @@ func (srv *Server) UpdatesMetricsHandlerJSON() http.HandlerFunc { return } - for _, metric := range metrics { - if err := srv.Usecase.UpdateMetric(req.Context(), metric.Type(), metric.Name(), metric.Value()); err != nil { - http.Error(w, fmt.Sprintf("failed update metric %v", err), http.StatusInternalServerError) - return - } + if err := srv.Usecase.UpdateMetricList(req.Context(), metrics); err != nil { + log.Error().Err(err).Msg("failed update metrics") + http.Error(w, fmt.Sprintf("failed update metrics: %v", err), http.StatusInternalServerError) + return } w.WriteHeader(http.StatusOK) diff --git a/internal/repository/postgres.go b/internal/repository/postgres.go index 92dae6e..e4e5773 100644 --- a/internal/repository/postgres.go +++ b/internal/repository/postgres.go @@ -6,6 +6,8 @@ import ( "errors" "fmt" + sq "github.com/Masterminds/squirrel" + col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" errH "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/errors-handlers" @@ -36,7 +38,6 @@ func NewDatabase(ctx context.Context, dataBaseDSN string) (col.Collector, error) if err := db.Close(); err != nil { log.Error().Err(err).Msg("failed to close database") } - log.Error().Err(err).Msg("failed create table for database") return nil, fmt.Errorf("failed create table for database %w", err) } @@ -55,9 +56,17 @@ func (db *Database) GetMetric(ctx context.Context, mType, mName string) (models. ) getMtr := func() error { - row := db.DB.QueryRowContext(ctx, - `SELECT "ID", "MType", "Delta", "Value" FROM collector `+ - `WHERE "ID" = $1 AND "MType" = $2 LIMIT 1`, mName, mType) + builder := sq.Select(`"ID"`, `"MType"`, `"Delta"`, `"Value"`). + From("collector"). + Where(sq.Eq{`"ID"`: mName, `"MType"`: mType}). + Limit(1). + PlaceholderFormat(sq.Dollar) + + query, args, err := builder.ToSql() + if err != nil { + return fmt.Errorf("failed to build query: %w", err) + } + row := db.DB.QueryRowContext(ctx, query, args...) return row.Scan(&id, &Type, &delta, &value) } @@ -71,8 +80,8 @@ func (db *Database) GetMetric(ctx context.Context, mType, mName string) (models. } if Type != mType { - return nil, models.ErrInvalidMetricsType - } + return nil, models.ErrInvalidMetricsType + } switch { case value.Valid: @@ -95,8 +104,15 @@ func (db *Database) GetMetric(ctx context.Context, mType, mName string) (models. } func (db *Database) GetAllMetrics(ctx context.Context) ([]models.Metric, error) { - rows, err := db.DB.QueryContext(ctx, - "SELECT ID, MType, Delta, Value FROM collector") + builder := sq.Select(`"ID"`, `"MType"`, `"Delta"`, `"Value"`). + From("collector") + + query, args, err := builder.ToSql() + if err != nil { + return nil, fmt.Errorf("failed to build query: %w", err) + } + + rows, err := db.DB.QueryContext(ctx, query, args...) if err != nil { log.Error().Err(err).Msg("The request was not processed") @@ -179,14 +195,21 @@ func (db *Database) UpdateMetric(ctx context.Context, mType, mName string, mValu }() exec := func() error { - _, err := tx.ExecContext(ctx, ` - INSERT INTO collector ("ID", "MType", "Delta", "Value") - VALUES ($1, $2, $3, $4) - ON CONFLICT ("ID") DO UPDATE - SET "Delta" = collector."Delta" + EXCLUDED."Delta", - "Value" = EXCLUDED."Value", - "MType" = EXCLUDED."MType"`, - mName, mType, delta, value) + builder := sq.Insert("collector"). + Columns(`"ID"`, `"MType"`, `"Delta"`, `"Value"`). + Values(mName, mType, delta, value). + Suffix(`ON CONFLICT ("ID") DO UPDATE SET + "Delta" = collector."Delta" + EXCLUDED."Delta", + "Value" = EXCLUDED."Value", + "MType" = EXCLUDED."MType"`). + PlaceholderFormat(sq.Dollar) + + query, args, err := builder.ToSql() + if err != nil { + return err + } + + _, err = tx.ExecContext(ctx, query, args...) return err } diff --git a/internal/usecase/usecases.go b/internal/usecase/usecases.go index 852c81e..588582c 100644 --- a/internal/usecase/usecases.go +++ b/internal/usecase/usecases.go @@ -27,25 +27,6 @@ func (uc *MetricUsecase) GetMetric(ctx context.Context, mType, mName string) (mo return metric, nil } -func (uc *MetricUsecase) GetMetricStr(ctx context.Context, mType, mName string) (string, error) { - metric, err := uc.repo.GetMetric(ctx, mType, mName) - if err != nil { - return "", fmt.Errorf("metric not found: %w", err) - } - - var valueStr string - switch v := metric.Value().(type) { - case float64: - valueStr = strconv.FormatFloat(v, 'f', -1, 64) - case int64: - valueStr = strconv.FormatInt(v, 10) - default: - return "", models.ErrInvalidValueType - } - - return valueStr, nil -} - func (uc *MetricUsecase) GetAllMetrics(ctx context.Context) ([]models.MetricTable, error) { allMetrics, err := uc.repo.GetAllMetrics(ctx) if err != nil { @@ -98,6 +79,16 @@ func (uc *MetricUsecase) UpdateMetric(ctx context.Context, mType, mName string, return nil } +func (uc *MetricUsecase) UpdateMetricList(ctx context.Context, metrics []models.Metric) error { + for _, metric := range metrics { + if err := uc.UpdateMetric(ctx, metric.Type(), metric.Name(), metric.Value()); err != nil { + return fmt.Errorf("failed to update metric: %w", err) + } + } + + return nil +} + func (uc *MetricUsecase) Ping(ctx context.Context) error { if err := uc.repo.Ping(ctx); err != nil { return fmt.Errorf("failed ping database: %w", err) From 062269d3f33149da86f42659089a3bf2a937acb1 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Thu, 24 Jul 2025 02:42:36 +0300 Subject: [PATCH 117/126] add pinger --- cmd/agent/cfg_agent.go | 3 ++- cmd/server/cfg_server.go | 14 ++++++++--- internal/collector/collector.go | 1 - internal/handlers/agent/agent.go | 15 ++++++++--- internal/handlers/server/server.go | 14 ++++++----- internal/handlers/server/server_test.go | 10 ++++---- internal/repository/file.go | 4 --- internal/repository/memory.go | 4 --- .../agent/agent-usecase.go} | 0 internal/usecase/ping/ping-usecase.go | 25 +++++++++++++++++++ .../server-usecases.go} | 8 +----- 11 files changed, 64 insertions(+), 34 deletions(-) rename internal/{handlers/agent/usecase.go => usecase/agent/agent-usecase.go} (100%) create mode 100644 internal/usecase/ping/ping-usecase.go rename internal/usecase/{usecases.go => server/server-usecases.go} (92%) diff --git a/cmd/agent/cfg_agent.go b/cmd/agent/cfg_agent.go index 5abda25..795e52e 100644 --- a/cmd/agent/cfg_agent.go +++ b/cmd/agent/cfg_agent.go @@ -14,6 +14,7 @@ import ( "github.com/spf13/cobra" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/agent" + auc "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecase/agent" repo "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/repository" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -109,7 +110,7 @@ func init() { func startAgent(ctx context.Context) { metricStorage := repo.NewMemStorage() - agentUsecase := agent.NewAgentUsecase(metricStorage) + agentUsecase := agent.NewAgent(&auc.AgentUsecase{Storage: metricStorage}) client := resty.New(). SetTimeout(5 * time.Second). diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index f05d4d7..5811c2b 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -16,7 +16,8 @@ import ( "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/router" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecase" + srvUsecase "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecase/server" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecase/ping" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -118,9 +119,16 @@ func startServer(ctx context.Context, opts *config.Options) error { } }() - metricUsecase := usecase.NewMetricUsecase(collector) + metricUsecase := srvUsecase.NewMetricUsecase(collector) - r := router.NewRouter(server.NewServer(metricUsecase, opts)) + var pingUsecase *ping.PingUsecase + if pinger, ok := collector.(ping.Pinger); ok { + pingUsecase = ping.NewPingUsecase(pinger) + } else { + pingUsecase = nil + } + + r := router.NewRouter(server.NewServer(metricUsecase, pingUsecase)) srv := &http.Server{ Addr: opts.EndPointAddr, diff --git a/internal/collector/collector.go b/internal/collector/collector.go index 7d1ccea..e344eb8 100644 --- a/internal/collector/collector.go +++ b/internal/collector/collector.go @@ -11,6 +11,5 @@ type Collector interface { GetAllMetrics(ctx context.Context) ([]models.Metric, error) UpdateMetric(ctx context.Context, mType, mName string, mValue any) error - Ping(ctx context.Context) error Close() error } diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index ce01123..63a2d87 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -13,6 +13,7 @@ import ( "github.com/go-resty/resty/v2" "github.com/mailru/easyjson" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecase/agent" col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" @@ -20,6 +21,14 @@ import ( serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" ) +type Agent struct { + Usecase *agent.AgentUsecase +} + +func NewAgent(uc *agent.AgentUsecase) *Agent { + return &Agent{Usecase: uc} +} + func UpdateAllMetrics(ctx context.Context, storage col.Collector) { var memStats runtime.MemStats runtime.ReadMemStats(&memStats) @@ -47,15 +56,15 @@ func UpdateAllMetrics(ctx context.Context, storage col.Collector) { } } -func (uc *AgentUsecase) SendAllMetrics(ctx context.Context, client *resty.Client) { - allMetrics, err := uc.GetAllMetrics(ctx) +func (auc *Agent) SendAllMetrics(ctx context.Context, client *resty.Client) { + allMetrics, err := auc.Usecase.GetAllMetrics(ctx) if err != nil { log.Error().Err(err).Msg("failed to Get metrics") } log.Debug().Msgf("send all metrics: got %d metrics", len(allMetrics)) for _, metric := range allMetrics { - jsonMetric, err := uc.GetMetricJSON(ctx, metric) + jsonMetric, err := auc.Usecase.GetMetricJSON(ctx, metric) if err != nil { log.Error().Err(err).Msg("failed to get json metric") } diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index c498c98..ead24d8 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -11,20 +11,22 @@ import ( "github.com/go-chi/chi/v5" "github.com/mailru/easyjson" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecase" + srvUsecase "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecase/server" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecase/ping" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/converter" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" ) type Server struct { - Usecase *usecase.MetricUsecase + Usecase *srvUsecase.MetricUsecase + PingUsecase *ping.PingUsecase } -func NewServer(uc *usecase.MetricUsecase, opts *config.Options) *Server { +func NewServer(uc *srvUsecase.MetricUsecase, puc *ping.PingUsecase) *Server { return &Server{ - Usecase: uc, + Usecase: uc, + PingUsecase: puc, } } @@ -292,7 +294,7 @@ func (srv *Server) UpdatesMetricsHandlerJSON() http.HandlerFunc { func (srv *Server) PingHandler() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - if err := srv.Usecase.Ping(r.Context()); err != nil { + if err := srv.PingUsecase.Check(r.Context()); err != nil { log.Error().Err(err).Msg("failed ping") http.Error(w, "can't ping DB", http.StatusInternalServerError) return diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index df838db..9894816 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -14,7 +14,7 @@ import ( "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" repo "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/repository" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/router" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecase" + srvUsecase "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecase/server" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" "github.com/stretchr/testify/assert" ) @@ -30,8 +30,8 @@ func TestUpdateMetric(t *testing.T) { opt(opts) } - metricUsecase := usecase.NewMetricUsecase(repo.NewMemStorage()) - router := router.NewRouter(server.NewServer(metricUsecase, opts)) + metricUsecase := srvUsecase.NewMetricUsecase(repo.NewMemStorage()) + router := router.NewRouter(server.NewServer(metricUsecase, nil)) tests := []struct { name string @@ -118,8 +118,8 @@ func TestGetMetric(t *testing.T) { log.Error().Msgf("Failed to update metric requests_total: %v", err) } - metricUsecase := usecase.NewMetricUsecase(storage) - router := router.NewRouter(server.NewServer(metricUsecase, opts)) + metricUsecase := srvUsecase.NewMetricUsecase(storage) + router := router.NewRouter(server.NewServer(metricUsecase, nil)) tests := []struct { name string diff --git a/internal/repository/file.go b/internal/repository/file.go index cf9cea4..46b03c6 100644 --- a/internal/repository/file.go +++ b/internal/repository/file.go @@ -121,10 +121,6 @@ func (fs *FileStorage) GetAllMetrics(ctx context.Context) ([]models.Metric, erro return metrics, nil } -func (fs *FileStorage) Ping(ctx context.Context) error { - return nil -} - func (fs *FileStorage) Close() error { fs.wg.Wait() return nil diff --git a/internal/repository/memory.go b/internal/repository/memory.go index 959dde1..512ea5e 100644 --- a/internal/repository/memory.go +++ b/internal/repository/memory.go @@ -90,10 +90,6 @@ func (ms *MemStorage) GetAllMetrics(_ context.Context) ([]models.Metric, error) return result, nil } -func (ms *MemStorage) Ping(ctx context.Context) error { - return nil -} - func (ms *MemStorage) Close() error { return nil } diff --git a/internal/handlers/agent/usecase.go b/internal/usecase/agent/agent-usecase.go similarity index 100% rename from internal/handlers/agent/usecase.go rename to internal/usecase/agent/agent-usecase.go diff --git a/internal/usecase/ping/ping-usecase.go b/internal/usecase/ping/ping-usecase.go new file mode 100644 index 0000000..9b80216 --- /dev/null +++ b/internal/usecase/ping/ping-usecase.go @@ -0,0 +1,25 @@ +package ping + +import ( + "context" +) + +type Pinger interface { + Ping(ctx context.Context) error +} + +type Usecase interface { + Check(ctx context.Context) error +} + +type PingUsecase struct { + pinger Pinger +} + +func NewPingUsecase(pinger Pinger) *PingUsecase { + return &PingUsecase{pinger: pinger} +} + +func (puc *PingUsecase) Check(ctx context.Context) error { + return puc.pinger.Ping(ctx) +} diff --git a/internal/usecase/usecases.go b/internal/usecase/server/server-usecases.go similarity index 92% rename from internal/usecase/usecases.go rename to internal/usecase/server/server-usecases.go index 588582c..fee6501 100644 --- a/internal/usecase/usecases.go +++ b/internal/usecase/server/server-usecases.go @@ -1,4 +1,4 @@ -package usecase +package server import ( "context" @@ -89,9 +89,3 @@ func (uc *MetricUsecase) UpdateMetricList(ctx context.Context, metrics []models. return nil } -func (uc *MetricUsecase) Ping(ctx context.Context) error { - if err := uc.repo.Ping(ctx); err != nil { - return fmt.Errorf("failed ping database: %w", err) - } - return nil -} From f0967eceb3f57556101def016b10d89ebe02c466 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Fri, 25 Jul 2025 21:45:21 +0300 Subject: [PATCH 118/126] after code review --- cmd/agent/cfg_agent.go | 4 +- cmd/server/cfg_server.go | 6 +- internal/collector/config/config.go | 6 +- internal/handlers/agent/agent.go | 48 +++++----- internal/handlers/server/server.go | 52 ++++++++--- internal/handlers/server/server_test.go | 13 +-- internal/models/counter_test.go | 6 +- internal/models/gauge_test.go | 6 +- internal/repository/file.go | 14 ++- internal/repository/memory.go | 28 ++++-- internal/repository/postgres.go | 13 ++- internal/repository/repository_test.go | 22 ++--- internal/usecase/agent/agent-usecase.go | 60 ------------ internal/usecase/server/server-usecases.go | 91 ------------------- internal/usecases/agent/agent-usecase.go | 27 ++++++ .../collector.go => usecases/agent/deps.go} | 10 +- internal/usecases/ping/deps.go | 13 +++ .../ping/ping-usecase.go | 8 -- internal/usecases/server/deps.go | 27 ++++++ internal/usecases/server/server-usecases.go | 56 ++++++++++++ pkg/converter/converter.go | 43 ++++++++- pkg/files/file-storage.go | 26 ++---- 22 files changed, 318 insertions(+), 261 deletions(-) delete mode 100644 internal/usecase/agent/agent-usecase.go delete mode 100644 internal/usecase/server/server-usecases.go create mode 100644 internal/usecases/agent/agent-usecase.go rename internal/{collector/collector.go => usecases/agent/deps.go} (68%) create mode 100644 internal/usecases/ping/deps.go rename internal/{usecase => usecases}/ping/ping-usecase.go (68%) create mode 100644 internal/usecases/server/deps.go create mode 100644 internal/usecases/server/server-usecases.go diff --git a/cmd/agent/cfg_agent.go b/cmd/agent/cfg_agent.go index 795e52e..4218b7e 100644 --- a/cmd/agent/cfg_agent.go +++ b/cmd/agent/cfg_agent.go @@ -14,8 +14,8 @@ import ( "github.com/spf13/cobra" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/agent" - auc "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecase/agent" repo "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/repository" + auc "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecases/agent" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -110,7 +110,7 @@ func init() { func startAgent(ctx context.Context) { metricStorage := repo.NewMemStorage() - agentUsecase := agent.NewAgent(&auc.AgentUsecase{Storage: metricStorage}) + agentUsecase := agent.NewAgent(auc.NewAgentUsecase(metricStorage, metricStorage)) client := resty.New(). SetTimeout(5 * time.Second). diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 5811c2b..dc684ab 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -16,8 +16,8 @@ import ( "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/router" - srvUsecase "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecase/server" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecase/ping" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecases/ping" + srvUsecase "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecases/server" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -119,7 +119,7 @@ func startServer(ctx context.Context, opts *config.Options) error { } }() - metricUsecase := srvUsecase.NewMetricUsecase(collector) + metricUsecase := srvUsecase.NewMetricUsecase(collector, collector, collector) var pingUsecase *ping.PingUsecase if pinger, ok := collector.(ping.Pinger); ok { diff --git a/internal/collector/config/config.go b/internal/collector/config/config.go index fd1ff72..38e2ee0 100644 --- a/internal/collector/config/config.go +++ b/internal/collector/config/config.go @@ -4,9 +4,9 @@ import ( "context" "fmt" - col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" repo "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/repository" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecases/server" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" ) @@ -15,9 +15,9 @@ type Params struct { Opts *config.Options } -func NewCollector(params *Params) (col.Collector, error) { +func NewCollector(params *Params) (server.Collector, error) { var ( - collector col.Collector + collector server.Collector err error ) diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 63a2d87..bddaf54 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -13,9 +13,9 @@ import ( "github.com/go-resty/resty/v2" "github.com/mailru/easyjson" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecase/agent" - col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecases/agent" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/converter" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" rt "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/runtime-stats" serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" @@ -29,7 +29,7 @@ func NewAgent(uc *agent.AgentUsecase) *Agent { return &Agent{Usecase: uc} } -func UpdateAllMetrics(ctx context.Context, storage col.Collector) { +func UpdateAllMetrics(ctx context.Context, storage agent.MetricUpdater) { var memStats runtime.MemStats runtime.ReadMemStats(&memStats) @@ -56,24 +56,25 @@ func UpdateAllMetrics(ctx context.Context, storage col.Collector) { } } -func (auc *Agent) SendAllMetrics(ctx context.Context, client *resty.Client) { - allMetrics, err := auc.Usecase.GetAllMetrics(ctx) +func (ag *Agent) SendAllMetrics(ctx context.Context, client *resty.Client) { + allMetrics, err := ag.Usecase.GetAllMetrics(ctx) if err != nil { log.Error().Err(err).Msg("failed to Get metrics") } - log.Debug().Msgf("send all metrics: got %d metrics", len(allMetrics)) - for _, metric := range allMetrics { - jsonMetric, err := auc.Usecase.GetMetricJSON(ctx, metric) - if err != nil { - log.Error().Err(err).Msg("failed to get json metric") - } + metricsToSend, err := converter.ConvertToSerialization(allMetrics) + if err != nil { + log.Error().Err(err).Msg("failed to convert metrics to serialization") + } - sendMetric(client, jsonMetric) + if len(metricsToSend) > 0 { + sendBatch(client, metricsToSend) } + + log.Info().Int("count", len(metricsToSend)).Msg("Sending metrics batch") } -func sendMetric(client *resty.Client, metricJSON *serialize.Metric) { +func sendBatch(client *resty.Client, metrics []serialize.Metric) { backoffSchedule := []time.Duration{ 100 * time.Millisecond, 500 * time.Millisecond, @@ -81,7 +82,7 @@ func sendMetric(client *resty.Client, metricJSON *serialize.Metric) { } log.Debug().Msg("sendMetric") - buf, ok, err := ConvertToGzipData(metricJSON) + buf, ok, err := ConvertToGzipData(metrics) if err != nil { log.Error().Err(err).Msg("Failed to convert metric to gzip") return @@ -96,7 +97,7 @@ func sendMetric(client *resty.Client, metricJSON *serialize.Metric) { req.SetHeader("Content-Encoding", "gzip") } - res, err := req.Post("update/") + res, err := req.Post("updates/") if err != nil || res.StatusCode() != http.StatusOK { } else { @@ -107,19 +108,20 @@ func sendMetric(client *resty.Client, metricJSON *serialize.Metric) { } } -func ConvertToGzipData(metricJSON *serialize.Metric) (*bytes.Buffer, bool, error) { - jsonData, err := easyjson.Marshal(*metricJSON) +func ConvertToGzipData(metrics serialize.MetricsList) (*bytes.Buffer, bool, error) { + var jsonBuf bytes.Buffer + + _, err := easyjson.MarshalToWriter(metrics, &jsonBuf) if err != nil { - log.Error().Err(err).Msg("Failed to marshal metricJSON") + log.Error().Err(err).Msg("Failed to marshal metrics") return nil, false, err } - var buf bytes.Buffer - if len(jsonData) <= 1024 { - buf.Write(jsonData) - return &buf, false, nil + if jsonBuf.Len() <= 1024 { + return &jsonBuf, false, nil } + var buf bytes.Buffer gz, err := gzip.NewWriterLevel(&buf, gzip.BestSpeed) if err != nil { log.Error().Err(err).Msg("Failed to create gzip writer") @@ -132,7 +134,7 @@ func ConvertToGzipData(metricJSON *serialize.Metric) (*bytes.Buffer, bool, error } }() - _, err = gz.Write(jsonData) + _, err = gz.Write(jsonBuf.Bytes()) if err != nil { log.Error().Err(err).Msg("Failed to write gzip data") return nil, false, err diff --git a/internal/handlers/server/server.go b/internal/handlers/server/server.go index ead24d8..ebfbcaf 100644 --- a/internal/handlers/server/server.go +++ b/internal/handlers/server/server.go @@ -11,22 +11,22 @@ import ( "github.com/go-chi/chi/v5" "github.com/mailru/easyjson" - srvUsecase "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecase/server" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecase/ping" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecases/ping" + srvUsecase "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecases/server" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/converter" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" ) type Server struct { - Usecase *srvUsecase.MetricUsecase - PingUsecase *ping.PingUsecase + MetricUsecase *srvUsecase.MetricUsecase + PingUsecase *ping.PingUsecase } func NewServer(uc *srvUsecase.MetricUsecase, puc *ping.PingUsecase) *Server { return &Server{ - Usecase: uc, - PingUsecase: puc, + MetricUsecase: uc, + PingUsecase: puc, } } @@ -35,7 +35,7 @@ func (srv *Server) GetMetric() http.HandlerFunc { mType := chi.URLParam(req, "mType") mName := chi.URLParam(req, "mName") - metric, err := srv.Usecase.GetMetric(req.Context(), mType, mName) + metric, err := srv.MetricUsecase.GetMetric(req.Context(), mType, mName) if err != nil { log.Error().Err(err).Msg("can't get valid metric") http.Error(res, fmt.Sprintf("Metric %s was not found", mName), http.StatusNotFound) @@ -66,13 +66,20 @@ func (srv *Server) GetMetric() http.HandlerFunc { func (srv *Server) GetAllMetrics() http.HandlerFunc { return func(res http.ResponseWriter, req *http.Request) { - metricsToTable, err := srv.Usecase.GetAllMetrics(req.Context()) + metrics, err := srv.MetricUsecase.GetAllMetrics(req.Context()) if err != nil { log.Error().Err(err).Msg("failed to Get metrics") http.Error(res, "failed to get metrics", http.StatusNotFound) return } + metricsToTable, err := converter.ConvertToMetricTable(metrics) + if err != nil { + log.Error().Err(err).Msg("failed to convert metrics to table") + http.Error(res, "failed to convert metrics to table", http.StatusInternalServerError) + return + } + const htmlTemplate = ` @@ -145,7 +152,7 @@ func (srv *Server) UpdateMetric() http.HandlerFunc { return } - if err := srv.Usecase.UpdateMetric(req.Context(), mType, mName, val); err != nil { + if err := srv.MetricUsecase.UpdateMetric(req.Context(), mType, mName, val); err != nil { http.Error(res, err.Error(), http.StatusBadRequest) return } @@ -174,7 +181,7 @@ func (srv *Server) GetMetricsHandlerJSON() http.HandlerFunc { } log.Debug().Str("type", jsonMetric.MType).Str("name", jsonMetric.ID).Msg("") - metric, err := srv.Usecase.GetMetric(req.Context(), jsonMetric.MType, jsonMetric.ID) + metric, err := srv.MetricUsecase.GetMetric(req.Context(), jsonMetric.MType, jsonMetric.ID) if err != nil { log.Error().Err(err).Msg("can't get valid metric") http.Error(resp, "can't get valid metric", http.StatusNotFound) @@ -235,12 +242,12 @@ func (srv *Server) UpdateMetricsHandlerJSON() http.HandlerFunc { return } - if err := srv.Usecase.UpdateMetric(req.Context(), jsonMetric.MType, jsonMetric.ID, value); err != nil { + if err := srv.MetricUsecase.UpdateMetric(req.Context(), jsonMetric.MType, jsonMetric.ID, value); err != nil { http.Error(resp, fmt.Sprintf("invalid update metric %s: %v", jsonMetric.ID, err), http.StatusBadRequest) return } - newMetric, err := srv.Usecase.GetMetric(req.Context(), jsonMetric.MType, jsonMetric.ID) + newMetric, err := srv.MetricUsecase.GetMetric(req.Context(), jsonMetric.MType, jsonMetric.ID) if err != nil { log.Error().Err(err).Msg("can't get new value metric") http.Error(resp, "can't get new value metric", http.StatusNotFound) @@ -263,6 +270,23 @@ func (srv *Server) UpdateMetricsHandlerJSON() http.HandlerFunc { func (srv *Server) UpdatesMetricsHandlerJSON() http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { + var reader io.Reader + if req.Header.Get("Content-Encoding") == "gzip" { + gz, err := gzip.NewReader(req.Body) + if err != nil { + http.Error(w, "failed to create gzip reader", http.StatusBadRequest) + return + } + defer func() { + if err := gz.Close(); err != nil { + log.Error().Err(err).Msg("failed close gz reader") + } + }() + reader = gz + } else { + reader = req.Body + } + var jsonMetrics serialize.MetricsList log.Info().Msg("UpdatesMetricsHandlerJSON called") @@ -271,7 +295,7 @@ func (srv *Server) UpdatesMetricsHandlerJSON() http.HandlerFunc { return } - if err := easyjson.UnmarshalFromReader(req.Body, &jsonMetrics); err != nil { + if err := easyjson.UnmarshalFromReader(reader, &jsonMetrics); err != nil { http.Error(w, fmt.Sprintf("invalid json body: %v", err), http.StatusBadRequest) return } @@ -282,7 +306,7 @@ func (srv *Server) UpdatesMetricsHandlerJSON() http.HandlerFunc { return } - if err := srv.Usecase.UpdateMetricList(req.Context(), metrics); err != nil { + if err := srv.MetricUsecase.UpdateMetricList(req.Context(), metrics); err != nil { log.Error().Err(err).Msg("failed update metrics") http.Error(w, fmt.Sprintf("failed update metrics: %v", err), http.StatusInternalServerError) return diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index 9894816..a83cf5a 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -14,7 +14,7 @@ import ( "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" repo "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/repository" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/router" - srvUsecase "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecase/server" + srvUsecase "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecases/server" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" "github.com/stretchr/testify/assert" ) @@ -30,7 +30,8 @@ func TestUpdateMetric(t *testing.T) { opt(opts) } - metricUsecase := srvUsecase.NewMetricUsecase(repo.NewMemStorage()) + storage := repo.NewMemStorage() + metricUsecase := srvUsecase.NewMetricUsecase(storage, storage, storage) router := router.NewRouter(server.NewServer(metricUsecase, nil)) tests := []struct { @@ -90,7 +91,7 @@ func TestUpdateMetric(t *testing.T) { router.ServeHTTP(rr, req) - assert.Equal(t, tt.wantStatus, rr.Code, "Test %s: expected status %d, got %d", tt.name, tt.wantStatus, rr.Code) + assert.Equal(t, tt.wantStatus, rr.Code) }) } } @@ -118,7 +119,7 @@ func TestGetMetric(t *testing.T) { log.Error().Msgf("Failed to update metric requests_total: %v", err) } - metricUsecase := srvUsecase.NewMetricUsecase(storage) + metricUsecase := srvUsecase.NewMetricUsecase(storage, storage, storage) router := router.NewRouter(server.NewServer(metricUsecase, nil)) tests := []struct { @@ -166,9 +167,9 @@ func TestGetMetric(t *testing.T) { router.ServeHTTP(rr, req) - assert.Equal(t, tt.wantStatus, rr.Code, "Test %s: expected status %d, got %d", tt.name, tt.wantStatus, rr.Code) + assert.Equal(t, tt.wantStatus, rr.Code) body, _ := io.ReadAll(rr.Body) - assert.Equal(t, tt.wantBody, string(body), "Test %s: expected body %q, got %q", tt.name, tt.wantBody, string(body)) + assert.Equal(t, tt.wantBody, string(body)) }) } } diff --git a/internal/models/counter_test.go b/internal/models/counter_test.go index 969a7ec..fa36d69 100644 --- a/internal/models/counter_test.go +++ b/internal/models/counter_test.go @@ -77,11 +77,11 @@ func TestCounterUpdate(t *testing.T) { c := models.NewCounter(tt.fields.name, tt.fields.value) err := c.Update(tt.args.mValue) if tt.wantErr { - require.Error(t, err, "Update() error = %v, wantErr = %v", err, tt.wantErr) + require.Error(t, err) } else { - require.NoError(t, err, "Update() error = %v, wantErr = %v", err, tt.wantErr) + require.NoError(t, err) } - assert.Equal(t, tt.wantValue, c.Value(), "Update() value = %v, wantValue = %v", c.Value(), tt.wantValue) + assert.Equal(t, tt.wantValue, c.Value()) }) } } diff --git a/internal/models/gauge_test.go b/internal/models/gauge_test.go index 7ac0cf5..7840498 100644 --- a/internal/models/gauge_test.go +++ b/internal/models/gauge_test.go @@ -53,11 +53,11 @@ func TestGaugeUpdate(t *testing.T) { g := models.NewGauge(tt.fields.name, tt.fields.value) err := g.Update(tt.args.mValue) if tt.wantErr { - require.Error(t, err, "gauge.Update() error = %v, wantErr = %v", err, tt.wantErr) + require.Error(t, err) } else { - require.NoError(t, err, "gauge.Update() error = %v, wantErr = %v", err, tt.wantErr) + require.NoError(t, err) } - assert.Equal(t, tt.want, g.Value(), "gauge.value = %v, want %v", g.Value(), tt.want) + assert.Equal(t, tt.want, g.Value()) }) } } diff --git a/internal/repository/file.go b/internal/repository/file.go index 46b03c6..d02d18c 100644 --- a/internal/repository/file.go +++ b/internal/repository/file.go @@ -8,7 +8,6 @@ import ( "sync" "time" - col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/files" "github.com/rs/zerolog/log" @@ -18,7 +17,7 @@ type FileStorage struct { mutex sync.RWMutex wg sync.WaitGroup filePath string - storage col.Collector + storage *MemStorage SyncRecord bool } @@ -37,7 +36,7 @@ func (fs *FileStorage) save(ctx context.Context) { } } -func NewFileStorage(ctx context.Context, fp *FileParams) (col.Collector, error) { +func NewFileStorage(ctx context.Context, fp *FileParams) (*FileStorage, error) { fs := &FileStorage{ wg: sync.WaitGroup{}, filePath: fp.FileStoragePath, @@ -98,6 +97,15 @@ func (fs *FileStorage) UpdateMetric(ctx context.Context, mType, mName string, mV return nil } +func (fs *FileStorage) UpdateMetricList(ctx context.Context, metrics []models.Metric) error { + if err := fs.storage.UpdateMetricList(ctx, metrics); err != nil { + log.Error().Err(err).Msg("failed update metric list from file storage") + return fmt.Errorf("failed update metric list from file storage %w", err) + } + + return nil +} + func (fs *FileStorage) GetMetric(ctx context.Context, mType, mName string) (models.Metric, error) { metric, err := fs.storage.GetMetric(ctx, mType, mName) if err != nil { diff --git a/internal/repository/memory.go b/internal/repository/memory.go index 512ea5e..e72cb8a 100644 --- a/internal/repository/memory.go +++ b/internal/repository/memory.go @@ -21,13 +21,7 @@ func NewMemStorage() *MemStorage { } } -func (ms *MemStorage) UpdateMetric(_ context.Context, mType, mName string, mValue any) error { - ms.mutex.Lock() - defer ms.mutex.Unlock() - if mName == "" { - return models.ErrMetricsNotFound - } - +func updateMetric(ms *MemStorage, mType, mName string, mValue any) error { if _, ok := ms.storage[mType]; !ok { return models.ErrInvalidMetricsType } @@ -54,6 +48,26 @@ func (ms *MemStorage) UpdateMetric(_ context.Context, mType, mName string, mValu return nil } +func (ms *MemStorage) UpdateMetric(_ context.Context, mType, mName string, mValue any) error { + ms.mutex.Lock() + defer ms.mutex.Unlock() + + return updateMetric(ms, mType, mName, mValue) +} + +func (ms *MemStorage) UpdateMetricList(_ context.Context, metrics []models.Metric) error { + ms.mutex.Lock() + defer ms.mutex.Unlock() + + for _, metric := range metrics { + if err := updateMetric(ms, metric.Type(), metric.Name(), metric.Value()); err != nil { + return err + } + } + + return nil +} + func (ms *MemStorage) GetMetric(_ context.Context, mType, mName string) (models.Metric, error) { ms.mutex.RLock() defer ms.mutex.RUnlock() diff --git a/internal/repository/postgres.go b/internal/repository/postgres.go index e4e5773..6d9e7a5 100644 --- a/internal/repository/postgres.go +++ b/internal/repository/postgres.go @@ -8,7 +8,6 @@ import ( sq "github.com/Masterminds/squirrel" - col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" errH "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/errors-handlers" "github.com/rs/zerolog/log" @@ -18,7 +17,7 @@ type Database struct { DB *sql.DB } -func NewDatabase(ctx context.Context, dataBaseDSN string) (col.Collector, error) { +func NewDatabase(ctx context.Context, dataBaseDSN string) (*Database, error) { log.Info().Msgf("DSN: %s", dataBaseDSN) db, err := sql.Open("pgx", dataBaseDSN) if err != nil { @@ -221,6 +220,16 @@ func (db *Database) UpdateMetric(ctx context.Context, mType, mName string, mValu return tx.Commit() } +func (db *Database) UpdateMetricList(ctx context.Context, metrics []models.Metric) error { + for _, metric := range metrics { + if err := db.UpdateMetric(ctx, metric.Type(), metric.Name(), metric.Value()); err != nil { + return err + } + } + + return nil +} + func (db *Database) Ping(ctx context.Context) error { if err := db.DB.PingContext(ctx); err != nil { return fmt.Errorf("failed ping database: %w", err) diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index b4acec6..ac1ad09 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -109,15 +109,15 @@ func TestMemStorage_UpdateMetric(t *testing.T) { err := ms.UpdateMetric(ctx, tt.args.mType, tt.args.mName, tt.args.mValue) if tt.wantErr { - require.Error(t, err, "MemStorage.UpdateMetric() error = %v, wantErr = %v", err, tt.wantErr) + require.Error(t, err) } else { - require.NoError(t, err, "MemStorage.UpdateMetric() error = %v, wantErr = %v", err, tt.wantErr) + require.NoError(t, err) } if !tt.wantErr { metric, err := ms.GetMetric(ctx, tt.args.mType, tt.args.mName) - require.NoError(t, err, "metric not found") - assert.Equal(t, tt.wantResult, metric.Value(), "got = %v, want = %v", metric.Value(), tt.wantResult) + require.NoError(t, err) + assert.Equal(t, tt.wantResult, metric.Value()) } }) } @@ -190,10 +190,10 @@ func TestMemStorage_GetMetric(t *testing.T) { metric, err := ms.GetMetric(ctx, tt.mType, tt.mName) if tt.wantOk { - require.NoError(t, err, "GetMetric() error = %v, wantOk = %v", err, tt.wantOk) - assert.Equal(t, tt.wantVal, metric.Value(), "GetMetric() gotVal = %v, want %v", metric.Value(), tt.wantVal) + require.NoError(t, err) + assert.Equal(t, tt.wantVal, metric.Value()) } else { - require.Error(t, err, "GetMetric() error = %v, wantOk = %v", err, tt.wantOk) + require.Error(t, err) } }) } @@ -261,7 +261,7 @@ func TestMemStorage_GetAllMetrics(t *testing.T) { gotMap[m.Name()] = m } - assert.Equal(t, len(tt.want), len(gotMap), "wrong number of metrics: got %d, want %d", len(gotMap), len(tt.want)) + assert.Equal(t, len(tt.want), len(gotMap)) for name, wantMetric := range tt.want { gotMetric, ok := gotMap[name] @@ -270,13 +270,13 @@ func TestMemStorage_GetAllMetrics(t *testing.T) { continue } - assert.Equal(t, wantMetric.Type(), gotMetric.Type(), "metric %s type mismatch: got %s, want %s", name, gotMetric.Type(), wantMetric.Type()) + assert.Equal(t, wantMetric.Type(), gotMetric.Type()) switch wantMetric.Type() { case models.CounterType: - assert.NotNil(t, gotMetric.Value(), "metric %s: expected counter delta, got nil", name) + assert.NotNil(t, gotMetric.Value()) case models.GaugeType: - assert.NotNil(t, gotMetric.Value(), "metric %s: expected gauge value, got nil", name) + assert.NotNil(t, gotMetric.Value()) default: assert.Fail(t, "unknown metric type", "metric %s: unknown type %s", name, wantMetric.Type()) } diff --git a/internal/usecase/agent/agent-usecase.go b/internal/usecase/agent/agent-usecase.go deleted file mode 100644 index de20531..0000000 --- a/internal/usecase/agent/agent-usecase.go +++ /dev/null @@ -1,60 +0,0 @@ -package agent - -import ( - "context" - - col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" - log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" - serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" -) - -type AgentUsecase struct { - Storage col.Collector -} - -func NewAgentUsecase(storage col.Collector) *AgentUsecase { - return &AgentUsecase{ - Storage: storage, - } -} - -func (uc *AgentUsecase) GetAllMetrics(ctx context.Context) ([]models.Metric, error) { - return uc.Storage.GetAllMetrics(ctx) -} - -func (uc *AgentUsecase) GetMetricJSON(ctx context.Context, metric models.Metric) (*serialize.Metric, error) { - mType := metric.Type() - mName := metric.Name() - - metricJSON := &serialize.Metric{ - ID: mName, - MType: mType, - } - - switch mType { - case models.GaugeType: - val, ok := metric.Value().(float64) - if !ok { - log.Error().Str("metric_name", mName).Str("metric_type", mType). - Msg("Invalid metric value type") - - return nil, models.ErrInvalidValueType - } - - metricJSON.Value = &val - - case models.CounterType: - val, ok := metric.Value().(int64) - if !ok { - log.Error().Str("metric_name", mName).Str("metric_type", mType). - Msg("Invalid metric value type") - - return nil, models.ErrInvalidValueType - } - - metricJSON.Delta = &val - } - - return metricJSON, nil -} diff --git a/internal/usecase/server/server-usecases.go b/internal/usecase/server/server-usecases.go deleted file mode 100644 index fee6501..0000000 --- a/internal/usecase/server/server-usecases.go +++ /dev/null @@ -1,91 +0,0 @@ -package server - -import ( - "context" - "fmt" - "strconv" - - col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" - "github.com/rs/zerolog/log" -) - -type MetricUsecase struct { - repo col.Collector -} - -func NewMetricUsecase(repo col.Collector) *MetricUsecase { - return &MetricUsecase{repo: repo} -} - -func (uc *MetricUsecase) GetMetric(ctx context.Context, mType, mName string) (models.Metric, error) { - metric, err := uc.repo.GetMetric(ctx, mType, mName) - if err != nil { - return nil, fmt.Errorf("metric not found: %w", err) - } - - return metric, nil -} - -func (uc *MetricUsecase) GetAllMetrics(ctx context.Context) ([]models.MetricTable, error) { - allMetrics, err := uc.repo.GetAllMetrics(ctx) - if err != nil { - log.Error().Err(err).Msg("failed to Get metrics") - return nil, fmt.Errorf("failed to get metrics: %v", err) - } - - var metricsToTable []models.MetricTable - - for _, metric := range allMetrics { - var valStr string - mName := metric.Name() - mType := metric.Type() - - switch mType { - case models.GaugeType: - val, ok := metric.Value().(float64) - if !ok { - log.Error().Str("metric_name", mName).Str("metric_type", mType). - Msg("Invalid metric value type") - continue - } - valStr = strconv.FormatFloat(val, 'f', -1, 64) - - case models.CounterType: - val, ok := metric.Value().(int64) - if !ok { - log.Error().Str("metric_name", mName).Str("metric_type", mType). - Msg("Invalid metric value type") - continue - } - valStr = strconv.FormatInt(val, 10) - } - - metricsToTable = append(metricsToTable, models.MetricTable{ - Name: mName, - Type: mType, - Value: valStr, - }) - } - - return metricsToTable, nil -} - -func (uc *MetricUsecase) UpdateMetric(ctx context.Context, mType, mName string, value any) error { - if err := uc.repo.UpdateMetric(ctx, mType, mName, value); err != nil { - return fmt.Errorf("failed to update metric: %w", err) - } - - return nil -} - -func (uc *MetricUsecase) UpdateMetricList(ctx context.Context, metrics []models.Metric) error { - for _, metric := range metrics { - if err := uc.UpdateMetric(ctx, metric.Type(), metric.Name(), metric.Value()); err != nil { - return fmt.Errorf("failed to update metric: %w", err) - } - } - - return nil -} - diff --git a/internal/usecases/agent/agent-usecase.go b/internal/usecases/agent/agent-usecase.go new file mode 100644 index 0000000..a2f4629 --- /dev/null +++ b/internal/usecases/agent/agent-usecase.go @@ -0,0 +1,27 @@ +package agent + +import ( + "context" + + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" +) + +type AgentUsecase struct { + updater MetricUpdater + getter MetricGetter +} + +func NewAgentUsecase(u MetricUpdater, g MetricGetter) *AgentUsecase { + return &AgentUsecase{ + updater: u, + getter: g, + } +} + +func (uc *AgentUsecase) GetAllMetrics(ctx context.Context) ([]models.Metric, error) { + return uc.getter.GetAllMetrics(ctx) +} + +func (uc *AgentUsecase) GetMetric(ctx context.Context, mType, mName string) (models.Metric, error) { + return uc.getter.GetMetric(ctx, mType, mName) +} diff --git a/internal/collector/collector.go b/internal/usecases/agent/deps.go similarity index 68% rename from internal/collector/collector.go rename to internal/usecases/agent/deps.go index e344eb8..943335d 100644 --- a/internal/collector/collector.go +++ b/internal/usecases/agent/deps.go @@ -1,4 +1,4 @@ -package collector +package agent import ( "context" @@ -6,10 +6,12 @@ import ( "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" ) -type Collector interface { +type MetricGetter interface { GetMetric(ctx context.Context, mType, mName string) (models.Metric, error) GetAllMetrics(ctx context.Context) ([]models.Metric, error) - UpdateMetric(ctx context.Context, mType, mName string, mValue any) error +} - Close() error +type MetricUpdater interface { + UpdateMetric(ctx context.Context, mType, mName string, mValue any) error + UpdateMetricList(ctx context.Context, metrics []models.Metric) error } diff --git a/internal/usecases/ping/deps.go b/internal/usecases/ping/deps.go new file mode 100644 index 0000000..5223d20 --- /dev/null +++ b/internal/usecases/ping/deps.go @@ -0,0 +1,13 @@ +package ping + +import ( + "context" +) + +type Pinger interface { + Ping(ctx context.Context) error +} + +type Closer interface { + Close() error +} diff --git a/internal/usecase/ping/ping-usecase.go b/internal/usecases/ping/ping-usecase.go similarity index 68% rename from internal/usecase/ping/ping-usecase.go rename to internal/usecases/ping/ping-usecase.go index 9b80216..12c1935 100644 --- a/internal/usecase/ping/ping-usecase.go +++ b/internal/usecases/ping/ping-usecase.go @@ -4,14 +4,6 @@ import ( "context" ) -type Pinger interface { - Ping(ctx context.Context) error -} - -type Usecase interface { - Check(ctx context.Context) error -} - type PingUsecase struct { pinger Pinger } diff --git a/internal/usecases/server/deps.go b/internal/usecases/server/deps.go new file mode 100644 index 0000000..e86836f --- /dev/null +++ b/internal/usecases/server/deps.go @@ -0,0 +1,27 @@ +package server + +import ( + "context" + + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" +) + +type MetricGetter interface { + GetMetric(ctx context.Context, mType, mName string) (models.Metric, error) + GetAllMetrics(ctx context.Context) ([]models.Metric, error) +} + +type MetricUpdater interface { + UpdateMetric(ctx context.Context, mType, mName string, mValue any) error + UpdateMetricList(ctx context.Context, metrics []models.Metric) error +} + +type Closer interface { + Close() error +} + +type Collector interface { + MetricGetter + MetricUpdater + Closer +} diff --git a/internal/usecases/server/server-usecases.go b/internal/usecases/server/server-usecases.go new file mode 100644 index 0000000..ff36af1 --- /dev/null +++ b/internal/usecases/server/server-usecases.go @@ -0,0 +1,56 @@ +package server + +import ( + "context" + "fmt" + + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" +) + +type MetricUsecase struct { + getter MetricGetter + updater MetricUpdater + closer Closer +} + +func NewMetricUsecase(g MetricGetter, u MetricUpdater, c Closer) *MetricUsecase { + return &MetricUsecase{ + getter: g, + updater: u, + closer: c, + } +} + +func (uc *MetricUsecase) GetMetric(ctx context.Context, mType, mName string) (models.Metric, error) { + metric, err := uc.getter.GetMetric(ctx, mType, mName) + if err != nil { + return nil, fmt.Errorf("metric not found: %w", err) + } + + return metric, nil +} + +func (uc *MetricUsecase) GetAllMetrics(ctx context.Context) ([]models.Metric, error) { + allMetrics, err := uc.getter.GetAllMetrics(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get all metrics: %w", err) + } + + return allMetrics, nil +} + +func (uc *MetricUsecase) UpdateMetric(ctx context.Context, mType, mName string, value any) error { + if err := uc.updater.UpdateMetric(ctx, mType, mName, value); err != nil { + return fmt.Errorf("failed to update metric: %w", err) + } + + return nil +} + +func (uc *MetricUsecase) UpdateMetricList(ctx context.Context, metrics []models.Metric) error { + if err := uc.updater.UpdateMetricList(ctx, metrics); err != nil { + return fmt.Errorf("failed to update metric list: %w", err) + } + + return nil +} diff --git a/pkg/converter/converter.go b/pkg/converter/converter.go index 639296c..b1cb3c2 100644 --- a/pkg/converter/converter.go +++ b/pkg/converter/converter.go @@ -28,6 +28,46 @@ func ConvertByType(mType, mValue string) (any, error) { } } +func ConvertToMetricTable(src []models.Metric) ([]models.MetricTable, error) { + converted := make([]models.MetricTable, 0, len(src)) + + for _, metric := range src { + var valStr string + mName := metric.Name() + mType := metric.Type() + + switch mType { + case models.GaugeType: + val, ok := metric.Value().(float64) + if !ok { + log.Error().Str("metric_name", mName).Str("metric_type", mType). + Msg("Invalid metric value type") + + return nil, fmt.Errorf("invalid metric value type: %s", mType) + } + valStr = strconv.FormatFloat(val, 'f', -1, 64) + + case models.CounterType: + val, ok := metric.Value().(int64) + if !ok { + log.Error().Str("metric_name", mName).Str("metric_type", mType). + Msg("Invalid metric value type") + + return nil, fmt.Errorf("invalid metric value type: %s", mType) + } + valStr = strconv.FormatInt(val, 10) + } + + converted = append(converted, models.MetricTable{ + Name: mName, + Type: mType, + Value: valStr, + }) + } + + return converted, nil +} + func ConvertMetrics(src serialize.MetricsList) ([]models.Metric, error) { converted := make([]models.Metric, 0, len(src)) @@ -99,13 +139,12 @@ func сonvertToSerialization(src models.Metric) (serialize.Metric, error) { return converted, nil } -func ConverToSerialization(src []models.Metric) ([]serialize.Metric, error) { +func ConvertToSerialization(src []models.Metric) ([]serialize.Metric, error) { converted := make([]serialize.Metric, 0, len(src)) for _, mtr := range src { metric, err := сonvertToSerialization(mtr) if err != nil { - log.Error().Err(err).Msg("failed convert metric") return nil, fmt.Errorf("metric failed convert %+v", mtr) } diff --git a/pkg/files/file-storage.go b/pkg/files/file-storage.go index a3ac516..eaeebb9 100644 --- a/pkg/files/file-storage.go +++ b/pkg/files/file-storage.go @@ -7,22 +7,23 @@ import ( "os" "path/filepath" - col "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector" + server "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecases/server" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/converter" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" ) -func SaveToDB(ctx context.Context, collector col.Collector, path string) error { - allMetrics, err := collector.GetAllMetrics(ctx) +func SaveToDB(ctx context.Context, getter server.MetricGetter, path string) error { + allMetrics, err := getter.GetAllMetrics(ctx) if err != nil { log.Error().Err(err).Msg("failed to Get metrics") } data := make(serialize.MetricsList, 0, len(allMetrics)) - jsonMetrics, err := converter.ConverToSerialization(allMetrics) + jsonMetrics, err := converter.ConvertToSerialization(allMetrics) if err != nil { + log.Error().Err(err).Msg("failed convert metric") return fmt.Errorf("convert metrics error: %w", err) } @@ -36,7 +37,7 @@ func SaveToDB(ctx context.Context, collector col.Collector, path string) error { return fmt.Errorf("json marshal error: %w", err) } - err = WriteFileAtomic(path, bytes) + err = WriteFile(path, bytes) if err != nil { return fmt.Errorf("write file atomic error: %w", err) } @@ -49,14 +50,12 @@ func SaveToDB(ctx context.Context, collector col.Collector, path string) error { return nil } -func WriteFileAtomic(path string, data []byte) error { +func WriteFile(path string, data []byte) error { if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { return fmt.Errorf("failed to create directory: %w", err) } - tmpPath := path + ".tmp" - - file, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return fmt.Errorf("failed to create temp file: %w", err) } @@ -74,15 +73,10 @@ func WriteFileAtomic(path string, data []byte) error { return fmt.Errorf("sync failed: %w", err) } - if err := os.Rename(tmpPath, path); err != nil { - return fmt.Errorf("rename failed: %w", err) - } - return nil } - -func LoadFromDB(ctx context.Context, collector col.Collector, path string) error { +func LoadFromDB(ctx context.Context, updater server.MetricUpdater, path string) error { log.Info(). Str("path", path). Msg("Trying to load metrics from file") @@ -109,7 +103,7 @@ func LoadFromDB(ctx context.Context, collector col.Collector, path string) error } for _, metric := range metrics { - if err := collector.UpdateMetric(ctx, metric.Type(), metric.Name(), metric.Value()); err != nil { + if err := updater.UpdateMetric(ctx, metric.Type(), metric.Name(), metric.Value()); err != nil { log.Error().Err(err).Msg("update metric error") return fmt.Errorf("update metric error %w", err) } From b69c2e381ef5a5df2c6e749792d7fd443dc1cc93 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Sat, 26 Jul 2025 15:06:05 +0300 Subject: [PATCH 119/126] add hashing --- .github/workflows/metricstest.yml | 2 + cmd/agent/cfg_agent.go | 12 ++- cmd/server/cfg_server.go | 8 +- internal/config/server-config.go | 21 ++++ internal/handlers/agent/agent.go | 25 ++++- internal/handlers/server/hashing.go | 77 +++++++++++++++ internal/handlers/server/server_test.go | 4 +- internal/router/router.go | 7 +- pkg/hash/hash.go | 30 ++++++ pkg/hash/hash_test.go | 126 ++++++++++++++++++++++++ 10 files changed, 302 insertions(+), 10 deletions(-) create mode 100644 internal/handlers/server/hashing.go create mode 100644 pkg/hash/hash.go create mode 100644 pkg/hash/hash_test.go diff --git a/.github/workflows/metricstest.yml b/.github/workflows/metricstest.yml index 5390604..c6afc84 100644 --- a/.github/workflows/metricstest.yml +++ b/.github/workflows/metricstest.yml @@ -463,6 +463,7 @@ jobs: - name: "Code increment #14" if: | + github.ref == 'refs/heads/main' || github.head_ref == 'iter14' || github.head_ref == 'iter15' || github.head_ref == 'iter16' || @@ -488,6 +489,7 @@ jobs: - name: "Code increment #14 (race detection)" if: | + github.ref == 'refs/heads/main' || github.head_ref == 'iter14' || github.head_ref == 'iter15' || github.head_ref == 'iter16' || diff --git a/cmd/agent/cfg_agent.go b/cmd/agent/cfg_agent.go index 4218b7e..a07ae43 100644 --- a/cmd/agent/cfg_agent.go +++ b/cmd/agent/cfg_agent.go @@ -23,18 +23,21 @@ const ( defaultEndpoint = "localhost:8080" defaultPollInterval = 2 defaultReportInterval = 10 + defaultKey = "key" ) type options struct { endPointAddr string pollInterval int reportInterval int + key string } type envConfig struct { EndPointAddr string `env:"ADDRESS"` PollInterval int `env:"POLL_INTERVAL"` ReportInterval int `env:"REPORT_INTERVAL"` + Key string `env:"KEY"` } var opts = &options{} @@ -73,6 +76,10 @@ var rootCmd = &cobra.Command{ opts.reportInterval = cfg.ReportInterval } + if cfg.Key != "" { + opts.key = cfg.Key + } + if opts.pollInterval <= 0 || opts.reportInterval <= 0 { return fmt.Errorf("poll and report intervals must be > 0") } @@ -102,10 +109,12 @@ func init() { opts.endPointAddr = defaultEndpoint opts.pollInterval = defaultPollInterval opts.reportInterval = defaultReportInterval + opts.key = defaultKey rootCmd.Flags().StringVarP(&opts.endPointAddr, "a", "a", opts.endPointAddr, "endpoint HTTP-server addr") rootCmd.Flags().IntVarP(&opts.pollInterval, "p", "p", opts.pollInterval, "PollInterval value") rootCmd.Flags().IntVarP(&opts.reportInterval, "r", "r", opts.reportInterval, "PollInterval value") + rootCmd.Flags().StringVarP(&opts.key, "k", "k", opts.key, "key for hash") } func startAgent(ctx context.Context) { @@ -117,6 +126,7 @@ func startAgent(ctx context.Context) { SetBaseURL("http://" + opts.endPointAddr) log.Info().Msg("Starting collection and reporting loops") + log.Debug().Str("key", opts.key).Msg("") pollTimer := time.NewTicker(time.Duration(opts.pollInterval) * time.Second) reportTimer := time.NewTicker(time.Duration(opts.reportInterval) * time.Second) @@ -132,7 +142,7 @@ func startAgent(ctx context.Context) { case <-pollTimer.C: agent.UpdateAllMetrics(ctx, metricStorage) case <-reportTimer.C: - agentUsecase.SendAllMetrics(ctx, client) + agentUsecase.SendAllMetrics(ctx, client, opts.key) } } } diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 46b72ca..22457b8 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -27,6 +27,7 @@ var ( fileStoragePath string restoreOnStart bool dataBaseDSN string + key string opts *config.Options ) @@ -45,6 +46,7 @@ func init() { rootCmd.Flags().StringVarP(&fileStoragePath, "f", "f", config.DefaultFileStoragePath, "file to store metrics") rootCmd.Flags().BoolVarP(&restoreOnStart, "r", "r", config.DefaultRestoreOnStart, "restore metrics from file on start") rootCmd.Flags().StringVarP(&dataBaseDSN, "d", "d", config.DefaultDataBaseDSN, "database dsn") + rootCmd.Flags().StringVarP(&key, "k", "k", config.DefaultKey, "key for hash") } func preRunE(cmd *cobra.Command, args []string) error { @@ -54,7 +56,8 @@ func preRunE(cmd *cobra.Command, args []string) error { StoreInterval: storeInterval, FileStoragePath: fileStoragePath, RestoreOnStart: restoreOnStart, - DataBaseDSN: dataBaseDSN}) + DataBaseDSN: dataBaseDSN, + Key: key,}) opts = config.NewServerOptions( config.WithAddress(opts.EndPointAddr), @@ -62,6 +65,7 @@ func preRunE(cmd *cobra.Command, args []string) error { config.WithFileStoragePath(opts.FileStoragePath), config.WithRestoreOnStart(opts.RestoreOnStart), config.WithDataBaseDSN(opts.DataBaseDSN), + config.WithKey(opts.Key), ) return err @@ -128,7 +132,7 @@ func startServer(ctx context.Context, opts *config.Options) error { pingUsecase = nil } - r := router.NewRouter(server.NewServer(metricUsecase, pingUsecase)) + r := router.NewRouter(server.NewServer(metricUsecase, pingUsecase), opts) srv := &http.Server{ Addr: opts.EndPointAddr, diff --git a/internal/config/server-config.go b/internal/config/server-config.go index 4c78d2a..e5735e2 100644 --- a/internal/config/server-config.go +++ b/internal/config/server-config.go @@ -15,6 +15,7 @@ const ( DefaultFileStoragePath = "" DefaultRestoreOnStart = true DefaultDataBaseDSN = "" + DefaultKey = "" ) type Options struct { @@ -23,6 +24,7 @@ type Options struct { FileStoragePath string RestoreOnStart bool DataBaseDSN string + Key string } type EnvConfig struct { @@ -31,6 +33,7 @@ type EnvConfig struct { FileStoragePath string `env:"FILE_STORAGE_PATH"` RestoreOnStart bool `env:"RESTORE"` DataBaseDSN string `env:"DATABASE_DSN"` + Key string `env:"KEY"` } type Option func(*Options) @@ -42,6 +45,7 @@ func NewServerOptions(options ...Option) *Options { FileStoragePath: DefaultFileStoragePath, RestoreOnStart: DefaultRestoreOnStart, DataBaseDSN: DefaultDataBaseDSN, + Key: DefaultKey, } for _, opt := range options { @@ -81,6 +85,12 @@ func WithDataBaseDSN(dsn string) Option { } } +func WithKey(key string) Option { + return func(o *Options) { + o.Key = key + } +} + func ParseOptionsFromCmdAndEnvs(cmd *cobra.Command, src *Options) (*Options, error) { opts, err := ParseFlags(cmd, src) if err != nil { @@ -123,6 +133,12 @@ func ParseFlags(cmd *cobra.Command, src *Options) (*Options, error) { opts.RestoreOnStart = src.RestoreOnStart } + if cmd.Flags().Changed("k") { + if src.Key != "" { + opts.Key = src.Key + } + } + return &opts, nil } @@ -146,6 +162,11 @@ func ParseEnvs(cmd *cobra.Command, opts *Options) error { if envCfg.FileStoragePath != "" { opts.FileStoragePath = envCfg.FileStoragePath } + + if envCfg.Key != "" { + opts.Key = envCfg.Key + } + opts.RestoreOnStart = envCfg.RestoreOnStart return nil diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index bddaf54..b29cdc9 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -4,6 +4,7 @@ import ( "bytes" "compress/gzip" "context" + "encoding/hex" "fmt" "math/rand" "net/http" @@ -16,6 +17,7 @@ import ( "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecases/agent" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/converter" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/hash" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" rt "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/runtime-stats" serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" @@ -56,7 +58,7 @@ func UpdateAllMetrics(ctx context.Context, storage agent.MetricUpdater) { } } -func (ag *Agent) SendAllMetrics(ctx context.Context, client *resty.Client) { +func (ag *Agent) SendAllMetrics(ctx context.Context, client *resty.Client, key string) { allMetrics, err := ag.Usecase.GetAllMetrics(ctx) if err != nil { log.Error().Err(err).Msg("failed to Get metrics") @@ -68,13 +70,13 @@ func (ag *Agent) SendAllMetrics(ctx context.Context, client *resty.Client) { } if len(metricsToSend) > 0 { - sendBatch(client, metricsToSend) + sendBatch(client, metricsToSend, key) } log.Info().Int("count", len(metricsToSend)).Msg("Sending metrics batch") } -func sendBatch(client *resty.Client, metrics []serialize.Metric) { +func sendBatch(client *resty.Client, metrics []serialize.Metric, key string) { backoffSchedule := []time.Duration{ 100 * time.Millisecond, 500 * time.Millisecond, @@ -88,15 +90,30 @@ func sendBatch(client *resty.Client, metrics []serialize.Metric) { return } + var h string + if key != "" { + hashBytes, err := hash.GetHash([]byte(key), buf.Bytes()) + if err != nil { + log.Error().Err(err).Msg("can't get hash") + return + } + + h = hex.EncodeToString(hashBytes) + } + for _, backoff := range backoffSchedule { req := client.R(). SetHeader("Content-Type", "application/json"). - SetBody(buf) + SetBody(buf.Bytes()) if ok { req.SetHeader("Content-Encoding", "gzip") } + if h != "" { + req.SetHeader("HashSHA256", h) + } + res, err := req.Post("updates/") if err != nil || res.StatusCode() != http.StatusOK { diff --git a/internal/handlers/server/hashing.go b/internal/handlers/server/hashing.go new file mode 100644 index 0000000..0725409 --- /dev/null +++ b/internal/handlers/server/hashing.go @@ -0,0 +1,77 @@ +package server + +import ( + "bytes" + "encoding/hex" + "fmt" + "io" + "net/http" + + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/hash" + log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" +) + +type hashResponseWriter struct { + http.ResponseWriter + body *bytes.Buffer + key []byte +} + +func (hsw *hashResponseWriter) Write(b []byte) (int, error) { + if size, err := hsw.body.Write(b); err != nil { + return size, fmt.Errorf("failed write body %v", err) + } + + if hsw.key != nil && hsw.body.Len() > 0 { + newHash, err := hash.GetHash(hsw.key, hsw.body.Bytes()) + if err != nil { + log.Error().Err(err).Msg("failed to get hash") + } + + hsw.Header().Set("HashSHA256", hex.EncodeToString(newHash)) + } + + return hsw.ResponseWriter.Write(b) +} + +func WithHashing(key []byte) func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Body != nil { + body, err := io.ReadAll(r.Body) + if err != nil { + log.Error().Err(err).Msg("failed read body") + http.Error(w, fmt.Sprintf("failed read body %v", err), http.StatusInternalServerError) + return + } + + r.Body = io.NopCloser(bytes.NewBuffer(body)) + + h := r.Header.Get("HashSHA256") + if h != "" { + decoded, err := hex.DecodeString(h) + if err != nil { + log.Error().Err(err).Msg("failed to decode hash") + http.Error(w, "invalid hash format", http.StatusBadRequest) + return + } + valid := hash.CheckHash(key, body, decoded) + if !valid { + log.Error().Msg("invalid hash message") + http.Error(w, "invalid hash message", http.StatusBadRequest) + + return + } + } + + hw := &hashResponseWriter{ + ResponseWriter: w, + body: bytes.NewBuffer(nil), + key: key, + } + + next.ServeHTTP(hw, r) + } + }) + } +} diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index a83cf5a..5ff871b 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -32,7 +32,7 @@ func TestUpdateMetric(t *testing.T) { storage := repo.NewMemStorage() metricUsecase := srvUsecase.NewMetricUsecase(storage, storage, storage) - router := router.NewRouter(server.NewServer(metricUsecase, nil)) + router := router.NewRouter(server.NewServer(metricUsecase, nil), opts) tests := []struct { name string @@ -120,7 +120,7 @@ func TestGetMetric(t *testing.T) { } metricUsecase := srvUsecase.NewMetricUsecase(storage, storage, storage) - router := router.NewRouter(server.NewServer(metricUsecase, nil)) + router := router.NewRouter(server.NewServer(metricUsecase, nil), opts) tests := []struct { name string diff --git a/internal/router/router.go b/internal/router/router.go index 34166f4..a55f4ee 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -5,15 +5,20 @@ import ( "github.com/go-chi/chi/v5" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" ) -func NewRouter(srv *server.Server) http.Handler { +func NewRouter(srv *server.Server, opts *config.Options) http.Handler { r := chi.NewRouter() r.Use(server.WithLogging) r.Use(server.WithGzipCompress) + if opts.Key != "" { + r.Use(server.WithHashing([]byte(opts.Key))) + } + r.Route("/", func(r chi.Router) { r.Get("/", srv.GetAllMetrics()) r.Route("/update", func(r chi.Router) { diff --git a/pkg/hash/hash.go b/pkg/hash/hash.go new file mode 100644 index 0000000..984ef11 --- /dev/null +++ b/pkg/hash/hash.go @@ -0,0 +1,30 @@ +package hash + +import ( + "crypto/hmac" + "crypto/sha256" + "fmt" + + log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" +) + +func GetHash(key, data []byte) ([]byte, error) { + h := hmac.New(sha256.New, key) + + if _, err := h.Write(data); err != nil { + log.Error().Err(err).Msg("invalid hashing sha256") + return nil, fmt.Errorf("invalid hashing sha256 %v", err) + } + + return h.Sum(nil), nil +} + +func CheckHash(key, data, expdHash []byte) bool { + hash, err := GetHash(key, data) + if err != nil { + log.Error().Err(err).Msg("cat't get hash") + return false + } + + return hmac.Equal(hash, expdHash) +} diff --git a/pkg/hash/hash_test.go b/pkg/hash/hash_test.go new file mode 100644 index 0000000..c2974e2 --- /dev/null +++ b/pkg/hash/hash_test.go @@ -0,0 +1,126 @@ +package hash_test + +import ( + "crypto/hmac" + "crypto/sha256" + "reflect" + "testing" + + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/hash" +) + +func TestGetHash(t *testing.T) { + type args struct { + key []byte + data []byte + } + tests := []struct { + name string + args args + want []byte + wantErr bool + }{ + { + name: "Valid HMAC SHA256 hash", + args: args{ + key: []byte("secret"), + data: []byte("hello world"), + }, + want: func() []byte { + h := hmac.New(sha256.New, []byte("secret")) + h.Write([]byte("hello world")) + return h.Sum(nil) + }(), + wantErr: false, + }, + { + name: "Empty key and data", + args: args{ + key: []byte{}, + data: []byte{}, + }, + want: func() []byte { + h := hmac.New(sha256.New, []byte{}) + h.Write([]byte{}) + return h.Sum(nil) + }(), + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := hash.GetHash(tt.args.key, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("GetHash() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetHash() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestCheckHash(t *testing.T) { + type args struct { + key []byte + data []byte + expdHash []byte + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "Correct hash comparison", + args: func() args { + key := []byte("key123") + data := []byte("message") + h := hmac.New(sha256.New, key) + h.Write(data) + expected := h.Sum(nil) + return args{key: key, data: data, expdHash: expected} + }(), + want: true, + }, + { + name: "Incorrect hash", + args: func() args { + key := []byte("key123") + data := []byte("message") + wrongHash := []byte("invalidhash") + return args{key: key, data: data, expdHash: wrongHash} + }(), + want: false, + }, + { + name: "Empty key/data but correct expected hash", + args: func() args { + key := []byte{} + data := []byte{} + h := hmac.New(sha256.New, key) + h.Write(data) + expected := h.Sum(nil) + return args{key: key, data: data, expdHash: expected} + }(), + want: true, + }, + { + name: "Nil expected hash", + args: args{ + key: []byte("somekey"), + data: []byte("somedata"), + expdHash: nil, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := hash.CheckHash(tt.args.key, tt.args.data, tt.args.expdHash); got != tt.want { + t.Errorf("CheckHash() = %v, want %v", got, tt.want) + } + }) + } +} From 22239c626ca891a1fd4258ad3ce003749537c40a Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Sun, 27 Jul 2025 11:59:01 +0300 Subject: [PATCH 120/126] add worker-pool, +14test --- cmd/agent/cfg_agent.go | 37 ++++++++++++--------- go.mod | 5 +++ go.sum | 11 ++++++ internal/handlers/agent/agent.go | 57 ++++++++++++++++++++++++++++++++ pkg/worker-pool/pool.go | 57 ++++++++++++++++++++++++++++++++ 5 files changed, 151 insertions(+), 16 deletions(-) create mode 100644 pkg/worker-pool/pool.go diff --git a/cmd/agent/cfg_agent.go b/cmd/agent/cfg_agent.go index a07ae43..fc7c7f9 100644 --- a/cmd/agent/cfg_agent.go +++ b/cmd/agent/cfg_agent.go @@ -6,6 +6,7 @@ import ( "net" "os" "os/signal" + "sync" "syscall" "time" @@ -17,6 +18,7 @@ import ( repo "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/repository" auc "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecases/agent" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" + workerpool "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/worker-pool" ) const ( @@ -125,24 +127,27 @@ func startAgent(ctx context.Context) { SetTimeout(5 * time.Second). SetBaseURL("http://" + opts.endPointAddr) - log.Info().Msg("Starting collection and reporting loops") - log.Debug().Str("key", opts.key).Msg("") + wp := workerpool.New(3) - pollTimer := time.NewTicker(time.Duration(opts.pollInterval) * time.Second) - reportTimer := time.NewTicker(time.Duration(opts.reportInterval) * time.Second) + wp.Start(ctx) + defer wp.Wait() - defer pollTimer.Stop() - defer reportTimer.Stop() + var wg sync.WaitGroup + wg.Add(2) - for { - select { - case <-ctx.Done(): - return + go func() { + defer wg.Done() + agent.CollectMetrics(ctx, metricStorage, opts.pollInterval) + }() - case <-pollTimer.C: - agent.UpdateAllMetrics(ctx, metricStorage) - case <-reportTimer.C: - agentUsecase.SendAllMetrics(ctx, client, opts.key) - } - } + go func() { + defer wg.Done() + agent.SendMetrics(ctx, agentUsecase, client, wp, opts.reportInterval, opts.key) + }() + + <-ctx.Done() + + wg.Wait() + + log.Info().Msg("Agent stopped gracefully.") } diff --git a/go.mod b/go.mod index fbc65a4..cb8c1e4 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( require ( github.com/Masterminds/squirrel v1.5.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgio v1.0.0 // indirect @@ -34,7 +35,11 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/spf13/pflag v1.0.6 // indirect + github.com/tklauser/go-sysconf v0.3.15 // indirect + github.com/tklauser/numcpus v0.10.0 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect golang.org/x/crypto v0.37.0 // indirect golang.org/x/net v0.33.0 // indirect golang.org/x/sync v0.13.0 // indirect diff --git a/go.sum b/go.sum index 6b7ca96..a773d19 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618= github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM= github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -64,6 +66,8 @@ github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= @@ -75,12 +79,19 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= +github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= +github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= +github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index b29cdc9..15d4bb5 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -13,14 +13,18 @@ import ( "github.com/go-resty/resty/v2" "github.com/mailru/easyjson" + "github.com/shirou/gopsutil/cpu" + "github.com/shirou/gopsutil/mem" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" + repo "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/repository" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecases/agent" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/converter" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/hash" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" rt "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/runtime-stats" serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" + worker "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/worker-pool" ) type Agent struct { @@ -56,6 +60,20 @@ func UpdateAllMetrics(ctx context.Context, storage agent.MetricUpdater) { if err := storage.UpdateMetric(ctx, models.GaugeType, "RandomValue", rand.Float64()); err != nil { log.Error().Msgf("Failed to update RandomValue metric: %v", err) } + + v, _ := mem.VirtualMemory() + if err := storage.UpdateMetric(ctx, models.GaugeType, "TotalMemory", float64(v.Total)); err != nil { + log.Error().Msgf("Failed to update TotalMemory metric: %v", err) + } + + if err := storage.UpdateMetric(ctx, models.GaugeType, "FreeMemory", float64(v.Free)); err != nil { + log.Error().Msgf("Failed to update FreeMemory metric: %v", err) + } + + percent, _ := cpu.Percent(0, false) + if err := storage.UpdateMetric(ctx, models.GaugeType, "CPUutilization1", percent[0]); err != nil { + log.Error().Msgf("Failed to update CPUutilization1 metric: %v", err) + } } func (ag *Agent) SendAllMetrics(ctx context.Context, client *resty.Client, key string) { @@ -159,3 +177,42 @@ func ConvertToGzipData(metrics serialize.MetricsList) (*bytes.Buffer, bool, erro return &buf, true, nil } + +func CollectMetrics(ctx context.Context, storage *repo.MemStorage, pollInterval int) { + ticker := time.NewTicker(time.Duration(pollInterval) * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + UpdateAllMetrics(ctx, storage) + } + } +} + +func SendMetrics(ctx context.Context, + ag *Agent, + client *resty.Client, + wp *worker.WorkerPool, + reportInterval int, + key string) { + + ticker := time.NewTicker(time.Duration(reportInterval) * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + wp.AddTask(&worker.Task{ + Execute: func() { + ag.SendAllMetrics(ctx, client, key) + }, + }) + } + } + +} diff --git a/pkg/worker-pool/pool.go b/pkg/worker-pool/pool.go new file mode 100644 index 0000000..11c0f31 --- /dev/null +++ b/pkg/worker-pool/pool.go @@ -0,0 +1,57 @@ +package workerpool + +import ( + "context" + "sync" +) + +type Task struct { + Execute func() +} + + +type WorkerPool struct { + workers int + tasks chan Task + wg sync.WaitGroup +} + +func New(workers int) *WorkerPool { + return &WorkerPool{ + workers: workers, + tasks: make(chan Task, workers), + } +} + +func (wp *WorkerPool) Start(ctx context.Context) { + for i := 0; i < wp.workers; i++ { + wp.wg.Add(1) + go worker(ctx, &wp.wg, wp.tasks) + } +} + +func (wp *WorkerPool) AddTask(task *Task) { + wp.tasks <- *task +} + +func (wp *WorkerPool) Wait() { + close(wp.tasks) + wp.wg.Wait() +} + +func worker(ctx context.Context, wg *sync.WaitGroup, tasks <- chan Task) { + defer wg.Done() + + for { + select { + case <-ctx.Done(): + return + + case task, ok := <-tasks: + if !ok { + return + } + task.Execute() + } + } +} From c2ae22336c9419078ffcc97e44e5e73f8e3e9763 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Sun, 27 Jul 2025 12:12:19 +0300 Subject: [PATCH 121/126] add rateLimit for worker-pool --- cmd/agent/cfg_agent.go | 12 ++++++- cmd/server/cfg_server.go | 36 +++++++++---------- .../collector/collector-config.go} | 6 ++-- internal/config/{ => server}/server-config.go | 4 +-- internal/handlers/server/server_test.go | 26 +++++++------- internal/router/router.go | 4 +-- 6 files changed, 49 insertions(+), 39 deletions(-) rename internal/{collector/config/config.go => config/collector/collector-config.go} (89%) rename internal/config/{ => server}/server-config.go (99%) diff --git a/cmd/agent/cfg_agent.go b/cmd/agent/cfg_agent.go index fc7c7f9..fe3776f 100644 --- a/cmd/agent/cfg_agent.go +++ b/cmd/agent/cfg_agent.go @@ -26,6 +26,7 @@ const ( defaultPollInterval = 2 defaultReportInterval = 10 defaultKey = "key" + defaultRateLimit = 3 ) type options struct { @@ -33,6 +34,7 @@ type options struct { pollInterval int reportInterval int key string + rateLimit int } type envConfig struct { @@ -40,6 +42,7 @@ type envConfig struct { PollInterval int `env:"POLL_INTERVAL"` ReportInterval int `env:"REPORT_INTERVAL"` Key string `env:"KEY"` + RateLimit int `env:"RATE_LIMIT"` } var opts = &options{} @@ -82,6 +85,12 @@ var rootCmd = &cobra.Command{ opts.key = cfg.Key } + if cfg.RateLimit > 0 { + opts.rateLimit = cfg.RateLimit + } else { + opts.rateLimit = defaultRateLimit + } + if opts.pollInterval <= 0 || opts.reportInterval <= 0 { return fmt.Errorf("poll and report intervals must be > 0") } @@ -117,6 +126,7 @@ func init() { rootCmd.Flags().IntVarP(&opts.pollInterval, "p", "p", opts.pollInterval, "PollInterval value") rootCmd.Flags().IntVarP(&opts.reportInterval, "r", "r", opts.reportInterval, "PollInterval value") rootCmd.Flags().StringVarP(&opts.key, "k", "k", opts.key, "key for hash") + rootCmd.Flags().IntVarP(&opts.rateLimit, "l", "l", opts.rateLimit, "rate limit") } func startAgent(ctx context.Context) { @@ -127,7 +137,7 @@ func startAgent(ctx context.Context) { SetTimeout(5 * time.Second). SetBaseURL("http://" + opts.endPointAddr) - wp := workerpool.New(3) + wp := workerpool.New(opts.rateLimit) wp.Start(ctx) defer wp.Wait() diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 22457b8..080bbd3 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -12,8 +12,8 @@ import ( _ "github.com/jackc/pgx/v5/stdlib" "github.com/spf13/cobra" - colcfg "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/collector/config" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" + colcfg "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config/collector" + srvCfg "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config/server" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/router" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecases/ping" @@ -28,7 +28,7 @@ var ( restoreOnStart bool dataBaseDSN string key string - opts *config.Options + opts *srvCfg.Options ) var rootCmd = &cobra.Command{ @@ -41,17 +41,17 @@ var rootCmd = &cobra.Command{ } func init() { - rootCmd.Flags().StringVarP(&endPointAddr, "a", "a", config.DefaultEndpoint, "endpoint HTTP-server addr") - rootCmd.Flags().IntVarP(&storeInterval, "i", "i", config.DefaultStoreInterval, "store interval in seconds (0 = sync)") - rootCmd.Flags().StringVarP(&fileStoragePath, "f", "f", config.DefaultFileStoragePath, "file to store metrics") - rootCmd.Flags().BoolVarP(&restoreOnStart, "r", "r", config.DefaultRestoreOnStart, "restore metrics from file on start") - rootCmd.Flags().StringVarP(&dataBaseDSN, "d", "d", config.DefaultDataBaseDSN, "database dsn") - rootCmd.Flags().StringVarP(&key, "k", "k", config.DefaultKey, "key for hash") + rootCmd.Flags().StringVarP(&endPointAddr, "a", "a", srvCfg.DefaultEndpoint, "endpoint HTTP-server addr") + rootCmd.Flags().IntVarP(&storeInterval, "i", "i", srvCfg.DefaultStoreInterval, "store interval in seconds (0 = sync)") + rootCmd.Flags().StringVarP(&fileStoragePath, "f", "f", srvCfg.DefaultFileStoragePath, "file to store metrics") + rootCmd.Flags().BoolVarP(&restoreOnStart, "r", "r", srvCfg.DefaultRestoreOnStart, "restore metrics from file on start") + rootCmd.Flags().StringVarP(&dataBaseDSN, "d", "d", srvCfg.DefaultDataBaseDSN, "database dsn") + rootCmd.Flags().StringVarP(&key, "k", "k", srvCfg.DefaultKey, "key for hash") } func preRunE(cmd *cobra.Command, args []string) error { var err error - opts, err = config.ParseOptionsFromCmdAndEnvs(cmd, &config.Options{ + opts, err = srvCfg.ParseOptionsFromCmdAndEnvs(cmd, &srvCfg.Options{ EndPointAddr: endPointAddr, StoreInterval: storeInterval, FileStoragePath: fileStoragePath, @@ -59,13 +59,13 @@ func preRunE(cmd *cobra.Command, args []string) error { DataBaseDSN: dataBaseDSN, Key: key,}) - opts = config.NewServerOptions( - config.WithAddress(opts.EndPointAddr), - config.WithStoreInterval(opts.StoreInterval), - config.WithFileStoragePath(opts.FileStoragePath), - config.WithRestoreOnStart(opts.RestoreOnStart), - config.WithDataBaseDSN(opts.DataBaseDSN), - config.WithKey(opts.Key), + opts = srvCfg.NewServerOptions( + srvCfg.WithAddress(opts.EndPointAddr), + srvCfg.WithStoreInterval(opts.StoreInterval), + srvCfg.WithFileStoragePath(opts.FileStoragePath), + srvCfg.WithRestoreOnStart(opts.RestoreOnStart), + srvCfg.WithDataBaseDSN(opts.DataBaseDSN), + srvCfg.WithKey(opts.Key), ) return err @@ -105,7 +105,7 @@ func runE(cmd *cobra.Command, args []string) error { } } -func startServer(ctx context.Context, opts *config.Options) error { +func startServer(ctx context.Context, opts *srvCfg.Options) error { log.Info(). Str("address", opts.EndPointAddr). Msg("Server configuration") diff --git a/internal/collector/config/config.go b/internal/config/collector/collector-config.go similarity index 89% rename from internal/collector/config/config.go rename to internal/config/collector/collector-config.go index 38e2ee0..eb59760 100644 --- a/internal/collector/config/config.go +++ b/internal/config/collector/collector-config.go @@ -1,10 +1,10 @@ -package config +package collector import ( "context" "fmt" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" + srvCfg "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config/server" repo "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/repository" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecases/server" log "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/logger" @@ -12,7 +12,7 @@ import ( type Params struct { Ctx context.Context - Opts *config.Options + Opts *srvCfg.Options } func NewCollector(params *Params) (server.Collector, error) { diff --git a/internal/config/server-config.go b/internal/config/server/server-config.go similarity index 99% rename from internal/config/server-config.go rename to internal/config/server/server-config.go index e5735e2..102acd0 100644 --- a/internal/config/server-config.go +++ b/internal/config/server/server-config.go @@ -1,4 +1,4 @@ -package config +package server import ( "fmt" @@ -166,7 +166,7 @@ func ParseEnvs(cmd *cobra.Command, opts *Options) error { if envCfg.Key != "" { opts.Key = envCfg.Key } - + opts.RestoreOnStart = envCfg.RestoreOnStart return nil diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index 5ff871b..8d43865 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -9,7 +9,7 @@ import ( _ "github.com/jackc/pgx/v5/stdlib" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" + srvCfg "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config/server" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" repo "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/repository" @@ -20,12 +20,12 @@ import ( ) func TestUpdateMetric(t *testing.T) { - opts := &config.Options{} - for _, opt := range []func(*config.Options){ - config.WithAddress("localhost:8080"), - config.WithStoreInterval(300), - config.WithFileStoragePath("/tmp/metrics-db.json"), - config.WithRestoreOnStart(true), + opts := &srvCfg.Options{} + for _, opt := range []func(*srvCfg.Options){ + srvCfg.WithAddress("localhost:8080"), + srvCfg.WithStoreInterval(300), + srvCfg.WithFileStoragePath("/tmp/metrics-db.json"), + srvCfg.WithRestoreOnStart(true), } { opt(opts) } @@ -97,12 +97,12 @@ func TestUpdateMetric(t *testing.T) { } func TestGetMetric(t *testing.T) { - opts := &config.Options{} - for _, opt := range []func(*config.Options){ - config.WithAddress("localhost:8080"), - config.WithStoreInterval(300), - config.WithFileStoragePath("/tmp/metrics-db.json"), - config.WithRestoreOnStart(true), + opts := &srvCfg.Options{} + for _, opt := range []func(*srvCfg.Options){ + srvCfg.WithAddress("localhost:8080"), + srvCfg.WithStoreInterval(300), + srvCfg.WithFileStoragePath("/tmp/metrics-db.json"), + srvCfg.WithRestoreOnStart(true), } { opt(opts) } diff --git a/internal/router/router.go b/internal/router/router.go index a55f4ee..111d42b 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -5,11 +5,11 @@ import ( "github.com/go-chi/chi/v5" - "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config" + srvCfg "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config/server" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/server" ) -func NewRouter(srv *server.Server, opts *config.Options) http.Handler { +func NewRouter(srv *server.Server, opts *srvCfg.Options) http.Handler { r := chi.NewRouter() r.Use(server.WithLogging) From 4c367f55eaebf4ef43f9f444e21c0f4912af41d5 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Sun, 27 Jul 2025 12:19:17 +0300 Subject: [PATCH 122/126] add funcs options for agent --- cmd/agent/cfg_agent.go | 158 +++++++++-------------- internal/config/agent/agent-config.go | 178 ++++++++++++++++++++++++++ 2 files changed, 240 insertions(+), 96 deletions(-) create mode 100644 internal/config/agent/agent-config.go diff --git a/cmd/agent/cfg_agent.go b/cmd/agent/cfg_agent.go index fe3776f..8d4e258 100644 --- a/cmd/agent/cfg_agent.go +++ b/cmd/agent/cfg_agent.go @@ -3,17 +3,16 @@ package main import ( "context" "fmt" - "net" "os" "os/signal" "sync" "syscall" "time" - "github.com/caarlos0/env/v6" "github.com/go-resty/resty/v2" "github.com/spf13/cobra" + agCfg "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/config/agent" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/handlers/agent" repo "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/repository" auc "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecases/agent" @@ -21,112 +20,79 @@ import ( workerpool "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/worker-pool" ) -const ( - defaultEndpoint = "localhost:8080" - defaultPollInterval = 2 - defaultReportInterval = 10 - defaultKey = "key" - defaultRateLimit = 3 -) - -type options struct { +var ( endPointAddr string pollInterval int reportInterval int - key string rateLimit int -} - -type envConfig struct { - EndPointAddr string `env:"ADDRESS"` - PollInterval int `env:"POLL_INTERVAL"` - ReportInterval int `env:"REPORT_INTERVAL"` - Key string `env:"KEY"` - RateLimit int `env:"RATE_LIMIT"` -} - -var opts = &options{} + key string + opts *agCfg.Options +) var rootCmd = &cobra.Command{ - Use: "agent", - Short: "MetricService", - Long: "MetricService", - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - logFile, err := log.InitLogger("logFileAgent.log") - if err != nil { - fmt.Fprintf(os.Stderr, "Logger init error: %v\n", err) - os.Exit(1) - } - - defer func() { - if err := logFile.Close(); err != nil { - log.Error().Err(err).Msg("Failed to close log file") - } - }() - - var cfg envConfig - err = env.Parse(&cfg) - if err != nil { - return fmt.Errorf("poll and report intervals must be > 0") - } - - if cfg.EndPointAddr != "" { - opts.endPointAddr = cfg.EndPointAddr - } - if cfg.PollInterval > 0 { - opts.pollInterval = cfg.PollInterval - } - if cfg.ReportInterval > 0 { - opts.reportInterval = cfg.ReportInterval - } + Use: "agent", + Short: "MetricService", + Long: "MetricService", + Args: cobra.NoArgs, + PreRunE: preRunE, + RunE: runE, +} - if cfg.Key != "" { - opts.key = cfg.Key - } +func init() { + rootCmd.Flags().StringVarP(&endPointAddr, "a", "a", agCfg.DefaultEndpoint, "endpoint HTTP-server addr") + rootCmd.Flags().IntVarP(&pollInterval, "p", "p", agCfg.DefaultPollInterval, "PollInterval value") + rootCmd.Flags().IntVarP(&reportInterval, "r", "r", agCfg.DefaultReportInterval, "PollInterval value") + rootCmd.Flags().StringVarP(&key, "k", "k", agCfg.DefaultKey, "key for hash") + rootCmd.Flags().IntVarP(&rateLimit, "l", "l", agCfg.DefaultRateLimit, "rate limit") +} - if cfg.RateLimit > 0 { - opts.rateLimit = cfg.RateLimit - } else { - opts.rateLimit = defaultRateLimit - } +func preRunE(cmd *cobra.Command, args []string) error { + var err error + opts, err = agCfg.ParseOptionsFromCmdAndEnvs(cmd, &agCfg.Options{ + EndPointAddr: endPointAddr, + PollInterval: pollInterval, + ReportInterval: reportInterval, + Key: key, + RateLimit: rateLimit}) + + opts = agCfg.NewAgentOptions( + agCfg.WithAddress(opts.EndPointAddr), + agCfg.WithPollInterval(opts.PollInterval), + agCfg.WithReportInterval(opts.ReportInterval), + agCfg.WithRateLimit(opts.RateLimit), + agCfg.WithKey(opts.Key), + ) + + return err +} - if opts.pollInterval <= 0 || opts.reportInterval <= 0 { - return fmt.Errorf("poll and report intervals must be > 0") - } +func runE(cmd *cobra.Command, args []string) error { + logFile, err := log.InitLogger("logFileAgent.log") + if err != nil { + fmt.Fprintf(os.Stderr, "Logger init error: %v\n", err) + os.Exit(1) + } - if _, _, err := net.SplitHostPort(opts.endPointAddr); err != nil { - return fmt.Errorf("invalid address %s: %w", opts.endPointAddr, err) + defer func() { + if err := logFile.Close(); err != nil { + log.Error().Err(err).Msg("Failed to close log file") } + }() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - stop := make(chan os.Signal, 1) - signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - go func() { - <-stop - cancel() - }() + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM) - startAgent(ctx) + go func() { + <-stop + cancel() + }() - return nil - }, -} + startAgent(ctx) -func init() { - opts.endPointAddr = defaultEndpoint - opts.pollInterval = defaultPollInterval - opts.reportInterval = defaultReportInterval - opts.key = defaultKey - - rootCmd.Flags().StringVarP(&opts.endPointAddr, "a", "a", opts.endPointAddr, "endpoint HTTP-server addr") - rootCmd.Flags().IntVarP(&opts.pollInterval, "p", "p", opts.pollInterval, "PollInterval value") - rootCmd.Flags().IntVarP(&opts.reportInterval, "r", "r", opts.reportInterval, "PollInterval value") - rootCmd.Flags().StringVarP(&opts.key, "k", "k", opts.key, "key for hash") - rootCmd.Flags().IntVarP(&opts.rateLimit, "l", "l", opts.rateLimit, "rate limit") + return nil } func startAgent(ctx context.Context) { @@ -135,9 +101,9 @@ func startAgent(ctx context.Context) { client := resty.New(). SetTimeout(5 * time.Second). - SetBaseURL("http://" + opts.endPointAddr) + SetBaseURL("http://" + opts.EndPointAddr) - wp := workerpool.New(opts.rateLimit) + wp := workerpool.New(opts.RateLimit) wp.Start(ctx) defer wp.Wait() @@ -147,12 +113,12 @@ func startAgent(ctx context.Context) { go func() { defer wg.Done() - agent.CollectMetrics(ctx, metricStorage, opts.pollInterval) + agent.CollectMetrics(ctx, metricStorage, opts.PollInterval) }() go func() { defer wg.Done() - agent.SendMetrics(ctx, agentUsecase, client, wp, opts.reportInterval, opts.key) + agent.SendMetrics(ctx, agentUsecase, client, wp, opts.ReportInterval, opts.Key) }() <-ctx.Done() diff --git a/internal/config/agent/agent-config.go b/internal/config/agent/agent-config.go new file mode 100644 index 0000000..8eb5019 --- /dev/null +++ b/internal/config/agent/agent-config.go @@ -0,0 +1,178 @@ +package agent + +import ( + "fmt" + "net" + + "github.com/caarlos0/env/v6" + "github.com/spf13/cobra" +) + +const ( + DefaultEndpoint = "localhost:8080" + DefaultPollInterval = 2 + DefaultReportInterval = 10 + DefaultKey = "" + DefaultRateLimit = 3 +) + +type Options struct { + EndPointAddr string + PollInterval int + ReportInterval int + Key string + RateLimit int +} + +type EnvConfig struct { + EndPointAddr string `env:"ADDRESS"` + PollInterval int `env:"POLL_INTERVAL"` + ReportInterval int `env:"REPORT_INTERVAL"` + Key string `env:"KEY"` + RateLimit int `env:"RATE_LIMIT"` +} + +type Option func(*Options) + +func NewAgentOptions(options ...Option) *Options { + opts := &Options{ + EndPointAddr: DefaultEndpoint, + PollInterval: DefaultPollInterval, + ReportInterval: DefaultReportInterval, + Key: DefaultKey, + RateLimit: DefaultRateLimit, + } + + for _, opt := range options { + opt(opts) + } + + return opts +} + +func WithAddress(addr string) Option { + return func(o *Options) { + o.EndPointAddr = addr + } +} + +func WithKey(key string) Option { + return func(o *Options) { + o.Key = key + } +} + +func WithPollInterval(pollInterval int) Option { + return func(o *Options) { + o.PollInterval = pollInterval + } +} + +func WithReportInterval(reportInterval int) Option { + return func(o *Options) { + o.ReportInterval = reportInterval + } +} + +func WithRateLimit(rateLimit int) Option { + return func(o *Options) { + o.RateLimit = rateLimit + } +} + +func ParseOptionsFromCmdAndEnvs(cmd *cobra.Command, src *Options) (*Options, error) { + opts, err := ParseFlags(cmd, src) + if err != nil { + return nil, err + } + + if err := ParseEnvs(cmd, opts); err != nil { + return nil, err + } + + if _, _, err := net.SplitHostPort(opts.EndPointAddr); err != nil { + return nil, fmt.Errorf("invalid address %s: %w", opts.EndPointAddr, err) + } + + return opts, nil +} + +func ParseFlags(cmd *cobra.Command, src *Options) (*Options, error) { + opts := *src + + if cmd.Flags().Changed("a") { + opts.EndPointAddr = src.EndPointAddr + } + + if cmd.Flags().Changed("p") { + if src.PollInterval > 0 { + opts.PollInterval = src.PollInterval + } else { + return nil, fmt.Errorf("pollInterval need > 0") + } + } + + if cmd.Flags().Changed("r") { + if src.ReportInterval > 0 { + opts.ReportInterval = src.ReportInterval + } else { + return nil, fmt.Errorf("reportInterval need > 0") + } + } + + if cmd.Flags().Changed("l") { + if src.RateLimit > 0 { + opts.RateLimit = src.RateLimit + } else { + return nil, fmt.Errorf("rateLimit need > 0") + } + } + + if cmd.Flags().Changed("k") { + if src.Key != "" { + opts.Key = src.Key + } + } + + return &opts, nil +} + +func ParseEnvs(cmd *cobra.Command, opts *Options) error { + var cfg EnvConfig + err := env.Parse(&cfg) + if err != nil { + return fmt.Errorf("poll and report intervals must be > 0: %v", err) + } + + if cfg.EndPointAddr != "" { + opts.EndPointAddr = cfg.EndPointAddr + } + + if cfg.PollInterval > 0 { + opts.PollInterval = cfg.PollInterval + } + + if cfg.ReportInterval > 0 { + opts.ReportInterval = cfg.ReportInterval + } + + if cfg.Key != "" { + opts.Key = cfg.Key + } + + if cfg.RateLimit > 0 { + opts.RateLimit = cfg.RateLimit + } else { + opts.RateLimit = DefaultRateLimit + } + + if opts.PollInterval <= 0 || opts.ReportInterval <= 0 { + return fmt.Errorf("poll and report intervals must be > 0") + } + + if _, _, err := net.SplitHostPort(opts.EndPointAddr); err != nil { + return fmt.Errorf("invalid address %s: %w", opts.EndPointAddr, err) + } + + return nil +} From a62578aa272324e90f8a8716856ab3937e4c02c9 Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Sun, 27 Jul 2025 13:30:15 +0300 Subject: [PATCH 123/126] after code review --- cmd/agent/cfg_agent.go | 2 +- internal/handlers/agent/agent.go | 26 +++++++++++------------- internal/handlers/server/hashing.go | 14 +++++++++++-- internal/usecases/agent/agent-usecase.go | 4 ++++ pkg/worker-pool/pool.go | 19 +++++++++-------- 5 files changed, 40 insertions(+), 25 deletions(-) diff --git a/cmd/agent/cfg_agent.go b/cmd/agent/cfg_agent.go index 8d4e258..10f70d4 100644 --- a/cmd/agent/cfg_agent.go +++ b/cmd/agent/cfg_agent.go @@ -113,7 +113,7 @@ func startAgent(ctx context.Context) { go func() { defer wg.Done() - agent.CollectMetrics(ctx, metricStorage, opts.PollInterval) + agent.CollectMetrics(ctx, agentUsecase, opts.PollInterval) }() go func() { diff --git a/internal/handlers/agent/agent.go b/internal/handlers/agent/agent.go index 15d4bb5..2d27031 100644 --- a/internal/handlers/agent/agent.go +++ b/internal/handlers/agent/agent.go @@ -17,7 +17,6 @@ import ( "github.com/shirou/gopsutil/mem" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" - repo "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/repository" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/usecases/agent" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/converter" "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/hash" @@ -35,14 +34,14 @@ func NewAgent(uc *agent.AgentUsecase) *Agent { return &Agent{Usecase: uc} } -func UpdateAllMetrics(ctx context.Context, storage agent.MetricUpdater) { +func (ag *Agent) UpdateAllMetrics(ctx context.Context) { var memStats runtime.MemStats runtime.ReadMemStats(&memStats) for _, stat := range rt.MemRuntimeStats { val := stat.Get(&memStats) - if err := storage.UpdateMetric(ctx, stat.Type, stat.Name, val); err != nil { + if err := ag.Usecase.UpdateMetric(ctx, stat.Type, stat.Name, val); err != nil { log.Error(). Err(err). Str("metric", stat.Name). @@ -53,25 +52,25 @@ func UpdateAllMetrics(ctx context.Context, storage agent.MetricUpdater) { log.Debug().Msgf("update metric %s", stat.Name) } - if err := storage.UpdateMetric(ctx, models.CounterType, "PollCount", int64(1)); err != nil { + if err := ag.Usecase.UpdateMetric(ctx, models.CounterType, "PollCount", int64(1)); err != nil { log.Error().Msgf("Failed to update PollCount metric: %v", err) } - if err := storage.UpdateMetric(ctx, models.GaugeType, "RandomValue", rand.Float64()); err != nil { + if err := ag.Usecase.UpdateMetric(ctx, models.GaugeType, "RandomValue", rand.Float64()); err != nil { log.Error().Msgf("Failed to update RandomValue metric: %v", err) } v, _ := mem.VirtualMemory() - if err := storage.UpdateMetric(ctx, models.GaugeType, "TotalMemory", float64(v.Total)); err != nil { + if err := ag.Usecase.UpdateMetric(ctx, models.GaugeType, "TotalMemory", float64(v.Total)); err != nil { log.Error().Msgf("Failed to update TotalMemory metric: %v", err) } - if err := storage.UpdateMetric(ctx, models.GaugeType, "FreeMemory", float64(v.Free)); err != nil { + if err := ag.Usecase.UpdateMetric(ctx, models.GaugeType, "FreeMemory", float64(v.Free)); err != nil { log.Error().Msgf("Failed to update FreeMemory metric: %v", err) } percent, _ := cpu.Percent(0, false) - if err := storage.UpdateMetric(ctx, models.GaugeType, "CPUutilization1", percent[0]); err != nil { + if err := ag.Usecase.UpdateMetric(ctx, models.GaugeType, "CPUutilization1", percent[0]); err != nil { log.Error().Msgf("Failed to update CPUutilization1 metric: %v", err) } } @@ -178,7 +177,7 @@ func ConvertToGzipData(metrics serialize.MetricsList) (*bytes.Buffer, bool, erro return &buf, true, nil } -func CollectMetrics(ctx context.Context, storage *repo.MemStorage, pollInterval int) { +func CollectMetrics(ctx context.Context, ag *Agent, pollInterval int) { ticker := time.NewTicker(time.Duration(pollInterval) * time.Second) defer ticker.Stop() @@ -187,7 +186,7 @@ func CollectMetrics(ctx context.Context, storage *repo.MemStorage, pollInterval case <-ctx.Done(): return case <-ticker.C: - UpdateAllMetrics(ctx, storage) + ag.UpdateAllMetrics(ctx) } } } @@ -207,10 +206,9 @@ func SendMetrics(ctx context.Context, case <-ctx.Done(): return case <-ticker.C: - wp.AddTask(&worker.Task{ - Execute: func() { - ag.SendAllMetrics(ctx, client, key) - }, + wp.AddTask(func(ctx context.Context) error { + ag.SendAllMetrics(ctx, client, key) + return nil }) } } diff --git a/internal/handlers/server/hashing.go b/internal/handlers/server/hashing.go index 0725409..b9509ff 100644 --- a/internal/handlers/server/hashing.go +++ b/internal/handlers/server/hashing.go @@ -25,13 +25,21 @@ func (hsw *hashResponseWriter) Write(b []byte) (int, error) { if hsw.key != nil && hsw.body.Len() > 0 { newHash, err := hash.GetHash(hsw.key, hsw.body.Bytes()) if err != nil { - log.Error().Err(err).Msg("failed to get hash") + log.Error().Err(err).Msg("failed to get new hash") + http.Error(hsw.ResponseWriter, "failed to get new hash", http.StatusInternalServerError) + return 0, fmt.Errorf("failed to get new hash: %v", err) } hsw.Header().Set("HashSHA256", hex.EncodeToString(newHash)) } - return hsw.ResponseWriter.Write(b) + n, err := hsw.ResponseWriter.Write(b) + if err != nil { + log.Error().Err(err).Msg("failed to writer response") + return 0, err + } + + return n, nil } func WithHashing(key []byte) func(next http.Handler) http.Handler { @@ -72,6 +80,8 @@ func WithHashing(key []byte) func(next http.Handler) http.Handler { next.ServeHTTP(hw, r) } + + next.ServeHTTP(w, r) }) } } diff --git a/internal/usecases/agent/agent-usecase.go b/internal/usecases/agent/agent-usecase.go index a2f4629..9237e8f 100644 --- a/internal/usecases/agent/agent-usecase.go +++ b/internal/usecases/agent/agent-usecase.go @@ -25,3 +25,7 @@ func (uc *AgentUsecase) GetAllMetrics(ctx context.Context) ([]models.Metric, err func (uc *AgentUsecase) GetMetric(ctx context.Context, mType, mName string) (models.Metric, error) { return uc.getter.GetMetric(ctx, mType, mName) } + +func (uc *AgentUsecase) UpdateMetric(ctx context.Context, mType, mName string, mValue any) error { + return uc.updater.UpdateMetric(ctx, mType, mName, mValue) +} diff --git a/pkg/worker-pool/pool.go b/pkg/worker-pool/pool.go index 11c0f31..f548190 100644 --- a/pkg/worker-pool/pool.go +++ b/pkg/worker-pool/pool.go @@ -3,12 +3,11 @@ package workerpool import ( "context" "sync" -) -type Task struct { - Execute func() -} + "github.com/rs/zerolog/log" +) +type Task func(ctx context.Context) error type WorkerPool struct { workers int @@ -30,8 +29,8 @@ func (wp *WorkerPool) Start(ctx context.Context) { } } -func (wp *WorkerPool) AddTask(task *Task) { - wp.tasks <- *task +func (wp *WorkerPool) AddTask(task Task) { + wp.tasks <- task } func (wp *WorkerPool) Wait() { @@ -39,7 +38,7 @@ func (wp *WorkerPool) Wait() { wp.wg.Wait() } -func worker(ctx context.Context, wg *sync.WaitGroup, tasks <- chan Task) { +func worker(ctx context.Context, wg *sync.WaitGroup, tasks <-chan Task) { defer wg.Done() for { @@ -51,7 +50,11 @@ func worker(ctx context.Context, wg *sync.WaitGroup, tasks <- chan Task) { if !ok { return } - task.Execute() + err := task(ctx) + if err != nil { + log.Error().Err(err).Msg("worker pool failed") + return + } } } } From 15cdbb56649a284ce08f97baf1da5bd89cfcb5db Mon Sep 17 00:00:00 2001 From: Artyom Kaplin Date: Sun, 27 Jul 2025 18:20:22 +0300 Subject: [PATCH 124/126] add some tests --- internal/handlers/server/server_test.go | 40 +++++ pkg/converter/converter_test.go | 90 +++++++++++ pkg/serialization/serialization_test.go | 195 ++++++++++++++++++++++++ 3 files changed, 325 insertions(+) create mode 100644 pkg/converter/converter_test.go create mode 100644 pkg/serialization/serialization_test.go diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index 8d43865..0c7739a 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -173,3 +173,43 @@ func TestGetMetric(t *testing.T) { }) } } + +func TestGetAllMetrics(t *testing.T) { + opts := &srvCfg.Options{} + for _, opt := range []func(*srvCfg.Options){ + srvCfg.WithAddress("localhost:8080"), + srvCfg.WithStoreInterval(300), + srvCfg.WithFileStoragePath("/tmp/metrics-db.json"), + srvCfg.WithRestoreOnStart(true), + } { + opt(opts) + } + + ctx := context.Background() + storage := repo.NewMemStorage() + + if err := storage.UpdateMetric(ctx, models.GaugeType, "cpu_usage", 75.5); err != nil { + log.Error().Msgf("Failed to update metric cpu_usage: %v", err) + } + + if err := storage.UpdateMetric(ctx, models.CounterType, "requests_total", int64(100)); err != nil { + log.Error().Msgf("Failed to update metric requests_total: %v", err) + } + + metricUsecase := srvUsecase.NewMetricUsecase(storage, storage, storage) + router := router.NewRouter(server.NewServer(metricUsecase, nil), opts) + + t.Run("GetAllMetrics returned HTML metrics", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/", nil) + rr := httptest.NewRecorder() + + router.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusOK, rr.Code) + body, _ := io.ReadAll(rr.Body) + + assert.Contains(t, string(body), "cpu_usage") + assert.Contains(t, string(body), "requests_total") + assert.Contains(t, string(body), "") + }) +} diff --git a/pkg/converter/converter_test.go b/pkg/converter/converter_test.go new file mode 100644 index 0000000..d4cacf9 --- /dev/null +++ b/pkg/converter/converter_test.go @@ -0,0 +1,90 @@ +package converter_test + +import ( + "testing" + + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/converter" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConvertByType(t *testing.T) { + type args struct { + mType string + mValue string + } + tests := []struct { + name string + args args + want any + wantErr bool + }{ + { + name: "valid gauge", + args: args{ + mType: models.GaugeType, + mValue: "123.45", + }, + want: 123.45, + wantErr: false, + }, + { + name: "invalid gauge value", + args: args{ + mType: models.GaugeType, + mValue: "not_a_number", + }, + wantErr: true, + }, + { + name: "valid counter", + args: args{ + mType: models.CounterType, + mValue: "42", + }, + want: int64(42), + wantErr: false, + }, + { + name: "invalid counter value", + args: args{ + mType: models.CounterType, + mValue: "42.5", + }, + wantErr: true, + }, + { + name: "unknown type", + args: args{ + mType: "unknown", + mValue: "123", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := converter.ConvertByType(tt.args.mType, tt.args.mValue) + if tt.wantErr { + require.Error(t, err) + return + } + + require.NoError(t, err) + + switch tt.args.mType { + case models.GaugeType: + assert.IsType(t, float64(0), got) + assert.Equal(t, tt.want, got) + + case models.CounterType: + require.IsType(t, int64(0), got) + require.Equal(t, tt.want, got) + + default: + t.Fatalf("unexpected type %s", tt.args.mType) + } + }) + } +} diff --git a/pkg/serialization/serialization_test.go b/pkg/serialization/serialization_test.go new file mode 100644 index 0000000..f138d1a --- /dev/null +++ b/pkg/serialization/serialization_test.go @@ -0,0 +1,195 @@ +package serialization_test + +import ( + "testing" + + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" + "github.com/stretchr/testify/assert" + + serialize "github.com/rAch-kaplin/mipt-golang-course/MetricsService/pkg/serialization" +) + +func TestMetric_SetValue(t *testing.T) { + type fields struct { + ID string + MType string + Delta *int64 + Value *float64 + } + type args struct { + value any + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "set gauge with valid float64", + fields: fields{ + ID: "gauge1", + MType: models.GaugeType, + Delta: nil, + Value: nil, + }, + args: args{value: 123.45}, + wantErr: false, + }, + { + name: "set gauge with invalid type", + fields: fields{ + ID: "gauge2", + MType: models.GaugeType, + Delta: nil, + Value: nil, + }, + args: args{value: "string_instead_of_float"}, + wantErr: true, + }, + { + name: "set counter with valid int64", + fields: fields{ + ID: "counter1", + MType: models.CounterType, + Delta: nil, + Value: nil, + }, + args: args{value: int64(100)}, + wantErr: false, + }, + { + name: "set counter with invalid type", + fields: fields{ + ID: "counter2", + MType: models.CounterType, + Delta: nil, + Value: nil, + }, + args: args{value: 12.34}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mtr := &serialize.Metric{ + ID: tt.fields.ID, + MType: tt.fields.MType, + Delta: tt.fields.Delta, + Value: tt.fields.Value, + } + + err := mtr.SetValue(tt.args.value) + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + switch mtr.MType { + case models.GaugeType: + assert.NotNil(t, mtr.Value) + assert.Equal(t, tt.args.value, *mtr.Value) + + case models.CounterType: + assert.NotNil(t, mtr.Delta) + assert.Equal(t, tt.args.value, *mtr.Delta) + + default: + t.Fatalf("unexpected type %s", mtr.MType) + } + } + }) + } +} + +func float64Ptr(f float64) *float64 { return &f } +func int64Ptr(i int64) *int64 { return &i } + +func TestMetric_GetValue(t *testing.T) { + type fields struct { + ID string + MType string + Delta *int64 + Value *float64 + } + tests := []struct { + name string + fields fields + want any + wantErr bool + }{ + { + name: "get gauge with valid value", + fields: fields{ + ID: "gauge1", + MType: models.GaugeType, + Value: float64Ptr(123.45), + Delta: nil, + }, + want: 123.45, + wantErr: false, + }, + { + name: "get gauge with nil value", + fields: fields{ + ID: "gauge2", + MType: models.GaugeType, + Value: nil, + Delta: nil, + }, + want: nil, + wantErr: true, + }, + { + name: "get counter with valid value", + fields: fields{ + ID: "counter1", + MType: models.CounterType, + Delta: int64Ptr(42), + Value: nil, + }, + want: int64(42), + wantErr: false, + }, + { + name: "get counter with nil value", + fields: fields{ + ID: "counter2", + MType: models.CounterType, + Delta: nil, + Value: nil, + }, + want: nil, + wantErr: true, + }, + { + name: "unknown metric type", + fields: fields{ + ID: "unknown", + MType: "unknown", + Delta: nil, + Value: nil, + }, + want: nil, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mtr := &serialize.Metric{ + ID: tt.fields.ID, + MType: tt.fields.MType, + Delta: tt.fields.Delta, + Value: tt.fields.Value, + } + got, err := mtr.GetValue() + if tt.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + } + }) + } +} From af7eb8095f7677b9bca27826f35c52117553adcf Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Mon, 28 Jul 2025 19:37:27 +0300 Subject: [PATCH 125/126] add bench, tests, pprof --- .gitignore | 2 + Makefile | 3 + cmd/server/cfg_server.go | 33 +- go.mod | 1 + go.sum | 3 + internal/config/agent/agent-config.go | 2 +- .../{repository_test.go => memory_test.go} | 26 ++ internal/repository/postgres.go | 2 +- internal/repository/postgres_test.go | 334 ++++++++++++++++++ 9 files changed, 394 insertions(+), 12 deletions(-) rename internal/repository/{repository_test.go => memory_test.go} (91%) create mode 100644 internal/repository/postgres_test.go diff --git a/.gitignore b/.gitignore index c16c13f..a22b72d 100644 --- a/.gitignore +++ b/.gitignore @@ -36,4 +36,6 @@ logFileAgent.log cmd/server/server cmd/agent/agent +profiles/ + log.txt diff --git a/Makefile b/Makefile index 6a6c93e..709654d 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,9 @@ agent: deps test: @go test ./... -v +bench: + @go test -bench=. -benchmem ./... + deps: @go mod download diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 080bbd3..9c21c8d 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -9,6 +9,8 @@ import ( "syscall" "time" + _ "net/http/pprof" + _ "github.com/jackc/pgx/v5/stdlib" "github.com/spf13/cobra" @@ -57,7 +59,7 @@ func preRunE(cmd *cobra.Command, args []string) error { FileStoragePath: fileStoragePath, RestoreOnStart: restoreOnStart, DataBaseDSN: dataBaseDSN, - Key: key,}) + Key: key}) opts = srvCfg.NewServerOptions( srvCfg.WithAddress(opts.EndPointAddr), @@ -83,6 +85,11 @@ func runE(cmd *cobra.Command, args []string) error { } }() + go func() { + fmt.Println("pprof listening on :6060") + http.ListenAndServe("localhost:6060", nil) + }() + ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -139,24 +146,30 @@ func startServer(ctx context.Context, opts *srvCfg.Options) error { Handler: r, } + srvErrCh := make(chan error, 1) + go func() { log.Info().Msg("Starting HTTP server...") if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { log.Fatal().Err(err).Msg("HTTP server failed unexpectedly") + srvErrCh <- err } }() - <-ctx.Done() - shutdownCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - - if err := srv.Shutdown(shutdownCtx); err != nil { - log.Error().Err(err).Msg("Failed to gracefully shutdown server") + select { + case err := <-srvErrCh: return err - } + case <-ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + if err := srv.Shutdown(shutdownCtx); err != nil { + log.Error().Err(err).Msg("Failed to gracefully shutdown server") + return err + } - log.Info().Msg("Server gracefully stopped") + log.Info().Msg("Server gracefully stopped") + } return nil } - diff --git a/go.mod b/go.mod index cb8c1e4..eb1f8d9 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( ) require ( + github.com/DATA-DOG/go-sqlmock v1.5.2 // indirect github.com/Masterminds/squirrel v1.5.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-ole/go-ole v1.2.6 // indirect diff --git a/go.sum b/go.sum index a773d19..934e27f 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= +github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/caarlos0/env/v6 v6.10.1 h1:t1mPSxNpei6M5yAeu1qtRdPAK29Nbcf/n3G7x+b3/II= @@ -40,6 +42,7 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= diff --git a/internal/config/agent/agent-config.go b/internal/config/agent/agent-config.go index 8eb5019..c1678fc 100644 --- a/internal/config/agent/agent-config.go +++ b/internal/config/agent/agent-config.go @@ -13,7 +13,7 @@ const ( DefaultPollInterval = 2 DefaultReportInterval = 10 DefaultKey = "" - DefaultRateLimit = 3 + DefaultRateLimit = 10 ) type Options struct { diff --git a/internal/repository/repository_test.go b/internal/repository/memory_test.go similarity index 91% rename from internal/repository/repository_test.go rename to internal/repository/memory_test.go index ac1ad09..c5ee845 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/memory_test.go @@ -284,3 +284,29 @@ func TestMemStorage_GetAllMetrics(t *testing.T) { }) } } + +func BenchmarkMemStorage_UpdateMetricList(b *testing.B) { + ctx := context.Background() + + ms := repository.NewMemStorage() + + metrics := []models.Metric{ + models.NewCounter("test_counter", 100), + models.NewGauge("test_gauge", 100.0), + } + + for i := 0; i < b.N; i++ { + _ = ms.UpdateMetricList(ctx, metrics) + } +} + +func BenchmarkMemStorage_GetMetric(b *testing.B) { + ctx := context.Background() + m := repository.NewMemStorage() + _ = m.UpdateMetric(ctx, models.CounterType, "counter", int64(1)) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = m.GetMetric(ctx, models.CounterType, "counter") + } +} diff --git a/internal/repository/postgres.go b/internal/repository/postgres.go index 6d9e7a5..0e1db85 100644 --- a/internal/repository/postgres.go +++ b/internal/repository/postgres.go @@ -115,7 +115,7 @@ func (db *Database) GetAllMetrics(ctx context.Context) ([]models.Metric, error) if err != nil { log.Error().Err(err).Msg("The request was not processed") - return nil, fmt.Errorf("failed to query all metrics: %v", err) + return nil, fmt.Errorf("failed to query all metrics: %w", err) } defer func() { if err := rows.Close(); err != nil { diff --git a/internal/repository/postgres_test.go b/internal/repository/postgres_test.go new file mode 100644 index 0000000..996e0e5 --- /dev/null +++ b/internal/repository/postgres_test.go @@ -0,0 +1,334 @@ +package repository_test + +import ( + "context" + "database/sql" + "database/sql/driver" + "regexp" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + sq "github.com/Masterminds/squirrel" + "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/models" + repo "github.com/rAch-kaplin/mipt-golang-course/MetricsService/internal/repository" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDatabase_GetMetric(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("failed to create sqlmock: %v", err) + } + defer db.Close() + + repo := &repo.Database{ + DB: db, + } + + t.Run("GetMetric_Gauge", func(t *testing.T) { + builder := sq.Select(`"ID"`, `"MType"`, `"Delta"`, `"Value"`). + From("collector"). + Where(sq.Eq{`"ID"`: "test_gauge", `"MType"`: "gauge"}). + Limit(1). + PlaceholderFormat(sq.Dollar) + + query, args, err := builder.ToSql() + if err != nil { + t.Fatalf("failed to build query: %v", err) + } + + driverArgs := make([]driver.Value, len(args)) + for i, a := range args { + driverArgs[i] = a + } + + mock.ExpectQuery(regexp.QuoteMeta(query)). + WithArgs(driverArgs...). + WillReturnRows(sqlmock.NewRows([]string{"ID", "MType", "Delta", "Value"}). + AddRow("test_gauge", "gauge", nil, 100.0)) + + metric, err := repo.GetMetric(context.Background(), "gauge", "test_gauge") + require.NoError(t, err) + + assert.Equal(t, "test_gauge", metric.Name()) + assert.Equal(t, "gauge", metric.Type()) + assert.Equal(t, 100.0, metric.Value()) + + require.NoError(t, mock.ExpectationsWereMet()) + }) + + t.Run("GetMetric_Counter", func(t *testing.T) { + builder := sq.Select(`"ID"`, `"MType"`, `"Delta"`, `"Value"`). + From("collector"). + Where(sq.Eq{`"ID"`: "test_counter", `"MType"`: "counter"}). + Limit(1). + PlaceholderFormat(sq.Dollar) + + query, args, err := builder.ToSql() + if err != nil { + t.Fatalf("failed to build query: %v", err) + } + + driverArgs := make([]driver.Value, len(args)) + for i, a := range args { + driverArgs[i] = a + } + + mock.ExpectQuery(regexp.QuoteMeta(query)). + WithArgs(driverArgs...). + WillReturnRows(sqlmock.NewRows([]string{"ID", "MType", "Delta", "Value"}). + AddRow("test_counter", "counter", 100, nil)) + + metric, err := repo.GetMetric(context.Background(), "counter", "test_counter") + + require.NoError(t, err) + + assert.Equal(t, "test_counter", metric.Name()) + assert.Equal(t, "counter", metric.Type()) + assert.Equal(t, int64(100), metric.Value()) + + require.NoError(t, mock.ExpectationsWereMet()) + }) + + t.Run("GetMetric_InvalidType", func(t *testing.T) { + builder := sq.Select(`"ID"`, `"MType"`, `"Delta"`, `"Value"`). + From("collector"). + Where(sq.Eq{`"ID"`: "test_counter", `"MType"`: "counter"}). + Limit(1). + PlaceholderFormat(sq.Dollar) + + query, args, err := builder.ToSql() + if err != nil { + t.Fatalf("failed to build query: %v", err) + } + + driverArgs := make([]driver.Value, len(args)) + for i, a := range args { + driverArgs[i] = a + } + + mock.ExpectQuery(regexp.QuoteMeta(query)). + WithArgs(driverArgs...). + WillReturnError(models.ErrInvalidMetricsType) + + _, err = repo.GetMetric(context.Background(), "counter", "test_counter") + + require.Error(t, err) + assert.ErrorIs(t, err, models.ErrInvalidMetricsType) + }) + + t.Run("GetMetric_NotFound", func(t *testing.T) { + mock.ExpectQuery(`SELECT "ID", "MType", "Delta", "Value" FROM collector WHERE "ID" = \$1 AND "MType" = \$2 LIMIT 1`). + WithArgs("unknown_metric", "gauge"). + WillReturnError(sql.ErrNoRows) + + metric, err := repo.GetMetric(context.Background(), "gauge", "unknown_metric") + + require.Error(t, err) + assert.ErrorIs(t, err, models.ErrMetricsNotFound) + assert.Nil(t, metric) + + require.NoError(t, mock.ExpectationsWereMet()) + }) + +} + +func TestDatabase_GetAllMetrics(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("failed to create sqlmock: %v", err) + } + defer db.Close() + + repo := &repo.Database{ + DB: db, + } + + t.Run("GetAllMetrics_Success", func(t *testing.T) { + builder := sq.Select(`"ID"`, `"MType"`, `"Delta"`, `"Value"`). + From("collector"). + PlaceholderFormat(sq.Dollar) + + query, args, err := builder.ToSql() + if err != nil { + t.Fatalf("failed to build query: %v", err) + } + + driverArgs := make([]driver.Value, len(args)) + for i, a := range args { + driverArgs[i] = a + } + + mock.ExpectQuery(regexp.QuoteMeta(query)). + WithArgs(driverArgs...). + WillReturnRows(sqlmock.NewRows([]string{"ID", "MType", "Delta", "Value"}). + AddRow("test_gauge", "gauge", nil, 100.0). + AddRow("test_counter", "counter", 100, nil)) + + metrics, err := repo.GetAllMetrics(context.Background()) + require.NoError(t, err) + + assert.Equal(t, 2, len(metrics)) + assert.Equal(t, "test_gauge", metrics[0].Name()) + assert.Equal(t, "gauge", metrics[0].Type()) + assert.Equal(t, 100.0, metrics[0].Value()) + + assert.Equal(t, "test_counter", metrics[1].Name()) + assert.Equal(t, "counter", metrics[1].Type()) + assert.Equal(t, int64(100), metrics[1].Value()) + + require.NoError(t, mock.ExpectationsWereMet()) + }) + + t.Run("GetAllMetrics_Error", func(t *testing.T) { + builder := sq.Select(`"ID"`, `"MType"`, `"Delta"`, `"Value"`). + From("collector"). + PlaceholderFormat(sq.Dollar) + + query, args, err := builder.ToSql() + if err != nil { + t.Fatalf("failed to build query: %v", err) + } + + driverArgs := make([]driver.Value, len(args)) + for i, a := range args { + driverArgs[i] = a + } + + mock.ExpectQuery(regexp.QuoteMeta(query)). + WithArgs(driverArgs...). + WillReturnError(sql.ErrNoRows) + + metrics, err := repo.GetAllMetrics(context.Background()) + require.Error(t, err) + assert.ErrorIs(t, err, sql.ErrNoRows) + assert.Empty(t, metrics) + + require.NoError(t, mock.ExpectationsWereMet()) + }) +} + +func TestDatabase_UpdateMetric(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("failed to create sqlmock: %v", err) + } + defer db.Close() + + repo := &repo.Database{ + DB: db, + } + + t.Run("UpdateMetric_Gauge", func(t *testing.T) { + builder := sq.Insert("collector"). + Columns(`"ID"`, `"MType"`, `"Delta"`, `"Value"`). + Values("test_gauge", "gauge", nil, 100.0). + Suffix(`ON CONFLICT ("ID") DO UPDATE SET + "Delta" = collector."Delta" + EXCLUDED."Delta", + "Value" = EXCLUDED."Value", + "MType" = EXCLUDED."MType"`). + PlaceholderFormat(sq.Dollar) + + query, args, err := builder.ToSql() + require.NoError(t, err) + + driverArgs := make([]driver.Value, len(args)) + for i, a := range args { + driverArgs[i] = a + } + + mock.ExpectBegin() + + mock.ExpectExec(regexp.QuoteMeta(query)). + WithArgs(driverArgs...). + WillReturnResult(sqlmock.NewResult(1, 1)) + + mock.ExpectCommit() + + err = repo.UpdateMetric(context.Background(), "gauge", "test_gauge", 100.0) + require.NoError(t, err) + + builderGet := sq.Select(`"ID"`, `"MType"`, `"Delta"`, `"Value"`). + From("collector"). + Where(sq.Eq{`"ID"`: "test_gauge", `"MType"`: "gauge"}). + Limit(1). + PlaceholderFormat(sq.Dollar) + queryGet, argsGet, err := builderGet.ToSql() + require.NoError(t, err) + + driverArgsGet := make([]driver.Value, len(argsGet)) + for i, a := range argsGet { + driverArgsGet[i] = a + } + + mock.ExpectQuery(regexp.QuoteMeta(queryGet)). + WithArgs(driverArgsGet...). + WillReturnRows(sqlmock.NewRows([]string{"ID", "MType", "Delta", "Value"}). + AddRow("test_gauge", "gauge", nil, 100.0)) + + metric, err := repo.GetMetric(context.Background(), "gauge", "test_gauge") + require.NoError(t, err) + assert.Equal(t, "test_gauge", metric.Name()) + assert.Equal(t, "gauge", metric.Type()) + assert.Equal(t, 100.0, metric.Value()) + + require.NoError(t, mock.ExpectationsWereMet()) + }) + + t.Run("UpdateMetric_Counter", func(t *testing.T) { + builder := sq.Insert("collector"). + Columns(`"ID"`, `"MType"`, `"Delta"`, `"Value"`). + Values("test_counter", "counter", 100, nil). + Suffix(`ON CONFLICT ("ID") DO UPDATE SET + "Delta" = collector."Delta" + EXCLUDED."Delta", + "Value" = EXCLUDED."Value", + "MType" = EXCLUDED."MType"`). + PlaceholderFormat(sq.Dollar) + + query, args, err := builder.ToSql() + require.NoError(t, err) + + driverArgs := make([]driver.Value, len(args)) + for i, a := range args { + driverArgs[i] = a + } + + mock.ExpectBegin() + + mock.ExpectExec(regexp.QuoteMeta(query)). + WithArgs(driverArgs...). + WillReturnResult(sqlmock.NewResult(1, 1)) + + mock.ExpectCommit() + + err = repo.UpdateMetric(context.Background(), "counter", "test_counter", int64(100)) + require.NoError(t, err) + + builderGet := sq.Select(`"ID"`, `"MType"`, `"Delta"`, `"Value"`). + From("collector"). + Where(sq.Eq{`"ID"`: "test_counter", `"MType"`: "counter"}). + Limit(1). + PlaceholderFormat(sq.Dollar) + queryGet, argsGet, err := builderGet.ToSql() + require.NoError(t, err) + + driverArgsGet := make([]driver.Value, len(argsGet)) + for i, a := range argsGet { + driverArgsGet[i] = a + } + + mock.ExpectQuery(regexp.QuoteMeta(queryGet)). + WithArgs(driverArgsGet...). + WillReturnRows(sqlmock.NewRows([]string{"ID", "MType", "Delta", "Value"}). + AddRow("test_counter", "counter", 100, nil)) + + metric, err := repo.GetMetric(context.Background(), "counter", "test_counter") + require.NoError(t, err) + assert.Equal(t, "test_counter", metric.Name()) + assert.Equal(t, "counter", metric.Type()) + assert.Equal(t, int64(100), metric.Value()) + + require.NoError(t, mock.ExpectationsWereMet()) + }) +} From ab6b7d38b186c0c5fd4d7d9d06c92eeae5353d95 Mon Sep 17 00:00:00 2001 From: Kaplin Artyom Date: Mon, 28 Jul 2025 19:57:44 +0300 Subject: [PATCH 126/126] formatted the code --- cmd/server/cfg_server.go | 2 +- cmd/server/main.go | 1 - internal/handlers/server/server_test.go | 74 ++++++++++++------------- internal/models/counter.go | 1 - internal/models/gauge.go | 1 - internal/models/metrics.go | 1 + internal/repository/postgres_test.go | 12 +++- 7 files changed, 48 insertions(+), 44 deletions(-) diff --git a/cmd/server/cfg_server.go b/cmd/server/cfg_server.go index 9c21c8d..01f1ede 100644 --- a/cmd/server/cfg_server.go +++ b/cmd/server/cfg_server.go @@ -87,7 +87,7 @@ func runE(cmd *cobra.Command, args []string) error { go func() { fmt.Println("pprof listening on :6060") - http.ListenAndServe("localhost:6060", nil) + _ = http.ListenAndServe("localhost:6060", nil) }() ctx, cancel := context.WithCancel(context.Background()) diff --git a/cmd/server/main.go b/cmd/server/main.go index 4467103..068527e 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -9,4 +9,3 @@ func main() { os.Exit(1) } } - diff --git a/internal/handlers/server/server_test.go b/internal/handlers/server/server_test.go index 0c7739a..55694c9 100644 --- a/internal/handlers/server/server_test.go +++ b/internal/handlers/server/server_test.go @@ -175,41 +175,41 @@ func TestGetMetric(t *testing.T) { } func TestGetAllMetrics(t *testing.T) { - opts := &srvCfg.Options{} - for _, opt := range []func(*srvCfg.Options){ - srvCfg.WithAddress("localhost:8080"), - srvCfg.WithStoreInterval(300), - srvCfg.WithFileStoragePath("/tmp/metrics-db.json"), - srvCfg.WithRestoreOnStart(true), - } { - opt(opts) - } - - ctx := context.Background() - storage := repo.NewMemStorage() - - if err := storage.UpdateMetric(ctx, models.GaugeType, "cpu_usage", 75.5); err != nil { - log.Error().Msgf("Failed to update metric cpu_usage: %v", err) - } - - if err := storage.UpdateMetric(ctx, models.CounterType, "requests_total", int64(100)); err != nil { - log.Error().Msgf("Failed to update metric requests_total: %v", err) - } - - metricUsecase := srvUsecase.NewMetricUsecase(storage, storage, storage) - router := router.NewRouter(server.NewServer(metricUsecase, nil), opts) - - t.Run("GetAllMetrics returned HTML metrics", func(t *testing.T) { - req := httptest.NewRequest(http.MethodGet, "/", nil) - rr := httptest.NewRecorder() - - router.ServeHTTP(rr, req) - - assert.Equal(t, http.StatusOK, rr.Code) - body, _ := io.ReadAll(rr.Body) - - assert.Contains(t, string(body), "cpu_usage") - assert.Contains(t, string(body), "requests_total") - assert.Contains(t, string(body), "") - }) + opts := &srvCfg.Options{} + for _, opt := range []func(*srvCfg.Options){ + srvCfg.WithAddress("localhost:8080"), + srvCfg.WithStoreInterval(300), + srvCfg.WithFileStoragePath("/tmp/metrics-db.json"), + srvCfg.WithRestoreOnStart(true), + } { + opt(opts) + } + + ctx := context.Background() + storage := repo.NewMemStorage() + + if err := storage.UpdateMetric(ctx, models.GaugeType, "cpu_usage", 75.5); err != nil { + log.Error().Msgf("Failed to update metric cpu_usage: %v", err) + } + + if err := storage.UpdateMetric(ctx, models.CounterType, "requests_total", int64(100)); err != nil { + log.Error().Msgf("Failed to update metric requests_total: %v", err) + } + + metricUsecase := srvUsecase.NewMetricUsecase(storage, storage, storage) + router := router.NewRouter(server.NewServer(metricUsecase, nil), opts) + + t.Run("GetAllMetrics returned HTML metrics", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/", nil) + rr := httptest.NewRecorder() + + router.ServeHTTP(rr, req) + + assert.Equal(t, http.StatusOK, rr.Code) + body, _ := io.ReadAll(rr.Body) + + assert.Contains(t, string(body), "cpu_usage") + assert.Contains(t, string(body), "requests_total") + assert.Contains(t, string(body), "") + }) } diff --git a/internal/models/counter.go b/internal/models/counter.go index 9b44a79..e9e39c5 100644 --- a/internal/models/counter.go +++ b/internal/models/counter.go @@ -32,4 +32,3 @@ func (c *counter) Update(mValue any) error { c.value += value return nil } - diff --git a/internal/models/gauge.go b/internal/models/gauge.go index 9733350..bbcc871 100644 --- a/internal/models/gauge.go +++ b/internal/models/gauge.go @@ -32,4 +32,3 @@ func (g *gauge) Update(mValue any) error { g.value = value return nil } - diff --git a/internal/models/metrics.go b/internal/models/metrics.go index e3adcca..9c5f22c 100644 --- a/internal/models/metrics.go +++ b/internal/models/metrics.go @@ -3,6 +3,7 @@ package models import ( "errors" ) + type Metric interface { Value() any Name() string diff --git a/internal/repository/postgres_test.go b/internal/repository/postgres_test.go index 996e0e5..8edd194 100644 --- a/internal/repository/postgres_test.go +++ b/internal/repository/postgres_test.go @@ -20,7 +20,9 @@ func TestDatabase_GetMetric(t *testing.T) { if err != nil { t.Fatalf("failed to create sqlmock: %v", err) } - defer db.Close() + defer func() { + _ = db.Close() + }() repo := &repo.Database{ DB: db, @@ -139,7 +141,9 @@ func TestDatabase_GetAllMetrics(t *testing.T) { if err != nil { t.Fatalf("failed to create sqlmock: %v", err) } - defer db.Close() + defer func() { + _ = db.Close() + }() repo := &repo.Database{ DB: db, @@ -214,7 +218,9 @@ func TestDatabase_UpdateMetric(t *testing.T) { if err != nil { t.Fatalf("failed to create sqlmock: %v", err) } - defer db.Close() + defer func() { + _ = db.Close() + }() repo := &repo.Database{ DB: db,