diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index a426622..03c0816 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -6,24 +6,31 @@ on: pull_request: branches: [main, master, develop] +permissions: + contents: write + jobs: CI: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 with: persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal access token. fetch-depth: 0 # otherwise, there would be errors pushing refs to the destination repository. - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v6 with: - go-version: '1.18' + go-version-file: go.mod - name: Verify dependencies run: | - go get -v -t ./... go mod verify + - name: Validate module graph + run: | + go mod tidy + git diff --exit-code go.mod go.sum + - name: Build run: make build @@ -35,15 +42,15 @@ jobs: sudo swapon -s - name: Test - run: make test_coverage + run: make test - name: Go Coverage Badge # Pass the `coverage.out` output to this action - uses: tj-actions/coverage-badge-go@v1.2 + uses: tj-actions/coverage-badge-go@v3 with: filename: coverage.out - name: Verify changed files - uses: tj-actions/verify-changed-files@v9.1 + uses: tj-actions/verify-changed-files@v20 id: verify-changed-files with: files: README.md @@ -58,7 +65,7 @@ jobs: - name: Push changes if: steps.verify-changed-files.outputs.files_changed == 'true' - uses: ad-m/github-push-action@master + uses: ad-m/github-push-action@v1.0.0 with: github_token: ${{ github.token }} - branch: ${{ github.head_ref }} + branch: ${{ github.head_ref || github.ref_name }} diff --git a/.gitignore b/.gitignore index dc2c793..1d88472 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,10 @@ +__pycache__ .idea -test/infra/allocator/config.json +.vscode +allocator +bin/ +coverage*out test/allocator/allocator +test/infra/allocator/config.json test/integration/integration-test -venv -__pycache__ -coverage*out +venv \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml index fe319de..35f40a7 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,349 +1,328 @@ -# This file contains all available configuration options -# with their default values. - -# options for analysis running -run: - # default concurrency is a available CPU number - concurrency: 4 - - # timeout for analysis, e.g. 30s, 5m, default is 1m - timeout: 5m - - # exit code when at least one issue was found, default is 1 - issues-exit-code: 1 - - # include test files or not, default is true - tests: true - - # list of build tags, all linters use it. Default is empty list. - build-tags: - - mytag - - # which dirs to skip: issues from them won't be reported; - # can use regexp here: generated.*, regexp is applied on full path; - # default value is empty list, but default dirs are skipped independently - # from this option's value (see skip-dirs-use-default). - skip-dirs: - - src/external_libs - - autogenerated_by_my_lib - - # default is true. Enables skipping of directories: - # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ - skip-dirs-use-default: true - - # which files to skip: they will be analyzed, but issues from them - # won't be reported. Default value is empty list, but there is - # no need to include all autogenerated files, we confidently recognize - # autogenerated files. If it's not please let us know. - skip-files: - - ".*\\.my\\.go$" - - lib/bad.go - - # by default isn't set. If set we pass it to "go list -mod={option}". From "go help modules": - # If invoked with -mod=readonly, the go command is disallowed from the implicit - # automatic updating of go.mod described above. Instead, it fails when any changes - # to go.mod are needed. This setting is most useful to check that go.mod does - # not need updates, such as in a continuous integration and testing system. - # If invoked with -mod=vendor, the go command assumes that the vendor - # directory holds the correct copies of dependencies and ignores - # the dependency descriptions in go.mod. - # modules-download-mode: readonly|release|vendor - - -# output configuration options -output: - # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" - format: colored-line-number - - # print lines of code with issue, default is true - print-issued-lines: true - - # print linter name in the end of issue text, default is true - print-linter-name: true - - # make issues output unique by line, default is true - uniq-by-line: true - - -# all available settings of specific linters -linters-settings: - dogsled: - # checks assignments with too many blank identifiers; default is 2 - max-blank-identifiers: 2 - dupl: - # tokens count to trigger issue, 150 by default - threshold: 100 - errcheck: - # report about not checking of errors in type assertions: `a := b.(MyStruct)`; - # default is false: such cases aren't reported by default. - check-type-assertions: false - - # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; - # default is false: such cases aren't reported by default. - check-blank: false - - # [deprecated] comma-separated list of pairs of the form pkg:regex - # the regex is used to ignore names within pkg. (default "fmt:.*"). - # see https://github.com/kisielk/errcheck#the-deprecated-method for details - ignore: fmt:.*,io/ioutil:^Read.* - - # path to a file containing a list of functions to exclude from checking - # see https://github.com/kisielk/errcheck#excluding-functions for details - # exclude: /path/to/file.txt - funlen: - lines: 80 - statements: 40 - gocognit: - # minimal code complexity to report, 30 by default (but we recommend 10-20) - min-complexity: 20 - goconst: - # minimal length of string constant, 3 by default - min-len: 3 - # minimal occurrences count to trigger, 3 by default - min-occurrences: 3 - gocritic: - # Which checks should be enabled; can't be combined with 'disabled-checks'; - # See https://go-critic.github.io/overview#checks-overview - # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run` - # By default list of stable checks is used. - # enabled-checks: - # большинство проверок уже включены, сюда можно писать просто дополнительные - - # Which checks should be disabled; can't be combined with 'enabled-checks'; default is empty - disabled-checks: - # - regexpMust - - whyNoLint # слишком много писать, увы... - - dupImport # ломается на CGO: https://github.com/go-critic/go-critic/issues/845 - - octalLiteral # странный линтер, непонятно что хочет - - # Enable multiple checks by tags, run `GL_DEBUG=gocritic golangci-lint run` to see all tags and checks. - # Empty list by default. See https://github.com/go-critic/go-critic#usage -> section "Tags". - enabled-tags: - - diagnostic - - style - - performance - - experimental - - opinionated - - settings: # settings passed to gocritic - captLocal: # must be valid enabled check name - paramsOnly: true - rangeValCopy: - sizeThreshold: 32 - gocyclo: - # minimal code complexity to report, 30 by default (but we recommend 10-20) - min-complexity: 10 - godox: - # report any comments starting with keywords, this is useful for TODO or FIXME comments that - # might be left in the code accidentally and should be resolved before merging - keywords: # default keywords are TODO, BUG, and FIXME, these can be overwritten by this setting - - NOTE - - OPTIMIZE # marks code that should be optimized before merging - - HACK # marks hack-arounds that should be removed before merging - gofmt: - # simplify code: gofmt with `-s` option, true by default - simplify: true - goimports: - # put imports beginning with prefix after 3rd-party packages; - # it's a comma-separated list of prefixes - local-prefixes: github.com/org/project - golint: - # minimal confidence for issues, default is 0.8 - min-confidence: 0.21 - gomnd: - settings: - mnd: - # the list of enabled checks, see https://github.com/tommy-muehle/go-mnd/#checks for description. - checks: argument,case,condition,operation,return,assign - ignored-numbers: 0,1 - govet: - # report about shadowed variables - check-shadowing: true - # settings per analyzer - settings: - printf: # analyzer name, run `go tool vet help` to see all analyzers - funcs: # run `go tool vet help printf` to see available settings for `printf` analyzer - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf - - # enable or disable analyzers by name - enable-all: true - disable-all: false - depguard: - list-type: blacklist - include-go-root: false - packages: - - github.com/sirupsen/logrus - packages-with-error-message: - # specify an error message to output when a blacklisted package is used - - github.com/sirupsen/logrus: "logging is allowed only by logutils.Log" - lll: - # max line length, lines longer will be reported. Default is 120. - # '\t' is counted as 1 character by default, and can be changed with the tab-width option - line-length: 140 - # tab width in spaces. Default to 1. - tab-width: 1 - maligned: - # print struct with more effective memory layout or not, false by default - suggest-new: true - misspell: - # Correct spellings using locale preferences for US or UK. - # Default is to use a neutral variety of English. - # Setting locale to US will correct the British spelling of 'colour' to 'color'. - locale: US - ignore-words: - - someword - nakedret: - # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 - max-func-lines: 30 - prealloc: - # XXX: we don't recommend using this linter before doing performance profiling. - # For most programs usage of prealloc will be a premature optimization. - - # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them. - # True by default. - simple: true - range-loops: true # Report preallocation suggestions on range loops, true by default - for-loops: false # Report preallocation suggestions on for loops, false by default - rowserrcheck: - packages: - - github.com/jmoiron/sqlx - tagliatelle: - # check the struck tag name case - case: - # use the struct field name to check the name of the struct tag - use-field-name: true - rules: - # any struct tag type can be used. - # support string case: `camel`, `pascal`, `kebab`, `snake`, `goCamel`, `goPascal`, `goKebab`, `goSnake`, `upper`, `lower` - json: snake - yaml: snake - xml: snake - bson: snake - avro: snake - mapstructure: kebab - unparam: - # Inspect exported functions, default is false. Set to true if no external program/library imports your code. - # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors: - # if it's called for subdir of a project it can't find external interfaces. All text editor integrations - # with golangci-lint call it on a directory with the changed file. - check-exported: false - unused: - # treat code as a program (not a library) and report unused exported identifiers; default is false. - # XXX: if you enable this setting, unused will report a lot of false-positives in text editors: - # if it's called for subdir of a project it can't find funcs usages. All text editor integrations - # with golangci-lint call it on a directory with the changed file. - check-exported: false - whitespace: - multi-if: false # Enforces newlines (or comments) after every multi-line if statement - multi-func: false # Enforces newlines (or comments) after every multi-line function signature - wrapcheck: - ignorePackageGlobs: - - gitlab.stageoffice.ru/UCS-COMMON/utils/breaker - - github.com/stretchr/testify* - - google.golang.org/grpc* - wsl: - # If true append is only allowed to be cuddled if appending value is - # matching variables, fields or types on line above. Default is true. - strict-append: true - # Allow calls and assignments to be cuddled as long as the lines have any - # matching variables, fields or types. Default is true. - allow-assign-and-call: false - # Allow multiline assignments to be cuddled. Default is true. - allow-multiline-assign: false - # Allow declarations (var) to be cuddled. - allow-cuddle-declarations: false - # Allow trailing comments in ending of blocks - allow-trailing-comment: false - # Force newlines in end of case at this limit (0 = never). - force-case-trailing-whitespace: 0 +# Defines the configuration version. +# The only possible value is "2". +version: "2" linters: - enable-all: true - fast: false + # Default set of linters. + # The value can be: + # - `standard`: the "Default" linters https://golangci-lint.run/docs/linters/#all-linters + # - `all`: enables all linters by default. + # - `none`: disables all linters by default. + # - `fast`: enables only linters considered as "fast". + # Default: standard + default: all + + # Disable specific linters. disable: - - promlinter # too many bugs - - gci # relation with goimports is not clear - - gofumpt # conflicts with wsl - - godox # useless - - exhaustivestruct # bloats code - - paralleltest # it's not clear when and how to use t.Parallel() - - testpackage # seriously limits testing capabilities + - err113 # Often over-constrains error construction. + - exhaustruct # Huge maintenance tax on external structs. + - godox # Useless. + - ireturn # Too dogmatic for Go APIs, interfaces at boundaries are normal. + - lll # Replaced by golines formatter. + - mnd # Noisy in tests and math-heavy code. + - noinlineerr # Directly contradicts idiomatic if err := ...; err != nil style. + - paralleltest # It's not clear when and how to use t.Parallel(). + - promlinter # Too many bugs. + - testpackage # Seriously limits testing capabilities. + - varnamelen # Fights idiomatic Go short local names and creates review noise. + - wrapcheck # It causes "wrap for wrapping's sake" issues. + - wsl # Deprecated. + + # All available settings of specific linters. + settings: + depguard: + # Rules to apply. + # Default (applies if no custom rules are defined): only allow $gostd in all files. + rules: + # Name of a rule. + logger: + # logrus is allowed to use only in logutils package. + # List of file globs that will match this rule. + # Default: $all + files: + - "!**/pkg/logutils/**.go" + # List of packages that are not allowed. + # Default: [] + deny: + - pkg: github.com/sirupsen/logrus + desc: logging is allowed only by logutils.Log + + dupl: + # tokens count to trigger issue, 150 by default + threshold: 100 + + errcheck: + # old `ignore: fmt:.*,os:^Read.*` was removed in v2. + # `fmt:.*` remains covered by errcheck built-ins. + # `os:^Read.*` has to be approximated with explicit functions. + # List of functions to exclude from checking. + # See https://github.com/kisielk/errcheck#excluding-functions for details. + exclude-functions: + - os.ReadDir + - os.ReadFile + - os.Readlink + + funlen: + # Checks the number of lines in a function. + # If lower than 0, disable the check. + # Default: 60 + lines: 80 + + gocognit: + # minimal code complexity to report, 30 by default (but we recommend 10-20) + min-complexity: 20 + + gocritic: + # Which checks should be enabled in addition to default checks; can't be combined with 'disabled-checks'. + # By default, list of stable checks is used (https://go-critic.com/overview#checks-overview). + # To see which checks are enabled run `GL_DEBUG=gocritic golangci-lint run --enable=gocritic`. + # enabled-checks: + + # Which checks should be disabled; can't be combined with 'enabled-checks'. + # Default: [] + disabled-checks: + # - regexpMust + - whyNoLint # Too much information, alas... + - dupImport # Breaks on CGO: https://github.com/go-critic/go-critic/issues/845 + - octalLiteral # Strange linter, I don't know what it wants. + + # Enable multiple checks by tags in addition to default checks. + # Run `GL_DEBUG=gocritic golangci-lint run --enable=gocritic` to see all tags and checks. + # See https://github.com/go-critic/go-critic#usage -> section "Tags". + # Default: [] + enabled-tags: + - diagnostic + - style + - performance + - experimental + - opinionated + + # Settings passed to gocritic. + # The settings key is the name of a supported gocritic checker. + # The list of supported checkers can be found at https://go-critic.com/overview. + settings: + # Must be valid enabled check name. + rangeValCopy: + sizeThreshold: 32 + + gocyclo: + # minimal code complexity to report, 30 by default (but we recommend 10-20) + min-complexity: 10 + + godox: + # report any comments starting with keywords, this is useful for TODO or FIXME comments that + # might be left in the code accidentally and should be resolved before merging + keywords: # default keywords are TODO, BUG, and FIXME, these can be overwritten by this setting + - NOTE + - OPTIMIZE # marks code that should be optimized before merging + - HACK # marks hack-arounds that should be removed before merging + + govet: + # Settings per analyzer. + settings: + # Analyzer name, run `go tool vet help` to see all analyzers. + printf: + # Comma-separated list of print function names to check + # (in addition to defaults, see `go tool vet help printf`). + # Default: [] + funcs: + - (github.com/golangci/golangci-lint/v2/pkg/logutils.Log).Infof + - (github.com/golangci/golangci-lint/v2/pkg/logutils.Log).Warnf + - (github.com/golangci/golangci-lint/v2/pkg/logutils.Log).Errorf + - (github.com/golangci/golangci-lint/v2/pkg/logutils.Log).Fatalf + + # Enable analyzers by name. + # Default: [] + enable: + - shadow + enable-all: true + # Disable fieldalignment because it conflicts with embeddedstructfieldcheck + # ordering and hurts readability in embedded/mock-heavy structs. + disable: + - fieldalignment + + misspell: + # Correct spellings using locale preferences for US or UK. + # Setting locale to US will correct the British spelling of 'colour' to 'color'. + # Default is to use a neutral variety of English. + locale: US + # Typos to ignore. + # Should be in lower case. + # Default: [] + ignore-rules: + - someword + + mnd: + # List of numbers to exclude from analysis. + # The numbers should be written as string. + # Values always ignored: "1", "1.0", "0" and "0.0". + # Default: [] + ignored-numbers: + - "0" + - "1" + + rowserrcheck: + # database/sql is always checked. + # Default: [] + packages: + - github.com/jmoiron/sqlx + + tagliatelle: + # Checks the struct tag name case. + case: + # Use the struct field name to check the name of the struct tag. + use-field-name: true + # Defines the association between tag name and case. + # Any struct tag name can be used. + # Supported string cases: + # - `camel`, `pascal`, `kebab`, `snake`, `upperSnake` + # - `goCamel`, `goPascal`, `goKebab`, `goSnake` + # - `upper`, `lower`, `header` + rules: + json: snake + yaml: snake + xml: snake + bson: snake + avro: snake + mapstructure: kebab + + wrapcheck: + # An array of strings that specify globs of packages to ignore. + # Default: [] + ignore-package-globs: + - gitlab.stageoffice.ru/UCS-COMMON/utils/breaker + - github.com/stretchr/testify* + - google.golang.org/grpc* + + wsl: + # Allow calls and assignments to be cuddled as long as the lines have any + # matching variables, fields or types. Default is true. + allow-assign-and-call: false + # Allow multiline assignments to be cuddled. Default is true. + allow-multiline-assign: false + + # Defines a set of rules to ignore issues. + # It does not skip analysis and does not ignore "typecheck" errors. + exclusions: + # Mode of the generated files analysis. + # Default: strict + generated: lax + + # Excluding configuration per-path, per-linter, per-text and per-source. + rules: + - path: _test\.go + linters: + - gocyclo + - dupl + - gosec + - funlen + - gocognit + - mnd + + - path: "test_.*\\.go$" + linters: + - funlen + - mnd + - gocognit + + - path: mock.go + text: "exported type" + linters: + - revive + + - path: mock.go + text: "exported function" + linters: + - revive + + - path: mock.go + text: "exported method" + linters: + - revive + + - path: mock.go + linters: + - mnd + + - path: config.go + text: "fieldalignment" + + # Which file paths to exclude: they will be analyzed, + # but issues from them won't be reported. + # Default: [] + paths: + - vendor$ + - third_party$ + - testdata$ + - examples$ + - Godeps$ + - builtin$ + - src/external_libs + - autogenerated_by_my_lib + - ".*\\.my\\.go$" + - lib/bad.go + +formatters: + # Enable specific formatters. + # Default: [] (uses standard Go formatting) + enable: + - gofmt + - goimports + - golines + + # Formatters settings. + settings: + goimports: + # A list of prefixes, which, if set, checks import paths + # with the given prefixes are grouped after 3rd-party packages. + # Default: [] + local-prefixes: + - github.com/org/project + golines: + # Target maximum line length. + # Default: 100 + max-len: 140 + + # Formatter exclusions. + exclusions: + # Which file paths to exclude. + # Default: [] + paths: + - vendor$ + - third_party$ + - testdata$ + - examples$ + - Godeps$ + - builtin$ + - src/external_libs + - autogenerated_by_my_lib + - ".*\\.my\\.go$" + - lib/bad.go issues: - # List of regexps of issue texts to exclude, empty list by default. - # But independently from this option we use default exclude patterns, - # it can be disabled by `exclude-use-default: false`. To list all - # excluded by default patterns execute `golangci-lint run --help` - exclude: - - abcdef - - # Excluding configuration per-path, per-linter, per-text and per-source - exclude-rules: - - path: _test\.go - linters: - - gocyclo - - dupl - - gosec - - funlen - - gocognit - - gomnd - - - path: "test_.*\\.go$" - linters: - - funlen - - gomnd - - gocognit - - - path: mock.go - text: "exported type" - linters: - - golint - - revive - - - path: mock.go - text: "exported function" - linters: - - golint - - revive - - - path: mock.go - text: "exported method" - linters: - - golint - - revive - - - path: mock.go - linters: - - gomnd - - - path: config.go - text: "fieldalignment" - - # Independently from option `exclude` we use default exclude patterns, - # it can be disabled by this option. To list all - # excluded by default patterns execute `golangci-lint run --help`. - # Default value for this option is true. - exclude-use-default: false - - # Maximum issues count per one linter. Set to 0 to disable. Default is 50. + # Maximum issues count per one linter. + # Set to 0 to disable. + # Default: 50 max-issues-per-linter: 0 - # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. + # Maximum count of issues with the same text. + # Set to 0 to disable. + # Default: 3 max-same-issues: 0 - # Show only new issues: if there are unstaged changes or untracked files, - # only those changes are analyzed, else only changes in HEAD~ are analyzed. - # It's a super-useful option for integration of golangci-lint into existing - # large codebase. It's not practical to fix all existing issues at the moment - # of integration: much better don't allow issues in new code. - # Default is false. - new: false +# Options for analysis running. +run: + # Number of operating system threads (`GOMAXPROCS`) that can execute golangci-lint simultaneously. + # Default: 0 (automatically set to match Linux container CPU quota and + # fall back to the number of logical CPUs in the machine) + concurrency: 4 - # Show only new issues created after git revision `REV` - # new-from-rev: REV + # Timeout for total work, e.g. 30s, 5m, 5m30s. + # If the value is lower or equal to 0, the timeout is disabled. + # Default: 0 (disabled) + timeout: 5m - # Show only new issues created in git patch with set file path. - # new-from-patch: path/to/patch/file + # List of build tags, all linters use it. + # Default: [] + build-tags: + - mytag diff --git a/Makefile b/Makefile index 0ded4d6..9ce48b9 100644 --- a/Makefile +++ b/Makefile @@ -1,31 +1,92 @@ +SHELL := /bin/bash + +.ONESHELL: +.SHELLFLAGS := -euo pipefail -c + +# By default, show help. +.DEFAULT_GOAL := help + +LOCAL_BIN := $(CURDIR)/bin +GOLANGCI_LINT := $(LOCAL_BIN)/golangci-lint +GOLANGCI_LINT_VERSION := v2.11.4 +GOLANGCI_LINT_INSTALL_METHOD := build + +UNIT_TEST_PACKAGES := $(shell go list ./...) +UNIT_COVERAGE_FILE := coverage.unit.out +INTEGRATION_COVERAGE_FILE := coverage.integration.out +OVERALL_COVERAGE_FILE := coverage.overall.out +COVERAGE_SUMMARY_FILE := coverage.out +INTEGRATION_TEST_BIN := ./test/integration/integration-test + +.PHONY: help +help: +## Show help for available targets. + @awk -f scripts/make_help.awk $(MAKEFILE_LIST) + +.PHONY: install-lint +install-lint: +## Install pinned golangci-lint version into ./bin. +## Use scripts/install_golangci_lint.sh with selected install method. + @mkdir -p "$(LOCAL_BIN)" + @sh -s -- "$(LOCAL_BIN)" "$(GOLANGCI_LINT_VERSION)" "$(GOLANGCI_LINT_INSTALL_METHOD)" < scripts/install_golangci_lint.sh + +.PHONY: generate +generate: +## Regenerate protobuf stubs for allocator schema. +## Requires protoc to be installed: https://grpc.io/docs/protoc-installation/ + @command -v protoc >/dev/null || { \ + echo "error: protoc is required; install it first: https://grpc.io/docs/protoc-installation/"; \ + exit 1; \ + } + cd test/allocator/schema && ./generate.sh + +.PHONY: build build: +## Build all Go packages. +## Also build allocator demo binary. go build ./... - cd ./test/allocator && go build . && cd - - -#UNIT_TEST_PACKAGES=$(shell go list ./... | grep -v test/integration test/allocator/app) -UNIT_TEST_PACKAGES=$(shell go list ./...) -unit_test: - go test -v -count=1 -cover $(UNIT_TEST_PACKAGES) -coverprofile=coverage.unit.out -coverpkg ./... - -integration_test: - go test -c ./test/integration/main_test.go -o ./test/integration/integration-test -coverpkg ./... - ./test/integration/integration-test -test.v -test.coverprofile=coverage.integration.out - -test_coverage: unit_test integration_test - # merge outputs from unit and integration testing - cp coverage.unit.out coverage.overall.out - tail --lines=+2 coverage.integration.out >> coverage.overall.out - # cannot cover main function and CLI package - sed -i '/test\/allocator\/app/d' ./coverage.overall.out - sed -i '/test\/allocator\/main.go/d' ./coverage.overall.out - # final report - go tool cover -func=coverage.overall.out -o=coverage.out - -fix: - go fmt . - go mod tidy + go build ./test/allocator + +.PHONY: unit-test +unit-test: +## Run unit tests with coverage for all packages. + go test -v -count=1 -cover $(UNIT_TEST_PACKAGES) -coverprofile=$(UNIT_COVERAGE_FILE) -coverpkg ./... + +.PHONY: integration-test +integration-test: +## Build and run integration test binary. +## Write integration coverage to a separate profile. + go test -c ./test/integration -o $(INTEGRATION_TEST_BIN) -coverpkg ./... + $(INTEGRATION_TEST_BIN) -test.v -test.coverprofile=$(INTEGRATION_COVERAGE_FILE) -lint: - golangci-lint run -c .golangci.yml ./... +.PHONY: test +test: unit-test integration-test +## Merge unit and integration coverage reports. +## Produce a human-readable coverage summary. + ./scripts/merge_coverage.sh \ + "$(UNIT_COVERAGE_FILE)" \ + "$(INTEGRATION_COVERAGE_FILE)" \ + "$(OVERALL_COVERAGE_FILE)" \ + "$(COVERAGE_SUMMARY_FILE)" + +.PHONY: lint +lint: install-lint +## Analyze code locally. +## Use project-local golangci-lint from ./bin. +## Verify linter config before checks. + $(GOLANGCI_LINT) config verify + $(GOLANGCI_LINT) run ./... + +.PHONY: fix +fix: install-lint +## Apply automatic source fixes. +## Run go mod tidy, and golangci-lint --fix. + go mod tidy + $(GOLANGCI_LINT) config verify + $(GOLANGCI_LINT) run --fix ./... -.PHONY: test \ No newline at end of file +.PHONY: clean +clean: +## Remove generated test and coverage artifacts. +## Keep workspace clean between test runs. + rm -f $(UNIT_COVERAGE_FILE) $(INTEGRATION_COVERAGE_FILE) $(OVERALL_COVERAGE_FILE) $(COVERAGE_SUMMARY_FILE) $(INTEGRATION_TEST_BIN) diff --git a/README.md b/README.md index 2006f1a..91cfece 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,54 @@ +# MemLimiter + [![Go Reference](https://pkg.go.dev/badge/github.com/newcloudtechnologies/memlimiter.svg)](https://pkg.go.dev/github.com/newcloudtechnologies/memlimiter) [![Go Report Card](https://goreportcard.com/badge/github.com/newcloudtechnologies/memlimiter)](https://goreportcard.com/report/github.com/newcloudtechnologies/memlimiter) -![Coverage](https://img.shields.io/badge/Coverage-80.1%25-brightgreen) +![Coverage](https://img.shields.io/badge/Coverage-81.5%25-brightgreen) ![CI](https://github.com/newcloudtechnologies/memlimiter/actions/workflows/CI.yml/badge.svg) -# MemLimiter +`memlimiter` helps a Go service avoid OOM by combining adaptive GC tuning and request throttling under memory pressure. -Library that helps to limit memory consumption of your Go service. +It observes process memory (`RSS`) and Go heap pressure (`runtime.MemStats.NextGC`) and turns that into: -## Working principles -As of today (Go 1.18), there is a possibility for any Go application to be eventually stopped by OOM killer. The memory leak is because Go runtime knows nothing about the limitations imposed on the process by the operating system (for instance, using `cgroups`). However, an unexpected termination of a process because of OOM is highly undesirable, as it can lead to cache resetting, data integrity violation, distributed transaction hanging and even cascading failure of a distributed backend. Therefore, services should degrade gracefully instead of immediate stop due to `SIGKILL`. +- dynamic `debug.SetGCPercent` tuning, +- request shedding / backpressure via middleware. + +By default, stats come from: + +- [`runtime.ReadMemStats`](https://pkg.go.dev/runtime#ReadMemStats) for Go heap state, +- `gopsutil` for process RSS. + +For cgo/external-memory workloads, applications should provide their own `stats.ServiceStatsSubscription` and report non-Go allocations through `ConsumptionReport.Cgo`. + +The repo also includes: + +- gRPC middleware for admission control, +- an allocator demo under `test/allocator`, +- integration tests and plotting scripts. + +## Is this still needed on Go 1.26.1? -A universal solution for programming languages with automatic memory management comprises two parts: +For pure-Go services, usually not as a first step: start with `GOMEMLIMIT` / `runtime/debug.SetMemoryLimit` and standard admission control (see [`SetMemoryLimit`](https://pkg.go.dev/runtime/debug#SetMemoryLimit) and [Go 1.19 runtime notes](https://go.dev/doc/go1.19#runtime)). -1. **Garbage collection intensification**. The more often GC starts, the more garbage will be collected, the fewer new physical memory allocations we have to make for the service’s business logic. -2. **Request throttling**. By suppressing some of the incoming requests, we implement the backpressure: the middleware simply cuts off part of the load coming from the client in order to avoid too many memory allocations. +For cgo-heavy or mixed-memory services, it can still be useful because the Go memory limit does not account for external C allocations. In that setup, `memlimiter` can reduce the Go-side budget as external memory grows and apply backpressure. -MemLimiter represents a memory budget [automated control system](https://en.wikipedia.org/wiki/Control_system) that helps to keep the memory consumption of a Go service within a predefined limit. +## When memlimiter fits best in 2026 + +- You need explicit accounting of external/cgo memory. +- You want dynamic Go-side budget reduction. +- You need request shedding under pressure. + +## Go memory references + +- [A Guide to the Go Garbage Collector](https://go.dev/doc/gc-guide) +- [`runtime/debug.SetMemoryLimit`](https://pkg.go.dev/runtime/debug#SetMemoryLimit) +- [`runtime.MemStats`](https://pkg.go.dev/runtime#MemStats) + +## Working principles + +MemLimiter is a memory-budget [automated control system](https://en.wikipedia.org/wiki/Control_system) that combines: + +1. **Garbage collection intensification**. The more often GC starts, the more garbage is collected, so fewer new physical allocations are needed for business logic. +2. **Request throttling**. By suppressing part of incoming requests, middleware applies backpressure and reduces allocation pressure. ### Memory budget utilization @@ -24,9 +57,10 @@ The core of the MemLimiter is a special object quite similar to [P-controller](h $$ Utilization = \frac {NextGC} {RSS_{limit} - CGO} $$ where: -* $NextGC$ ([from here](https://pkg.go.dev/runtime#MemStats)) is a target size for heap, upon reaching which the Go runtime will launch the GC next time; -* $RSS_{limit}$ is a hard limit for service's physical memory (`RSS`) consumption (so that exceeding this limit will highly likely result in OOM); -* $CGO$ is a total size of heap allocations made beyond `Cgo` borders (within `C`/`C++`/.... libraries). + +- $NextGC$ ([from here](https://pkg.go.dev/runtime#MemStats)) is a target size for heap, upon reaching which the Go runtime will launch the GC next time; +- $RSS_{limit}$ is a hard limit for service's physical memory (`RSS`) consumption (so that exceeding this limit will highly likely result in OOM); +- $CGO$ is a total size of heap allocations made beyond `Cgo` borders (within `C`/`C++`/.... libraries). A few notes about $CGO$ component. Allocations made outside of the Go allocator, of course, are not controlled by the Go runtime in any way. At the same time, the memory consumption limit is common for both Go and non-Go allocators. Therefore, if non-Go allocations grow, all we can do is shrink the memory budget for Go allocations (which is why we subtract $CGO$ from the denominator of the previous expression). If your service uses `Cgo`, you need to figure out how much memory is allocated “on the other side” – **otherwise MemLimiter won’t be able to save your service from OOM**. @@ -53,7 +87,6 @@ $$ Output = \begin{cases} Finally we convert the dimensionless quantity $Output$ into specific $GOGC$ (for the further use in [`debug.SetGCPercent`](https://pkg.go.dev/runtime/debug#SetGCPercent)) and $Throttling$ (percentage of suppressed requests) values, however, only if the $Utilization$ exceeds the specified limits: - $$ GC = \begin{cases} \displaystyle Output \ \ \ Utilization \gt DangerZoneGC \\ \displaystyle 100 \ \ \ \ \ \ \ \ \ \ otherwise \\ @@ -89,24 +122,24 @@ You must also provide your own `stats.ServiceStatsSubscription` and `stats.Servi There are several key settings in MemLimiter [configuration](controller/nextgc/config.go): -* `RSSLimit` -* `DangerZoneGC` -* `DangerZoneThrottling` -* `Period` -* `WindowSize` -* `Coefficient` ($C_{p}$) +- `RSSLimit` +- `DangerZoneGC` +- `DangerZoneThrottling` +- `Period` +- `WindowSize` +- `Coefficient` ($C_{p}$) You have to pick them empirically for your service. The settings must correspond to the business logic features of a particular service and to the workload expected. We made a series of performance tests with [Allocator][test/allocator] - an example service which does nothing but allocations that reside in memory for some time. We used different settings, applied the same load and tracked the RSS of a process. Settings ranges: -* $RSS_{limit} = {1G}$ -* $DangerZoneGC = 50%$ -* $DangerZoneThrottling = 90%$ -* $Period = 100ms$ -* $WindowSize = 20$ -* $C_{p} \in \\{0, 0.5, 1, 5, 10, 50, 100\\}$ +- $RSS_{limit} = {1G}$ +- $DangerZoneGC = 50%$ +- $DangerZoneThrottling = 90%$ +- $Period = 100ms$ +- $WindowSize = 20$ +- $C_{p} \in \\{0, 0.5, 1, 5, 10, 50, 100\\}$ These plots may give you some inspiration on how $C_{p}$ value affects the physical memory consumption other things being equal: @@ -117,21 +150,21 @@ And the summary plot with RSS consumption dependence on $C_{p}$ value: ![RSS](docs/rss_hl.png) The general conclusion is that: -* The higher the $C_{p}$ is, the lower the $RSS$ consumption. -* Too low and too high $C_{p}$ values cause self-oscillation of control parameters. -* Disabling MemLimiter causes OOM. +- The higher the $C_{p}$ is, the lower the $RSS$ consumption. +- Too low and too high $C_{p}$ values cause self-oscillation of control parameters. +- Disabling MemLimiter causes OOM. ## TODO -* Extend middleware.Middleware to support more frameworks. -* Add GOGC limitations to prevent death spirals. -* Support popular Cgo allocators like Jemalloc or TCMalloc, parse their stats to provide information about Cgo memory consumption. +- Extend middleware.Middleware to support more frameworks. +- Add GOGC limitations to prevent death spirals. +- Support popular Cgo allocators like Jemalloc or TCMalloc, parse their stats to provide information about Cgo memory consumption. Your PRs are welcome! ## Publications -* Isaev V. A. Go runtime high memory consumption (in Russian). Evrone Go meetup. 2022. +- Isaev V. A. Go runtime high memory consumption (in Russian). Evrone Go meetup. 2022.\ [![Preview](https://yt-embed.herokuapp.com/embed?v=_BbhmaZupqs)]( https://www.youtube.com/watch?v=_BbhmaZupqs ) diff --git a/backpressure/doc.go b/backpressure/doc.go index 2751a29..f898e50 100644 --- a/backpressure/doc.go +++ b/backpressure/doc.go @@ -5,5 +5,5 @@ */ // Package backpressure contains code applying control signals issued by controller to Go runtime and -// and to gRPC server. +// to gRPC server. package backpressure diff --git a/backpressure/operator.go b/backpressure/operator.go index 5ad77ca..0b0085c 100644 --- a/backpressure/operator.go +++ b/backpressure/operator.go @@ -7,26 +7,47 @@ package backpressure import ( + "fmt" "runtime/debug" "sync/atomic" "github.com/go-logr/logr" "github.com/newcloudtechnologies/memlimiter/stats" - "github.com/pkg/errors" ) var _ Operator = (*operatorImpl)(nil) +// operatorImpl is the implementation of the Operator interface. type operatorImpl struct { *throttler + notificationChan chan<- *stats.MemLimiterStats lastControlParameters atomic.Value logger logr.Logger } +// NewOperator constructs a new Operator. +func NewOperator(logger logr.Logger, options ...Option) Operator { + out := &operatorImpl{ + logger: logger, + throttler: newThrottler(), + } + + //nolint:gocritic + for _, op := range options { + switch t := op.(type) { + case *notificationsOption: + out.notificationChan = t.val + } + } + + return out +} + +// GetStats returns the current backpressure stats. func (b *operatorImpl) GetStats() (*stats.BackpressureStats, error) { result := &stats.BackpressureStats{ - Throttling: b.throttler.getStats(), + Throttling: b.getStats(), } lastControlParameters := b.lastControlParameters.Load() @@ -35,37 +56,44 @@ func (b *operatorImpl) GetStats() (*stats.BackpressureStats, error) { result.ControlParameters, ok = lastControlParameters.(*stats.ControlParameters) if !ok { - return nil, errors.Errorf("ivalid type cast (%T)", lastControlParameters) + return nil, fmt.Errorf("invalid type cast (%T)", lastControlParameters) } } return result, nil } +// SetControlParameters sets the control parameters. func (b *operatorImpl) SetControlParameters(value *stats.ControlParameters) error { old := b.lastControlParameters.Swap(value) if old != nil { - // if control parameters didn't change, we do nothing - if value.EqualsTo(old.(*stats.ControlParameters)) { + oldControlParameters, ok := old.(*stats.ControlParameters) + if !ok { + return fmt.Errorf("invalid type cast (%T)", old) + } + + // If control parameters didn't change, we do nothing. + if value.EqualsTo(oldControlParameters) { return nil } } - // set the share of the requests that have to be throttled - if err := b.throttler.setThreshold(value.ThrottlingPercentage); err != nil { - return errors.Wrap(err, "throttler set threshold") + // Set the share of the requests that have to be throttled. + err := b.setThreshold(value.ThrottlingPercentage) + if err != nil { + return fmt.Errorf("throttler set threshold: %w", err) } - // tune GC pace + // Tune GC pace. debug.SetGCPercent(value.GOGC) b.logger.Info("control parameters changed", value.ToKeysAndValues()...) - // notify client about statistics change + // Notify client about statistics change. if b.notificationChan != nil { backpressureStats, err := b.GetStats() if err != nil { - return errors.Wrap(err, "get stats") + return fmt.Errorf("get stats: %w", err) } memLimiterStats := &stats.MemLimiterStats{ @@ -73,7 +101,7 @@ func (b *operatorImpl) SetControlParameters(value *stats.ControlParameters) erro Backpressure: backpressureStats, } - // if client's not ready to read, omit it + // If client's not ready to read, omit it. select { case b.notificationChan <- memLimiterStats: default: @@ -82,21 +110,3 @@ func (b *operatorImpl) SetControlParameters(value *stats.ControlParameters) erro return nil } - -// NewOperator builds new Operator. -func NewOperator(logger logr.Logger, options ...Option) Operator { - out := &operatorImpl{ - logger: logger, - throttler: newThrottler(), - } - - //nolint:gocritic - for _, op := range options { - switch t := op.(type) { - case *notificationsOption: - out.notificationChan = t.val - } - } - - return out -} diff --git a/backpressure/throttler.go b/backpressure/throttler.go index eb2ebb0..01caec6 100644 --- a/backpressure/throttler.go +++ b/backpressure/throttler.go @@ -7,71 +7,58 @@ package backpressure import ( + "errors" + "math/rand/v2" "sync/atomic" "github.com/newcloudtechnologies/memlimiter/stats" "github.com/newcloudtechnologies/memlimiter/utils" - "github.com/villenny/fastrand64-go" - - "github.com/pkg/errors" ) +// throttler is a struct that implements the throttler. +// It must not be copied after first use because it contains atomic fields. +// It is safe for concurrent use. type throttler struct { - // group of request counters - requestsTotal utils.Counter - requestsPassed utils.Counter - requestsThrottled utils.Counter - - // The following features of RNG are crucial for the backpressure subsystem: - // 1. uniform distribution of the output; - // 2. thread-safety; - - // In order to save time, we use a third party RNG library which is thread-safe, - // however, there are some concerns on the distribution uniformity. - // There are indicators that it's truly uniform because this RNG (providing only integer numbers) - // was used in the uniformly distributed float RNG implementation: https://prng.di.unimi.it/ - - // Here are the posts stating that (at least empirically) the distribution uniformity is - // observed for all RNGs belonging to this family but one: - // https://stackoverflow.com/questions/71050149/does-xoshiro-xoroshiro-prngs-provide-uniform-distribution - // https://crypto.stackexchange.com/questions/98597 - rng *fastrand64.ThreadsafePoolRNG - - // число в диапазоне [0; 100], показывающее, какой процент запросов должен быть отбит - threshold uint32 + // requestsTotal is the total number of requests. + requestsTotal utils.Counter[uint64] + // requestsPassed is the number of requests that were passed. + requestsPassed utils.Counter[uint64] + // requestsThrottled is the number of requests that were throttled. + requestsThrottled utils.Counter[uint64] + // threshold is the percentage of requests that should be throttled. + // It must be in the range [0; 100]. + threshold atomic.Uint32 } -func (t *throttler) setThreshold(value uint32) error { - if value > FullThrottling { - return errors.New("implementation error: threshold value must belong to [0;100]") - } - - atomic.StoreUint32(&t.threshold, value) - - return nil -} +// newThrottler creates a new throttler. +func newThrottler() *throttler { + requestsTotal := utils.NewUint64Counter(nil) -func (t *throttler) getStats() *stats.ThrottlingStats { - return &stats.ThrottlingStats{ - Total: uint64(t.requestsTotal.Count()), - Passed: uint64(t.requestsPassed.Count()), - Throttled: uint64(t.requestsThrottled.Count()), + return &throttler{ + requestsTotal: requestsTotal, + requestsPassed: utils.NewUint64Counter(requestsTotal), + requestsThrottled: utils.NewUint64Counter(requestsTotal), } } +// AllowRequest checks if the request should be allowed. func (t *throttler) AllowRequest() bool { - threshold := atomic.LoadUint32(&t.threshold) + threshold := t.threshold.Load() - // if throttling is disabled, allow any request + // If throttling is disabled, allow any request. if threshold == 0 { t.requestsPassed.Inc(1) return true } - // flip a coin in the range [0; 100]; if the actual value is less than the threshold value, - // throttle the request, otherwise allow it. - value := t.rng.Uint32n(FullThrottling) + // Flip a coin in the range [0; 100]. + // If the actual value is less than the threshold value, throttle the request. + // Otherwise, allow the request. + // math/rand/v2 top-level functions are safe for concurrent use and provide + // non-cryptographic uniformly distributed values, which is enough here. + //nolint:gosec // Non-cryptographic RNG is intentional for probabilistic throttling decisions. + value := rand.Uint32N(FullThrottling) allowed := value >= threshold @@ -84,13 +71,22 @@ func (t *throttler) AllowRequest() bool { return allowed } -func newThrottler() *throttler { - requestsTotal := utils.NewCounter(nil) +// setThreshold sets the threshold for the throttler. +func (t *throttler) setThreshold(value uint32) error { + if value > FullThrottling { + return errors.New("implementation error: threshold value must belong to [0;100]") + } - return &throttler{ - rng: fastrand64.NewSyncPoolXoshiro256ssRNG(), - requestsTotal: requestsTotal, - requestsPassed: utils.NewCounter(requestsTotal), - requestsThrottled: utils.NewCounter(requestsTotal), + t.threshold.Store(value) + + return nil +} + +// getStats returns the statistics of the throttler. +func (t *throttler) getStats() *stats.ThrottlingStats { + return &stats.ThrottlingStats{ + Total: t.requestsTotal.Count(), + Passed: t.requestsPassed.Count(), + Throttled: t.requestsThrottled.Count(), } } diff --git a/backpressure/throttler_test.go b/backpressure/throttler_test.go index eba7153..c3e06af 100644 --- a/backpressure/throttler_test.go +++ b/backpressure/throttler_test.go @@ -22,7 +22,7 @@ func TestThrottler(t *testing.T) { ) // Check different throttling levels with 10% step. - for i := 0; i < 10; i++ { + for i := range 10 { throttlingLevel := uint32(i) * 10 t.Run(fmt.Sprintf("throttling level = %v", throttlingLevel), func(t *testing.T) { @@ -34,10 +34,10 @@ func TestThrottler(t *testing.T) { wg := &sync.WaitGroup{} wg.Add(requests) - failed := utils.NewCounter(nil) - succeeded := utils.NewCounter(nil) + failed := utils.NewUint64Counter(nil) + succeeded := utils.NewUint64Counter(nil) - for i := 0; i < requests; i++ { + for range requests { go func() { defer wg.Done() @@ -53,7 +53,7 @@ func TestThrottler(t *testing.T) { wg.Wait() total := failed.Count() + succeeded.Count() - require.Equal(t, int64(requests), total) + require.Equal(t, uint64(requests), total) failedShareExpected := float64(throttlingLevel) / float64(100) failedShareActual := float64(failed.Count()) / float64(total) @@ -69,14 +69,14 @@ func TestThrottler(t *testing.T) { failedShareExpected, failedShareActual, 0.055, - fmt.Sprintf("expected = %v, actual = %v", failedShareExpected, failedShareActual), + "expected = %v, actual = %v", failedShareExpected, failedShareActual, ) require.InDelta( t, succeededShareExpected, succeededShareActual, 0.055, - fmt.Sprintf("expected = %v, actual = %v", succeededShareExpected, succeededShareActual), + "expected = %v, actual = %v", succeededShareExpected, succeededShareActual, ) // Check internal counters. @@ -103,8 +103,6 @@ func BenchmarkThrottler(b *testing.B) { const requests = 1000 for _, throttlingLevel := range []uint32{0, 50, 100} { - throttlingLevel := throttlingLevel - b.Run(fmt.Sprintf("throttling level = %v", throttlingLevel), func(b *testing.B) { th := newThrottler() @@ -115,21 +113,18 @@ func BenchmarkThrottler(b *testing.B) { var allowed bool - for k := 0; k < b.N; k++ { + for range b.N { wg := &sync.WaitGroup{} b.StartTimer() - for i := 0; i < requests; i++ { - wg.Add(1) - - go func() { - defer wg.Done() - + for range requests { + wg.Go(func() { // assign result to fictive variable to disallow compiler to optimize out function call allowed = th.AllowRequest() - }() + }) } + wg.Wait() b.StopTimer() diff --git a/config.go b/config.go index 6d13b16..d2a03b8 100644 --- a/config.go +++ b/config.go @@ -7,7 +7,7 @@ package memlimiter import ( - "github.com/pkg/errors" + "errors" "github.com/newcloudtechnologies/memlimiter/controller/nextgc" ) diff --git a/controller/nextgc/component_p.go b/controller/nextgc/component_p.go index a589786..b01be5f 100644 --- a/controller/nextgc/component_p.go +++ b/controller/nextgc/component_p.go @@ -7,26 +7,56 @@ package nextgc import ( + "fmt" "math" "github.com/go-logr/logr" - metrics "github.com/rcrowley/go-metrics" - - "github.com/pkg/errors" + "github.com/newcloudtechnologies/memlimiter/utils" ) // The proportional component of the controller. type componentP struct { - lastValues metrics.Sample - cfg *ComponentProportionalConfig - logger logr.Logger + // valueSmoother is a smoother for the raw proportional signal. + valueSmoother *utils.EMASmoother + // cfg is the configuration for the proportional component. + cfg *ComponentProportionalConfig + // logger is the logger for the proportional component. + logger logr.Logger } +// newComponentP creates a new proportional component. +func newComponentP(logger logr.Logger, cfg *ComponentProportionalConfig) *componentP { + out := &componentP{ + logger: logger, + cfg: cfg, + } + + if cfg.WindowSize != 0 { + // We smooth the raw proportional signal because memory usage is noisy: + // short spikes should not immediately trigger aggressive control actions. + // + // EMA formula: + // S_t = alpha*X_t + (1-alpha)*S_{t-1} + // + // To approximate a simple moving average window of size N, we use: + // alpha = 2 / (N + 1) + // + // Larger window -> smaller alpha -> smoother but slower reaction. + //nolint:gomnd + alpha := 2 / (float64(cfg.WindowSize + 1)) + + out.valueSmoother = utils.NewEMASmoother(alpha) + } + + return out +} + +// value returns the proportional component's output. func (c *componentP) value(utilization float64) (float64, error) { - if c.lastValues != nil { + if c.valueSmoother != nil { valueEMA, err := c.valueEMA(utilization) if err != nil { - return math.NaN(), errors.Wrap(err, "value EMA") + return math.NaN(), fmt.Errorf("value EMA: %w", err) } return valueEMA, nil @@ -34,15 +64,16 @@ func (c *componentP) value(utilization float64) (float64, error) { valueRaw, err := c.valueRaw(utilization) if err != nil { - return math.NaN(), errors.Wrap(err, "value raw") + return math.NaN(), fmt.Errorf("value raw: %w", err) } return valueRaw, nil } +// valueRaw returns the raw proportional component's output. func (c *componentP) valueRaw(utilization float64) (float64, error) { if utilization < 0 { - return math.NaN(), errors.Errorf("value is undefined if memory usage = %v", utilization) + return math.NaN(), fmt.Errorf("value is undefined if memory usage = %v", utilization) } if utilization >= 1 { @@ -63,36 +94,12 @@ func (c *componentP) valueRaw(utilization float64) (float64, error) { return c.cfg.Coefficient * (1 / (1 - utilization)), nil } +// valueEMA returns the exponential moving average of the raw proportional component's output. func (c *componentP) valueEMA(utilization float64) (float64, error) { valueRaw, err := c.valueRaw(utilization) if err != nil { - return 0, errors.Wrap(err, "value raw") + return 0, fmt.Errorf("value raw: %w", err) } - // TODO: need to find statistical library working with floats to make this conversion unnecessary - const reductionFactor = 100 - - c.lastValues.Update(int64(valueRaw * reductionFactor)) - - return c.lastValues.Mean() / reductionFactor, nil -} - -func newComponentP(logger logr.Logger, cfg *ComponentProportionalConfig) *componentP { - out := &componentP{ - logger: logger, - cfg: cfg, - } - - if cfg.WindowSize != 0 { - // alpha is a smoothing coefficient describing the degree of weighting decrease; - // the lesser the alpha is, the higher the impact of the elder historical values on the resulting value. - // alpha is choosed empirically, but can depend on a window size, like here: - // https://en.wikipedia.org/wiki/Moving_average#Relationship_between_SMA_and_EMA - //nolint:gomnd - alpha := 2 / (float64(cfg.WindowSize + 1)) - - out.lastValues = metrics.NewExpDecaySample(int(cfg.WindowSize), alpha) - } - - return out + return c.valueSmoother.Update(valueRaw), nil } diff --git a/controller/nextgc/config.go b/controller/nextgc/config.go index 88944a7..a07d510 100644 --- a/controller/nextgc/config.go +++ b/controller/nextgc/config.go @@ -7,9 +7,10 @@ package nextgc import ( + "errors" + "github.com/newcloudtechnologies/memlimiter/utils/config/bytes" "github.com/newcloudtechnologies/memlimiter/utils/config/duration" - "github.com/pkg/errors" ) // ControllerConfig - controller configuration. @@ -43,7 +44,7 @@ func (c *ControllerConfig) Prepare() error { } if c.DangerZoneThrottling == 0 || c.DangerZoneThrottling > 100 { - return errors.Errorf("invalid DangerZoneThrottling value (must belong to [0; 100])") + return errors.New("invalid DangerZoneThrottling value (must belong to [0; 100])") } if c.Period.Duration == 0 { diff --git a/controller/nextgc/controller.go b/controller/nextgc/controller.go index 8162a17..aa5aff7 100644 --- a/controller/nextgc/controller.go +++ b/controller/nextgc/controller.go @@ -7,13 +7,13 @@ package nextgc import ( + "fmt" "math" "time" "github.com/go-logr/logr" "github.com/newcloudtechnologies/memlimiter/stats" "github.com/newcloudtechnologies/memlimiter/utils/breaker" - "github.com/pkg/errors" "github.com/newcloudtechnologies/memlimiter/backpressure" "github.com/newcloudtechnologies/memlimiter/controller" @@ -24,8 +24,6 @@ import ( // described in control theory. But currently it has only proportional (P) component, and the proportionality // is non-linear (see component_p.go). It looks like integral (I) component will never be implemented. // But the differential controller (D) still may be implemented in future if we face self-oscillation. -// -//nolint:govet type controllerImpl struct { input stats.ServiceStatsSubscription // input: service tracker subscription. output backpressure.Operator // output: write control parameters here @@ -50,31 +48,74 @@ type controllerImpl struct { breaker *breaker.Breaker } +// getStatsRequest is a request to get the controller stats. type getStatsRequest struct { result chan *stats.ControllerStats } -func (r *getStatsRequest) respondWith(resp *stats.ControllerStats) { - r.result <- resp +// NewControllerFromConfig builds new controller. +func NewControllerFromConfig( + logger logr.Logger, + cfg *ControllerConfig, + serviceStatsSubscription stats.ServiceStatsSubscription, + backpressureOperator backpressure.Operator, +) (controller.Controller, error) { + c := &controllerImpl{ + input: serviceStatsSubscription, + output: backpressureOperator, + componentP: newComponentP(logger, cfg.ComponentProportional), + pValue: 0, + sumValue: 0, + controlParameters: &stats.ControlParameters{ + GOGC: backpressure.DefaultGOGC, + ThrottlingPercentage: backpressure.NoThrottling, + }, + getStatsChan: make(chan *getStatsRequest), + cfg: cfg, + logger: logger, + breaker: breaker.NewBreakerWithInitValue(1), + } + + // initialize backpressure operator with default control signal + err := c.applyControlValue() + if err != nil { + return nil, fmt.Errorf("apply control value: %w", err) + } + + go c.loop() + + return c, nil } +// GetStats returns the current controller stats. func (c *controllerImpl) GetStats() (*stats.ControllerStats, error) { req := &getStatsRequest{result: make(chan *stats.ControllerStats, 1)} select { case c.getStatsChan <- req: case <-c.breaker.Done(): - return nil, errors.Wrap(c.breaker.Err(), "breaker err") + return nil, fmt.Errorf("breaker err: %w", c.breaker.Err()) } select { case resp := <-req.result: return resp, nil case <-c.breaker.Done(): - return nil, errors.Wrap(c.breaker.Err(), "breaker err") + return nil, fmt.Errorf("breaker err: %w", c.breaker.Err()) } } +// Quit gracefully stops the controller. +func (c *controllerImpl) Quit() { + c.breaker.ShutdownAndWait() +} + +// respondWith responds with the controller stats. +func (r *getStatsRequest) respondWith(resp *stats.ControllerStats) { + r.result <- resp +} + +// loop is the main loop of the controller. func (c *controllerImpl) loop() { defer c.breaker.Dec() @@ -85,12 +126,14 @@ func (c *controllerImpl) loop() { select { case serviceStats := <-c.input.Updates(): // Update controller state every time we receive the actual tracker about the process. - if err := c.updateState(serviceStats); err != nil { + err := c.updateState(serviceStats) + if err != nil { c.logger.Error(err, "update state") } case <-ticker.C: // Generate control parameters based on the most recent state and send it to the backpressure operator. - if err := c.applyControlValue(); err != nil { + err := c.applyControlValue() + if err != nil { c.logger.Error(err, "apply control value") } case req := <-c.getStatsChan: @@ -101,14 +144,16 @@ func (c *controllerImpl) loop() { } } +// updateState updates the controller state. func (c *controllerImpl) updateState(serviceStats stats.ServiceStats) error { // Extract the latest report on special memory consumers if there are any. c.consumptionReport = serviceStats.ConsumptionReport() c.updateUtilization(serviceStats) - if err := c.updateControlValues(); err != nil { - return errors.Wrap(err, "update control values") + err := c.updateControlValues() + if err != nil { + return fmt.Errorf("update control values: %w", err) } c.updateControlParameters() @@ -116,6 +161,7 @@ func (c *controllerImpl) updateState(serviceStats stats.ServiceStats) error { return nil } +// updateUtilization updates the controller utilization. func (c *controllerImpl) updateUtilization(serviceStats stats.ServiceStats) { // The process memory (roughly) consists of two main parts: // 1. Allocations managed by Go runtime. @@ -144,12 +190,13 @@ func (c *controllerImpl) updateUtilization(serviceStats stats.ServiceStats) { c.rss = serviceStats.RSS() } +// updateControlValues updates the controller control values. func (c *controllerImpl) updateControlValues() error { var err error c.pValue, err = c.componentP.value(c.utilization) if err != nil { - return errors.Wrap(err, "component proportional value") + return fmt.Errorf("component proportional value: %w", err) } // TODO: if new components appear, summarize their outputs here: @@ -169,6 +216,7 @@ func (c *controllerImpl) updateControlValues() error { return nil } +// updateControlParameters updates the controller control parameters. func (c *controllerImpl) updateControlParameters() { c.controlParameters = &stats.ControlParameters{} c.updateControlParameterGOGC() @@ -179,6 +227,7 @@ func (c *controllerImpl) updateControlParameters() { const percents = 100 +// updateControlParameterGOGC updates the controller control parameter GOGC. func (c *controllerImpl) updateControlParameterGOGC() { // Control parameters are set to defaults in the "green zone". if uint32(c.utilization*percents) < c.cfg.DangerZoneGOGC { @@ -192,6 +241,7 @@ func (c *controllerImpl) updateControlParameterGOGC() { c.controlParameters.GOGC = int(backpressure.DefaultGOGC - roundedValue) } +// updateControlParameterThrottling updates the controller control parameter throttling. func (c *controllerImpl) updateControlParameterThrottling() { // Disable throttling in the "green zone". if uint32(c.utilization*percents) < c.cfg.DangerZoneThrottling { @@ -205,14 +255,17 @@ func (c *controllerImpl) updateControlParameterThrottling() { c.controlParameters.ThrottlingPercentage = roundedValue } +// applyControlValue applies the controller control value. func (c *controllerImpl) applyControlValue() error { - if err := c.output.SetControlParameters(c.controlParameters); err != nil { - return errors.Wrapf(err, "set control parameters: %v", c.controlParameters) + err := c.output.SetControlParameters(c.controlParameters) + if err != nil { + return fmt.Errorf("set control parameters: %v: %w", c.controlParameters, err) } return nil } +// aggregateStats aggregates the controller stats. func (c *controllerImpl) aggregateStats() *stats.ControllerStats { res := &stats.ControllerStats{ MemoryBudget: &stats.MemoryBudgetStats{ @@ -235,41 +288,3 @@ func (c *controllerImpl) aggregateStats() *stats.ControllerStats { return res } - -// Quit gracefully stops the controller. -func (c *controllerImpl) Quit() { - c.breaker.ShutdownAndWait() -} - -// NewControllerFromConfig builds new controller. -func NewControllerFromConfig( - logger logr.Logger, - cfg *ControllerConfig, - serviceStatsSubscription stats.ServiceStatsSubscription, - backpressureOperator backpressure.Operator, -) (controller.Controller, error) { - c := &controllerImpl{ - input: serviceStatsSubscription, - output: backpressureOperator, - componentP: newComponentP(logger, cfg.ComponentProportional), - pValue: 0, - sumValue: 0, - controlParameters: &stats.ControlParameters{ - GOGC: backpressure.DefaultGOGC, - ThrottlingPercentage: backpressure.NoThrottling, - }, - getStatsChan: make(chan *getStatsRequest), - cfg: cfg, - logger: logger, - breaker: breaker.NewBreakerWithInitValue(1), - } - - // initialize backpressure operator with default control signal - if err := c.applyControlValue(); err != nil { - return nil, errors.Wrap(err, "apply control value") - } - - go c.loop() - - return c, nil -} diff --git a/controller/nextgc/controller_test.go b/controller/nextgc/controller_test.go index 5994850..2457ded 100644 --- a/controller/nextgc/controller_test.go +++ b/controller/nextgc/controller_test.go @@ -110,7 +110,7 @@ func TestController(t *testing.T) { return val.GOGC == 78 && val.ThrottlingPercentage == 22 }), ).Return(nil).Once().Run( - func(args mock.Arguments) { + func(_ mock.Arguments) { // As soon as the control signal is delivered to the backpressure.Operator, // replace the ServiceStats instance to make controller think that memory // consumption returned to normal. @@ -122,7 +122,7 @@ func TestController(t *testing.T) { return val.GOGC == backpressure.DefaultGOGC && val.ThrottlingPercentage == backpressure.NoThrottling }), ).Return(nil).Once().Run( - func(args mock.Arguments) { + func(_ mock.Arguments) { close(terminateChan) }, ) diff --git a/go.mod b/go.mod index 672d470..fc291b7 100644 --- a/go.mod +++ b/go.mod @@ -1,42 +1,46 @@ module github.com/newcloudtechnologies/memlimiter -go 1.17 +go 1.26 require ( - code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5 - github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 - github.com/go-logr/logr v1.2.3 + code.cloudfoundry.org/bytefmt v0.69.0 + github.com/aclements/go-moremath v0.0.0-20241023150245-c8bbc672ef66 + github.com/go-logr/logr v1.4.3 github.com/go-logr/stdr v1.2.2 - github.com/golang/protobuf v1.5.2 - github.com/pkg/errors v0.9.1 - github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 - github.com/shirou/gopsutil/v3 v3.22.5 - github.com/stretchr/testify v1.7.1 - github.com/urfave/cli/v2 v2.8.1 - github.com/villenny/fastrand64-go v0.0.0-20201008161821-3d8fa521c558 - golang.org/x/time v0.0.0-20220411224347-583f2d630306 - google.golang.org/grpc v1.38.0 - google.golang.org/protobuf v1.26.0 + github.com/shirou/gopsutil/v3 v3.24.5 + github.com/stretchr/testify v1.11.1 + github.com/urfave/cli/v2 v2.27.7 + golang.org/x/time v0.15.0 + google.golang.org/grpc v1.80.0 + google.golang.org/protobuf v1.36.11 ) require ( - github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-ole/go-ole v1.2.6 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/stretchr/objx v0.2.0 // indirect - github.com/tklauser/go-sysconf v0.3.10 // indirect - github.com/tklauser/numcpus v0.4.0 // indirect - github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect - github.com/yusufpapurcu/wmi v1.2.2 // indirect - golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect - golang.org/x/sys v0.0.0-20220317061510-51cd9980dadf // indirect - golang.org/x/text v0.3.7 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/stretchr/objx v0.5.2 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + golang.org/x/net v0.53.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/text v0.36.0 // indirect google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d // indirect + google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.1 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +tool ( + google.golang.org/grpc/cmd/protoc-gen-go-grpc + google.golang.org/protobuf/cmd/protoc-gen-go ) diff --git a/go.sum b/go.sum index 028956e..5abbb2f 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,18 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5 h1:tM5+dn2C9xZw1RzgI6WTQW1rGqdUimKB3RFbyu4h6Hc= -code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5/go.mod h1:v4VVB6oBMz/c9fRY6vZrwr5xKRWOH5NPDjQZlPk0Gbs= +code.cloudfoundry.org/bytefmt v0.69.0 h1:iLYdNRsdebJd2Tz7SohOt0VHOqIXqIfxpdVx1YMC3ik= +code.cloudfoundry.org/bytefmt v0.69.0/go.mod h1:t6RyZJtu5JDi58uyWBemnTepT30jMJgB69Ue8D7ou30= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 h1:xlwdaKcTNVW4PtpQb8aKA4Pjy0CdJHEqvFbAnvR5m2g= -github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794/go.mod h1:7e+I0LQFUI9AXWxOfsQROs9xPhoJtbsyWcjJqDd4KPY= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/aclements/go-moremath v0.0.0-20241023150245-c8bbc672ef66 h1:siNQlUMcFUDZWCOt0p+RHl7et5Nnwwyq/sFZmr4iG1I= +github.com/aclements/go-moremath v0.0.0-20241023150245-c8bbc672ef66/go.mod h1:FDw7qicTbJ1y1SZcNnOvym2BogPdC3lY9Z1iUM4MVhw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= +github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -18,17 +21,15 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -41,8 +42,9 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -50,10 +52,13 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg= +github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= @@ -64,117 +69,103 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= -github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= +github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= +github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= +github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= -github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/shirou/gopsutil/v3 v3.22.5 h1:atX36I/IXgFiB81687vSiBI5zrMsxcIBkP9cQMJQoJA= -github.com/shirou/gopsutil/v3 v3.22.5/go.mod h1:so9G9VzeHt/hsd0YwqprnjHnfARAUktauykSbr+y2gA= +github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= +github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= -github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= -github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o= -github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= -github.com/tsuna/endian v0.0.0-20151020052604-29b3a4178852 h1:/HMzghBx/U8ZTQ+CCKRAsjeNNV12OCG3PfJcthNMBU0= -github.com/tsuna/endian v0.0.0-20151020052604-29b3a4178852/go.mod h1:7SvkOZYNBtjd5XUi2fuPMvAZS8rlCMaU69hj/3joIsE= -github.com/urfave/cli/v2 v2.8.1 h1:CGuYNZF9IKZY/rfBe3lJpccSoIY1ytfvmgQT90cNOl4= -github.com/urfave/cli/v2 v2.8.1/go.mod h1:Z41J9TPoffeoqP0Iza0YbAhGvymRdZAd2uPmZ5JxRdY= -github.com/valyala/fastrand v1.0.0 h1:LUKT9aKer2dVQNUi3waewTbKV+7H17kvWFNKs2ObdkI= -github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= -github.com/villenny/fastrand64-go v0.0.0-20201008161821-3d8fa521c558 h1:oNwFCUPi4ns2fMuaBtzMdQImdt25neDPJPBTNprmdF8= -github.com/villenny/fastrand64-go v0.0.0-20201008161821-3d8fa521c558/go.mod h1:0KogUQQf0cFYfgnOpYJqw1RnSb4S1oJwUb1CEpGJLJ4= -github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= -github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= -github.com/yalue/native_endian v0.0.0-20180607135909-51013b03be4f h1:nsQCScpQ8RRf+wIooqfyyEUINV2cAPuo2uVtHSBbA4M= -github.com/yalue/native_endian v0.0.0-20180607135909-51013b03be4f/go.mod h1:1cm5YQZdnDQBZVtFG2Ip8sFVN0eYZ8OFkCT2kIVl9mw= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU= +github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4= +github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= +github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= -github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= +go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= +go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= +go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= +go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= +go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= +go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220317061510-51cd9980dadf h1:Fm4IcnUL803i92qDlmB0obyHmosDrxZWxJL3gIeNqOw= -golang.org/x/sys v0.0.0-20220317061510-51cd9980dadf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/time v0.0.0-20220411224347-583f2d630306 h1:+gHMid33q6pen7kv9xvT+JRinntgeXO2AeZVd0AWD3w= -golang.org/x/time v0.0.0-20220411224347-583f2d630306/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -182,12 +173,15 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -199,8 +193,11 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.1 h1:/WILD1UcXj/ujCxgoL/DvRgt2CP3txG8+FwkUbb9110= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.6.1/go.mod h1:YNKnb2OAApgYn2oYY47Rn7alMr1zWjb2U8Q0aoGWiNc= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -211,23 +208,16 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/middleware/grpc.go b/middleware/grpc.go index 52338fb..3e4075f 100644 --- a/middleware/grpc.go +++ b/middleware/grpc.go @@ -33,10 +33,10 @@ type grpcImpl struct { func (g *grpcImpl) MakeUnaryServerInterceptor() grpc.UnaryServerInterceptor { return func( ctx context.Context, - req interface{}, - info *grpc.UnaryServerInfo, + req any, + _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler, - ) (interface{}, error) { + ) (any, error) { allowed := g.backpressureOperator.AllowRequest() if allowed { return handler(ctx, req) @@ -55,9 +55,9 @@ func (g *grpcImpl) MakeUnaryServerInterceptor() grpc.UnaryServerInterceptor { func (g *grpcImpl) MakeStreamServerInterceptor() grpc.StreamServerInterceptor { return func( - srv interface{}, + srv any, ss grpc.ServerStream, - info *grpc.StreamServerInfo, + _ *grpc.StreamServerInfo, handler grpc.StreamHandler, ) error { allowed := g.backpressureOperator.AllowRequest() diff --git a/scripts/install_golangci_lint.sh b/scripts/install_golangci_lint.sh new file mode 100755 index 0000000..72d8338 --- /dev/null +++ b/scripts/install_golangci_lint.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env sh +# Purpose: install/update golangci-lint into the specified directory +# if the current version is missing or differs from the desired one. +# Script is provided via stdin: +# sh -s -- < scripts/install_golangci_lint.sh + +# Stop on errors and undefined variables. +# /bin/sh has no pipefail, so use only -e and -u. +set -eu + +# Script arguments: +# $1 - destination directory for the golangci-lint binary. +# $2 - desired golangci-lint version (for example, v1.64.8 or v2.7.2). +# $3 - installation method: curl (default) or build. +# In some CI runners installation via curl may be disallowed, so use build. +# Validate that enough arguments were provided. +if [ "$#" -lt 2 ]; then + echo "Usage: sh -s -- [curl|build]" >&2 + exit 2 +fi + +# Save arguments into descriptive variable names. +LOCAL_BIN="$1" +DESIRED="$2" +INSTALL_METHOD="${3:-curl}" +BINARY="$LOCAL_BIN/golangci-lint" + +case "$INSTALL_METHOD" in + curl|download) + INSTALL_METHOD="curl" + ;; + build|go) + INSTALL_METHOD="build" + ;; + *) + echo "Unknown install method '$INSTALL_METHOD'. Use 'curl' or 'build'." >&2 + exit 2 + ;; +esac + +# Validate required tools for the chosen install method. +if [ "$INSTALL_METHOD" = "curl" ]; then + if ! command -v curl >/dev/null 2>&1; then + echo "curl is required to install golangci-lint via curl" >&2 + exit 1 + fi +else + if ! command -v go >/dev/null 2>&1; then + echo "go toolchain is required to build golangci-lint from source" >&2 + exit 1 + fi +fi + +# Create install directory if needed. +mkdir -p "$LOCAL_BIN" + +# extract_ver extracts a version like vX.Y.Z from arbitrary text. +extract_ver() { + # 1) "has version v?X.Y.Z". + ver=$(sed -n 's/.*has version[[:space:]]\{1,\}v\{0,1\}\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/v\1/p' | head -n1) + if [ -n "$ver" ]; then printf '%s\n' "$ver"; return; fi + + # 2) "... version v?X.Y.Z ..." (with spaces around "version"). + ver=$(sed -n 's/.*[[:space:]]version[[:space:]]\{1,\}v\{0,1\}\([0-9]\+\.[0-9]\+\.[0-9]\+\)[^0-9.].*/v\1/p' | head -n1) + if [ -n "$ver" ]; then printf '%s\n' "$ver"; return; fi + + # 3) Fallback: first standalone X.Y.Z token bounded by non-digit/non-dot. + # This avoids matching things like "go1.25.1". + ver=$(sed -n 's/.*[^0-9.]\([0-9]\+\.[0-9]\+\.[0-9]\+\)[^0-9.].*/v\1/p' | head -n1) + if [ -n "$ver" ]; then printf '%s\n' "$ver"; return; fi + + # 4) If version is at end of line (no trailing delimiter). + ver=$(sed -n 's/.*[^0-9.]\([0-9]\+\.[0-9]\+\.[0-9]\+\)$$/v\1/p' | head -n1) + [ -n "$ver" ] && printf '%s\n' "$ver" +} + +# Detect currently installed version if the binary already exists. +installed="" + +# Check whether executable binary exists. +if [ -x "$BINARY" ]; then + # Run binary to get version output. + out="$("$BINARY" --version 2>&1 || "$BINARY" version 2>&1 || true)" + + # Extract version from output. + installed=$(printf '%s\n' "$out" | extract_ver || true) +fi + +# Compare versions and install if needed. +if [ "$installed" != "$DESIRED" ]; then + # Print action with previous version (or "none"). + echo "Installing golangci-lint $DESIRED into $LOCAL_BIN via $INSTALL_METHOD (was: ${installed:-none})" + + if [ "$INSTALL_METHOD" = "curl" ]; then + # Download and execute official install script: + # -s --: pass script arguments. + # -b "$LOCAL_BIN": install directory. + # "$DESIRED": version. + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \ + | sh -s -- -b "$LOCAL_BIN" "$DESIRED" + else + # Use local Go toolchain to install the requested version. + # For v2 major, module path must include /v2. + case "$DESIRED" in + v2.*|2.*) + MODULE="github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$DESIRED" + ;; + *) + MODULE="github.com/golangci/golangci-lint/cmd/golangci-lint@$DESIRED" + ;; + esac + + env GOBIN="$LOCAL_BIN" GO111MODULE=on GOWORK=off \ + go install "$MODULE" + fi +else + # Report that the current version is already up-to-date. + echo "golangci-lint $DESIRED already installed in $LOCAL_BIN" +fi diff --git a/scripts/make_help.awk b/scripts/make_help.awk new file mode 100644 index 0000000..9f83576 --- /dev/null +++ b/scripts/make_help.awk @@ -0,0 +1,81 @@ +#!/usr/bin/awk -f +# Purpose: pretty-print Makefile targets with descriptions from "##" comments. +# Usage: awk -f scripts/make_help.awk $(MAKEFILE_LIST) + +# Global variables: +# - target: current target name (the block that owns found ## lines). +# - desc_count: number of description lines attached to target. +BEGIN { + target = "" + desc_count = 0 +} + +# print_entry prints current target with collected descriptions. +# Format: cyan target name, then each comment on a new line. +function print_entry() { + if (target == "") + return + + printf "\033[36m%s\033[0m\n", target + + if (desc_count == 0) { + printf " (description not provided)\n" + } else { + for (i = 1; i <= desc_count; i++) { + printf " %s\n", desc[i] + } + } + + # Empty line for visual separation between blocks. + printf "\n" +} + +# reset_desc clears collected descriptions +# (AWK has no built-in clear(), so delete items one by one). +function reset_desc() { + for (i = 1; i <= desc_count; i++) { + delete desc[i] + } + + desc_count = 0 +} + +# Match target definitions: +# - start of line; +# - valid identifier (letters/digits/._-/); +# - ':' is not part of assignment (exclude foo:=, foo::=, etc.). +/^[[:alnum:]_][[:alnum:]_.\/-]*:([^=]|$)/ { + match($0, /^[[:alnum:]_][[:alnum:]_.\/-]*/) + name = substr($0, RSTART, RLENGTH) + + if (target == name) + next + + if (target != "") { + print_entry() + reset_desc() + } else { + reset_desc() + } + + # Important: save name after printing previous entry, or context is lost. + target = name + next +} + +# Description line (starts with ## after optional spaces). +/^[[:space:]]*##[[:space:]]+/ { + if (target == "") + next + + line = $0 + sub(/^[[:space:]]*##[[:space:]]+/, "", line) + desc[++desc_count] = line + next +} + +# At EOF print final block (if at least one target was found). +END { + if (target != "") + print_entry() +} diff --git a/scripts/merge_coverage.sh b/scripts/merge_coverage.sh new file mode 100755 index 0000000..31564ae --- /dev/null +++ b/scripts/merge_coverage.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Merge unit and integration Go coverage profiles into one report. +# Usage: +# scripts/merge_coverage.sh + +set -euo pipefail + +if [[ "$#" -ne 4 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +# Input/output files. +unit_coverage_file="$1" +integration_coverage_file="$2" +overall_coverage_file="$3" +coverage_summary_file="$4" + +# Copy unit coverage and append integration coverage body (skip header line). +cp "${unit_coverage_file}" "${overall_coverage_file}" +tail --lines=+2 "${integration_coverage_file}" >> "${overall_coverage_file}" + +# Exclude binaries/entrypoints that are intentionally out of test coverage scope. +sed -i '/test\/allocator\/app/d' "${overall_coverage_file}" +sed -i '/test\/allocator\/main.go/d' "${overall_coverage_file}" + +# Produce a human-readable coverage summary. +go tool cover -func="${overall_coverage_file}" -o="${coverage_summary_file}" diff --git a/service_impl.go b/service_impl.go index 07b54c3..ff3f208 100644 --- a/service_impl.go +++ b/service_impl.go @@ -7,6 +7,9 @@ package memlimiter import ( + "errors" + "fmt" + "github.com/go-logr/logr" "github.com/newcloudtechnologies/memlimiter/backpressure" "github.com/newcloudtechnologies/memlimiter/controller" @@ -14,7 +17,6 @@ import ( "github.com/newcloudtechnologies/memlimiter/middleware" "github.com/newcloudtechnologies/memlimiter/stats" "github.com/newcloudtechnologies/memlimiter/utils/config/prepare" - "github.com/pkg/errors" ) var _ Service = (*serviceImpl)(nil) @@ -32,12 +34,12 @@ func (s *serviceImpl) Middleware() middleware.Middleware { return s.middleware } func (s *serviceImpl) GetStats() (*stats.MemLimiterStats, error) { controllerStats, err := s.controller.GetStats() if err != nil { - return nil, errors.Wrap(err, "controller tracker") + return nil, fmt.Errorf("controller tracker: %w", err) } backpressureStats, err := s.backpressureOperator.GetStats() if err != nil { - return nil, errors.Wrap(err, "backpressure tracker") + return nil, fmt.Errorf("backpressure tracker: %w", err) } return &stats.MemLimiterStats{ @@ -60,7 +62,7 @@ func newServiceImpl( backpressureOperator backpressure.Operator, ) (Service, error) { if err := prepare.Prepare(cfg); err != nil { - return nil, errors.Wrap(err, "prepare config") + return nil, fmt.Errorf("prepare config: %w", err) } if statsSubscription == nil { @@ -75,9 +77,8 @@ func newServiceImpl( statsSubscription, backpressureOperator, ) - if err != nil { - return nil, errors.Wrap(err, "new controller from config") + return nil, fmt.Errorf("new controller from config: %w", err) } return &serviceImpl{ diff --git a/service_stub.go b/service_stub.go index aa30c08..6d73269 100644 --- a/service_stub.go +++ b/service_stub.go @@ -23,24 +23,37 @@ type serviceStub struct { breaker *breaker.Breaker } -func (s *serviceStub) loop() { - defer s.breaker.Dec() - - for { - select { - case record := <-s.statsSubscription.Updates(): - s.latestStats.Store(record) - case <-s.breaker.Done(): - return +// newServiceStub constructs a new service stub. +func newServiceStub(statsSubscription stats.ServiceStatsSubscription) Service { + if statsSubscription == nil { + return &serviceStub{ + breaker: breaker.NewBreaker(), } } + + out := &serviceStub{ + statsSubscription: statsSubscription, + breaker: breaker.NewBreakerWithInitValue(1), + } + + go out.loop() + + return out } +// Middleware returns the middleware. func (s *serviceStub) Middleware() middleware.Middleware { // TODO: return stub return nil } +// Quit terminates the service stub gracefully. +func (s *serviceStub) Quit() { + s.breaker.Shutdown() + s.statsSubscription.Quit() +} + +// GetStats returns the current stats. func (s *serviceStub) GetStats() (*stats.MemLimiterStats, error) { if val := s.latestStats.Load(); val != nil { //nolint:forcetypeassert @@ -57,27 +70,20 @@ func (s *serviceStub) GetStats() (*stats.MemLimiterStats, error) { return out, nil } + //nolint:nilnil // This is a stub. return nil, nil } -func (s *serviceStub) Quit() { - s.breaker.Shutdown() - s.statsSubscription.Quit() -} +// loop is the main loop of the service stub. +func (s *serviceStub) loop() { + defer s.breaker.Dec() -func newServiceStub(statsSubscription stats.ServiceStatsSubscription) Service { - if statsSubscription == nil { - return &serviceStub{ - breaker: breaker.NewBreaker(), + for { + select { + case record := <-s.statsSubscription.Updates(): + s.latestStats.Store(record) + case <-s.breaker.Done(): + return } } - - out := &serviceStub{ - statsSubscription: statsSubscription, - breaker: breaker.NewBreakerWithInitValue(1), - } - - go out.loop() - - return out } diff --git a/stats/memlimiter.go b/stats/memlimiter.go index 0cff851..10a5bd3 100644 --- a/stats/memlimiter.go +++ b/stats/memlimiter.go @@ -91,8 +91,8 @@ func (cp *ControlParameters) String() string { } // ToKeysAndValues serializes struct for use in logr.Logger. -func (cp *ControlParameters) ToKeysAndValues() []interface{} { - return []interface{}{ +func (cp *ControlParameters) ToKeysAndValues() []any { + return []any{ "gogc", cp.GOGC, "throttling_percentage", cp.ThrottlingPercentage, } diff --git a/stats/mock.go b/stats/mock.go index c688f18..9927980 100644 --- a/stats/mock.go +++ b/stats/mock.go @@ -18,16 +18,19 @@ type ServiceStatsMock struct { } func (m *ServiceStatsMock) RSS() uint64 { + //nolint:forcetypeassert // Mocked method. return m.Called().Get(0).(uint64) } func (m *ServiceStatsMock) NextGC() uint64 { + //nolint:forcetypeassert // Mocked method. return m.Called().Get(0).(uint64) } func (m *ServiceStatsMock) ConsumptionReport() *ConsumptionReport { args := m.Called() + //nolint:forcetypeassert // Mocked method. return args.Get(0).(*ConsumptionReport) } @@ -35,9 +38,10 @@ var _ ServiceStatsSubscription = (*ServiceStatsSubscriptionMock)(nil) // ServiceStatsSubscriptionMock mocks ServiceStatsSubscription. type ServiceStatsSubscriptionMock struct { + mock.Mock ServiceStatsSubscription + Chan chan ServiceStats - mock.Mock } func (m *ServiceStatsSubscriptionMock) Updates() <-chan ServiceStats { diff --git a/stats/subscription.go b/stats/subscription.go index 2d662d4..4920d1a 100644 --- a/stats/subscription.go +++ b/stats/subscription.go @@ -7,13 +7,14 @@ package stats import ( + "fmt" + "math" "os" "runtime" "time" "github.com/go-logr/logr" "github.com/newcloudtechnologies/memlimiter/utils/breaker" - "github.com/pkg/errors" "github.com/shirou/gopsutil/v3/process" ) @@ -33,7 +34,6 @@ type subscriptionDefault struct { breaker *breaker.Breaker logger logr.Logger period time.Duration - pid int32 } func (s *subscriptionDefault) Updates() <-chan ServiceStats { return s.outChan } @@ -46,14 +46,19 @@ func (s *subscriptionDefault) makeServiceStats() (ServiceStats, error) { ms := &runtime.MemStats{} runtime.ReadMemStats(ms) - pr, err := process.NewProcess(s.pid) + pid, err := getCurrentPID() if err != nil { - return nil, errors.Wrap(err, "new pr") + return nil, fmt.Errorf("get current pid: %w", err) + } + + pr, err := process.NewProcess(pid) + if err != nil { + return nil, fmt.Errorf("new pr: %w", err) } processMemoryInfo, err := pr.MemoryInfoEx() if err != nil { - return nil, errors.Wrap(err, "process memory info ex") + return nil, fmt.Errorf("process memory info ex: %w", err) } return serviceStatsDefault{ @@ -62,13 +67,21 @@ func (s *subscriptionDefault) makeServiceStats() (ServiceStats, error) { }, nil } +func getCurrentPID() (int32, error) { + pid := os.Getpid() + if pid < 0 || pid > math.MaxInt32 { + return 0, fmt.Errorf("pid is out of int32 range: %d", pid) + } + + return int32(pid), nil +} + // NewSubscriptionDefault - default implementation of service tracker subscription. func NewSubscriptionDefault(logger logr.Logger, period time.Duration) ServiceStatsSubscription { ss := &subscriptionDefault{ outChan: make(chan ServiceStats), period: period, breaker: breaker.NewBreakerWithInitValue(1), - pid: int32(os.Getpid()), logger: logger, } diff --git a/test/allocator/Dockerfile b/test/allocator/Dockerfile index 75c73a9..e58a7a3 100644 --- a/test/allocator/Dockerfile +++ b/test/allocator/Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:36 +FROM fedora:43 ADD allocator /usr/local/bin diff --git a/test/allocator/app/app.go b/test/allocator/app/app.go index b11c29a..2f89041 100644 --- a/test/allocator/app/app.go +++ b/test/allocator/app/app.go @@ -7,12 +7,12 @@ package app import ( + "fmt" "os" "os/signal" "syscall" "github.com/go-logr/logr" - "github.com/pkg/errors" "github.com/urfave/cli/v2" ) @@ -22,6 +22,14 @@ type App struct { logger logr.Logger } +// NewApp prepares new application. +func NewApp(logger logr.Logger, factory Factory) *App { + return &App{ + logger: logger, + factory: factory, + } +} + // Run launches the application. func (a *App) Run() { app := &cli.App{ @@ -42,7 +50,7 @@ func (a *App) Run() { Action: func(context *cli.Context) error { r, err := a.factory.MakeServer(context) if err != nil { - return errors.Wrap(err, "make server") + return fmt.Errorf("make server: %w", err) } return runAndWaitSignal(r) @@ -62,7 +70,7 @@ func (a *App) Run() { Action: func(context *cli.Context) error { r, err := a.factory.MakePerfClient(context) if err != nil { - return errors.Wrap(err, "make perf client") + return fmt.Errorf("make perf client: %w", err) } return runAndWaitSignal(r) @@ -71,7 +79,8 @@ func (a *App) Run() { }, } - if err := app.Run(os.Args); err != nil { + err := app.Run(os.Args) + if err != nil { a.logger.Error(err, "application run") os.Exit(1) } @@ -89,16 +98,12 @@ func runAndWaitSignal(r Runnable) error { select { case err := <-errChan: - return errors.Wrap(err, "run error") + if err != nil { + return fmt.Errorf("run error: %w", err) + } + + return nil case <-signalChan: return nil } } - -// NewApp prepares new application. -func NewApp(logger logr.Logger, factory Factory) *App { - return &App{ - logger: logger, - factory: factory, - } -} diff --git a/test/allocator/app/factory.go b/test/allocator/app/factory.go index 50485b7..4da9371 100644 --- a/test/allocator/app/factory.go +++ b/test/allocator/app/factory.go @@ -8,13 +8,13 @@ package app import ( "encoding/json" - "io/ioutil" + "fmt" + "os" "path/filepath" "github.com/go-logr/logr" "github.com/newcloudtechnologies/memlimiter/test/allocator/perf" "github.com/newcloudtechnologies/memlimiter/test/allocator/server" - "github.com/pkg/errors" "github.com/urfave/cli/v2" ) @@ -42,20 +42,20 @@ type factoryDefault struct { func (f *factoryDefault) MakeServer(c *cli.Context) (Runnable, error) { filename := c.String("config") - data, err := ioutil.ReadFile(filepath.Clean(filename)) + data, err := os.ReadFile(filepath.Clean(filename)) if err != nil { - return nil, errors.Wrap(err, "ioutil readfile") + return nil, fmt.Errorf("os readfile: %w", err) } cfg := &server.Config{} if err = json.Unmarshal(data, cfg); err != nil { - return nil, errors.Wrap(err, "unmarshal") + return nil, fmt.Errorf("unmarshal: %w", err) } srv, err := server.NewServer(f.logger, cfg) if err != nil { - return nil, errors.Wrap(err, "new allocator server") + return nil, fmt.Errorf("new allocator server: %w", err) } return srv, nil @@ -65,20 +65,20 @@ func (f *factoryDefault) MakeServer(c *cli.Context) (Runnable, error) { func (f *factoryDefault) MakePerfClient(c *cli.Context) (Runnable, error) { filename := c.String("config") - data, err := ioutil.ReadFile(filepath.Clean(filename)) + data, err := os.ReadFile(filepath.Clean(filename)) if err != nil { - return nil, errors.Wrap(err, "ioutil readfile") + return nil, fmt.Errorf("os readfile: %w", err) } cfg := &perf.Config{} if err = json.Unmarshal(data, cfg); err != nil { - return nil, errors.Wrap(err, "unmarshal") + return nil, fmt.Errorf("unmarshal: %w", err) } cl, err := perf.NewClient(f.logger, cfg) if err != nil { - return nil, errors.Wrap(err, "new allocator server") + return nil, fmt.Errorf("new allocator server: %w", err) } return cl, nil diff --git a/test/allocator/perf/client.go b/test/allocator/perf/client.go index 0fa0616..5e00acf 100644 --- a/test/allocator/perf/client.go +++ b/test/allocator/perf/client.go @@ -8,35 +8,61 @@ package perf import ( "context" + "fmt" "time" "github.com/go-logr/logr" + "github.com/newcloudtechnologies/memlimiter/utils" "github.com/newcloudtechnologies/memlimiter/utils/config/prepare" - "github.com/rcrowley/go-metrics" "golang.org/x/time/rate" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/protobuf/types/known/durationpb" "github.com/newcloudtechnologies/memlimiter/test/allocator/schema" "github.com/newcloudtechnologies/memlimiter/utils/breaker" - "github.com/pkg/errors" ) // Client - client for performance testing. type Client struct { startTime time.Time client schema.AllocatorClient - requestsInFlight metrics.Counter + requestsInFlight utils.Counter[int64] grpcConn *grpc.ClientConn breaker *breaker.Breaker cfg *Config logger logr.Logger } +// NewClient creates new client for performance tests. +func NewClient(logger logr.Logger, cfg *Config) (*Client, error) { + if err := prepare.Prepare(cfg); err != nil { + return nil, fmt.Errorf("configs prepare: %w", err) + } + + grpcConn, err := grpc.NewClient(cfg.Endpoint, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, fmt.Errorf("dial error: %w", err) + } + + client := schema.NewAllocatorClient(grpcConn) + + return &Client{ + grpcConn: grpcConn, + logger: logger, + client: client, + startTime: time.Now(), + cfg: cfg, + requestsInFlight: utils.NewInt64Counter(nil), + breaker: breaker.NewBreaker(), + }, nil +} + // Run starts load session. func (p *Client) Run() error { - if err := p.breaker.Inc(); err != nil { - return errors.Wrap(err, "breaker inc") + err := p.breaker.Inc() + if err != nil { + return fmt.Errorf("breaker inc: %w", err) } defer p.breaker.Dec() @@ -49,36 +75,49 @@ func (p *Client) Run() error { limiter := rate.NewLimiter(p.cfg.RPS, 1) - // single threaded for simplicity + // Single threaded for simplicity. for { - // wait till limiter allows to fire a request - if err := limiter.Wait(p.breaker); err != nil { - return errors.Wrap(err, "limiter wait") + // Wait till limiter allows to fire a request. + err := limiter.Wait(p.breaker) + if err != nil { + return fmt.Errorf("limiter wait: %w", err) } - // increment request copies - if err := p.breaker.Inc(); err != nil { - return errors.Wrap(err, "breaker inc") + // Increment request copies. + err = p.breaker.Inc() + if err != nil { + return fmt.Errorf("breaker inc: %w", err) } go p.makeRequest() select { case <-monitoringTicker.C: - // print progress periodically + // Print progress periodically. p.printProgress() case <-timer.C: - // terminate load + // Terminate load. return nil default: } } } +// Quit terminates perf client gracefully. +func (p *Client) Quit() { + p.breaker.ShutdownAndWait() + + err := p.grpcConn.Close() + if err != nil { + p.logger.Error(err, "gprc connection close") + } +} + +// makeRequest makes a request to the allocator server. func (p *Client) makeRequest() { defer p.breaker.Dec() - // update in-flight request counter + // Update in-flight request counter. p.requestsInFlight.Inc(1) defer p.requestsInFlight.Dec(1) @@ -99,6 +138,7 @@ func (p *Client) makeRequest() { } } +// printProgress prints the progress of the load session. func (p *Client) printProgress() { p.logger.Info( "progress", @@ -106,36 +146,3 @@ func (p *Client) printProgress() { "in_flight", p.requestsInFlight.Count(), ) } - -// Quit terminates perf client gracefully. -func (p *Client) Quit() { - p.breaker.ShutdownAndWait() - - if err := p.grpcConn.Close(); err != nil { - p.logger.Error(err, "gprc connection close") - } -} - -// NewClient creates new client for performance tests. -func NewClient(logger logr.Logger, cfg *Config) (*Client, error) { - if err := prepare.Prepare(cfg); err != nil { - return nil, errors.Wrap(err, "configs prepare") - } - - grpcConn, err := grpc.Dial(cfg.Endpoint, grpc.WithInsecure()) - if err != nil { - return nil, errors.Wrap(err, "dial error") - } - - client := schema.NewAllocatorClient(grpcConn) - - return &Client{ - grpcConn: grpcConn, - logger: logger, - client: client, - startTime: time.Now(), - cfg: cfg, - requestsInFlight: metrics.NewCounter(), - breaker: breaker.NewBreaker(), - }, nil -} diff --git a/test/allocator/perf/config.go b/test/allocator/perf/config.go index 98a05a7..2c22f70 100644 --- a/test/allocator/perf/config.go +++ b/test/allocator/perf/config.go @@ -7,11 +7,12 @@ package perf import ( + "errors" + "golang.org/x/time/rate" "github.com/newcloudtechnologies/memlimiter/utils/config/bytes" "github.com/newcloudtechnologies/memlimiter/utils/config/duration" - "github.com/pkg/errors" ) // Config - performance client configuration. diff --git a/test/allocator/schema/allocator.pb.go b/test/allocator/schema/allocator.pb.go index 4f60ab3..7c20926 100644 --- a/test/allocator/schema/allocator.pb.go +++ b/test/allocator/schema/allocator.pb.go @@ -1,19 +1,18 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.25.0 -// protoc v3.14.0 +// protoc-gen-go v1.36.11 +// protoc v3.21.12 // source: allocator.proto package schema import ( - reflect "reflect" - sync "sync" - - proto "github.com/golang/protobuf/proto" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" durationpb "google.golang.org/protobuf/types/known/durationpb" + reflect "reflect" + sync "sync" + unsafe "unsafe" ) const ( @@ -23,29 +22,22 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// This is a compile-time assertion that a sufficiently up-to-date version -// of the legacy proto package is being used. -const _ = proto.ProtoPackageIsVersion4 - -// MakeAllocationRequest - запрос на аллокацию. +// MakeAllocationRequest is a request to make an allocation. type MakeAllocationRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // size - размер аллокации + state protoimpl.MessageState `protogen:"open.v1"` + // size is the size of the allocation. Size uint64 `protobuf:"varint,1,opt,name=size,proto3" json:"size,omitempty"` - // duration - продолжительность времени, на которое надо заблокировать запрос после аллокации - Duration *durationpb.Duration `protobuf:"bytes,2,opt,name=duration,proto3" json:"duration,omitempty"` + // duration is the duration of the allocation. + Duration *durationpb.Duration `protobuf:"bytes,2,opt,name=duration,proto3" json:"duration,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *MakeAllocationRequest) Reset() { *x = MakeAllocationRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_allocator_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_allocator_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *MakeAllocationRequest) String() string { @@ -56,7 +48,7 @@ func (*MakeAllocationRequest) ProtoMessage() {} func (x *MakeAllocationRequest) ProtoReflect() protoreflect.Message { mi := &file_allocator_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -85,23 +77,20 @@ func (x *MakeAllocationRequest) GetDuration() *durationpb.Duration { return nil } -// MakeAllocationResponse - ответ на запрос на аллокацию. +// MakeAllocationResponse is a response to a request to make an allocation. type MakeAllocationResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + // value is a simple value. + Value uint64 `protobuf:"varint,1,opt,name=value,proto3" json:"value,omitempty"` unknownFields protoimpl.UnknownFields - - // value - просто некоторое значение - Value uint64 `protobuf:"varint,1,opt,name=value,proto3" json:"value,omitempty"` + sizeCache protoimpl.SizeCache } func (x *MakeAllocationResponse) Reset() { *x = MakeAllocationResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_allocator_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_allocator_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *MakeAllocationResponse) String() string { @@ -112,7 +101,7 @@ func (*MakeAllocationResponse) ProtoMessage() {} func (x *MakeAllocationResponse) ProtoReflect() protoreflect.Message { mi := &file_allocator_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -136,47 +125,31 @@ func (x *MakeAllocationResponse) GetValue() uint64 { var File_allocator_proto protoreflect.FileDescriptor -var file_allocator_proto_rawDesc = []byte{ - 0x0a, 0x0f, 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x12, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x62, 0x0a, 0x15, 0x4d, 0x61, 0x6b, - 0x65, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x35, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x2e, 0x0a, - 0x16, 0x4d, 0x61, 0x6b, 0x65, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x32, 0x5e, 0x0a, - 0x09, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x51, 0x0a, 0x0e, 0x4d, 0x61, - 0x6b, 0x65, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1d, 0x2e, 0x73, - 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x4d, 0x61, 0x6b, 0x65, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x73, 0x63, - 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x4d, 0x61, 0x6b, 0x65, 0x41, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x43, 0x5a, - 0x41, 0x67, 0x69, 0x74, 0x6c, 0x61, 0x62, 0x2e, 0x73, 0x74, 0x61, 0x67, 0x65, 0x6f, 0x66, 0x66, - 0x69, 0x63, 0x65, 0x2e, 0x72, 0x75, 0x2f, 0x55, 0x43, 0x53, 0x2d, 0x43, 0x4f, 0x4d, 0x4d, 0x4f, - 0x4e, 0x2f, 0x6d, 0x65, 0x6d, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72, 0x2f, 0x74, 0x65, 0x73, - 0x74, 0x2f, 0x61, 0x6c, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x6f, 0x72, 0x2f, 0x73, 0x63, 0x68, 0x65, - 0x6d, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} +const file_allocator_proto_rawDesc = "" + + "\n" + + "\x0fallocator.proto\x12\x06schema\x1a\x1egoogle/protobuf/duration.proto\"b\n" + + "\x15MakeAllocationRequest\x12\x12\n" + + "\x04size\x18\x01 \x01(\x04R\x04size\x125\n" + + "\bduration\x18\x02 \x01(\v2\x19.google.protobuf.DurationR\bduration\".\n" + + "\x16MakeAllocationResponse\x12\x14\n" + + "\x05value\x18\x01 \x01(\x04R\x05value2^\n" + + "\tAllocator\x12Q\n" + + "\x0eMakeAllocation\x12\x1d.schema.MakeAllocationRequest\x1a\x1e.schema.MakeAllocationResponse\"\x00BCZAgitlab.stageoffice.ru/UCS-COMMON/memlimiter/test/allocator/schemab\x06proto3" var ( file_allocator_proto_rawDescOnce sync.Once - file_allocator_proto_rawDescData = file_allocator_proto_rawDesc + file_allocator_proto_rawDescData []byte ) func file_allocator_proto_rawDescGZIP() []byte { file_allocator_proto_rawDescOnce.Do(func() { - file_allocator_proto_rawDescData = protoimpl.X.CompressGZIP(file_allocator_proto_rawDescData) + file_allocator_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_allocator_proto_rawDesc), len(file_allocator_proto_rawDesc))) }) return file_allocator_proto_rawDescData } var file_allocator_proto_msgTypes = make([]protoimpl.MessageInfo, 2) -var file_allocator_proto_goTypes = []interface{}{ +var file_allocator_proto_goTypes = []any{ (*MakeAllocationRequest)(nil), // 0: schema.MakeAllocationRequest (*MakeAllocationResponse)(nil), // 1: schema.MakeAllocationResponse (*durationpb.Duration)(nil), // 2: google.protobuf.Duration @@ -197,37 +170,11 @@ func file_allocator_proto_init() { if File_allocator_proto != nil { return } - if !protoimpl.UnsafeEnabled { - file_allocator_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*MakeAllocationRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_allocator_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*MakeAllocationResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_allocator_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_allocator_proto_rawDesc), len(file_allocator_proto_rawDesc)), NumEnums: 0, NumMessages: 2, NumExtensions: 0, @@ -238,7 +185,6 @@ func file_allocator_proto_init() { MessageInfos: file_allocator_proto_msgTypes, }.Build() File_allocator_proto = out.File - file_allocator_proto_rawDesc = nil file_allocator_proto_goTypes = nil file_allocator_proto_depIdxs = nil } diff --git a/test/allocator/schema/allocator.proto b/test/allocator/schema/allocator.proto index 8771ce0..bdc9bd9 100644 --- a/test/allocator/schema/allocator.proto +++ b/test/allocator/schema/allocator.proto @@ -6,21 +6,22 @@ option go_package = "gitlab.stageoffice.ru/UCS-COMMON/memlimiter/test/allocator/ import "google/protobuf/duration.proto"; -// Allocator - тестовый сервис, который просто делает аллокации во время обработки запроса +// Allocator service is a test service that simply makes allocations during request processing. service Allocator { + // MakeAllocation makes an allocation. rpc MakeAllocation (MakeAllocationRequest) returns (MakeAllocationResponse) {} } -// MakeAllocationRequest - запрос на аллокацию +// MakeAllocationRequest is a request to make an allocation. message MakeAllocationRequest { - // size - размер аллокации + // size is the size of the allocation. uint64 size = 1; - // duration - продолжительность времени, на которое надо заблокировать запрос после аллокации + // duration is the duration of the allocation. google.protobuf.Duration duration = 2; } -// MakeAllocationResponse - ответ на запрос на аллокацию +// MakeAllocationResponse is a response to a request to make an allocation. message MakeAllocationResponse { - // value - просто некоторое значение + // value is a simple value. uint64 value = 1; } \ No newline at end of file diff --git a/test/allocator/schema/allocator_grpc.pb.go b/test/allocator/schema/allocator_grpc.pb.go index cb47df5..ac4a9df 100644 --- a/test/allocator/schema/allocator_grpc.pb.go +++ b/test/allocator/schema/allocator_grpc.pb.go @@ -1,10 +1,13 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.1 +// - protoc v3.21.12 +// source: allocator.proto package schema import ( context "context" - grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" @@ -12,13 +15,20 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + Allocator_MakeAllocation_FullMethodName = "/schema.Allocator/MakeAllocation" +) // AllocatorClient is the client API for Allocator service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Allocator service is a test service that simply makes allocations during request processing. type AllocatorClient interface { + // MakeAllocation makes an allocation. MakeAllocation(ctx context.Context, in *MakeAllocationRequest, opts ...grpc.CallOption) (*MakeAllocationResponse, error) } @@ -31,8 +41,9 @@ func NewAllocatorClient(cc grpc.ClientConnInterface) AllocatorClient { } func (c *allocatorClient) MakeAllocation(ctx context.Context, in *MakeAllocationRequest, opts ...grpc.CallOption) (*MakeAllocationResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(MakeAllocationResponse) - err := c.cc.Invoke(ctx, "/schema.Allocator/MakeAllocation", in, out, opts...) + err := c.cc.Invoke(ctx, Allocator_MakeAllocation_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -42,19 +53,26 @@ func (c *allocatorClient) MakeAllocation(ctx context.Context, in *MakeAllocation // AllocatorServer is the server API for Allocator service. // All implementations must embed UnimplementedAllocatorServer // for forward compatibility. +// +// Allocator service is a test service that simply makes allocations during request processing. type AllocatorServer interface { + // MakeAllocation makes an allocation. MakeAllocation(context.Context, *MakeAllocationRequest) (*MakeAllocationResponse, error) mustEmbedUnimplementedAllocatorServer() } -// UnimplementedAllocatorServer must be embedded to have forward compatible implementations. -type UnimplementedAllocatorServer struct { -} +// UnimplementedAllocatorServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedAllocatorServer struct{} func (UnimplementedAllocatorServer) MakeAllocation(context.Context, *MakeAllocationRequest) (*MakeAllocationResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method MakeAllocation not implemented") + return nil, status.Error(codes.Unimplemented, "method MakeAllocation not implemented") } func (UnimplementedAllocatorServer) mustEmbedUnimplementedAllocatorServer() {} +func (UnimplementedAllocatorServer) testEmbeddedByValue() {} // UnsafeAllocatorServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to AllocatorServer will @@ -64,6 +82,13 @@ type UnsafeAllocatorServer interface { } func RegisterAllocatorServer(s grpc.ServiceRegistrar, srv AllocatorServer) { + // If the following call panics, it indicates UnimplementedAllocatorServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } s.RegisterService(&Allocator_ServiceDesc, srv) } @@ -77,7 +102,7 @@ func _Allocator_MakeAllocation_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/schema.Allocator/MakeAllocation", + FullMethod: Allocator_MakeAllocation_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(AllocatorServer).MakeAllocation(ctx, req.(*MakeAllocationRequest)) @@ -87,7 +112,7 @@ func _Allocator_MakeAllocation_Handler(srv interface{}, ctx context.Context, dec // Allocator_ServiceDesc is the grpc.ServiceDesc for Allocator service. // It's only intended for direct use with grpc.RegisterService, -// and not to be introspected or modified (even as a copy). +// and not to be introspected or modified (even as a copy) var Allocator_ServiceDesc = grpc.ServiceDesc{ ServiceName: "schema.Allocator", HandlerType: (*AllocatorServer)(nil), diff --git a/test/allocator/schema/generate.sh b/test/allocator/schema/generate.sh index 37f07fd..07e5a10 100755 --- a/test/allocator/schema/generate.sh +++ b/test/allocator/schema/generate.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Copyright (c) New Cloud Technologies, Ltd. 2013-2022. @@ -6,13 +6,25 @@ # License: https://github.com/newcloudtechnologies/memlimiter/blob/master/LICENSE # -set -e +set -euo pipefail set -x -protoc -I/usr/include \ - -I. \ - --go_out=. \ - --go_opt=paths=source_relative \ - --go-grpc_out=. \ - --go-grpc_opt=paths=source_relative \ - allocator.proto +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../../.." && pwd)" +TOOLS_BIN="$(mktemp -d "${TMPDIR:-/tmp}/memlimiter-proto-tools.XXXXXX")" +trap 'rm -rf "${TOOLS_BIN}"' EXIT + +PROTOBUF_VERSION="$(cd "${REPO_ROOT}" && go list -m -f '{{.Version}}' google.golang.org/protobuf)" +PROTOC_GEN_GO_GRPC_VERSION="$(cd "${REPO_ROOT}" && go list -m -f '{{.Version}}' google.golang.org/grpc/cmd/protoc-gen-go-grpc)" + +GOBIN="${TOOLS_BIN}" go install "google.golang.org/protobuf/cmd/protoc-gen-go@${PROTOBUF_VERSION}" +GOBIN="${TOOLS_BIN}" go install "google.golang.org/grpc/cmd/protoc-gen-go-grpc@${PROTOC_GEN_GO_GRPC_VERSION}" + +PATH="${TOOLS_BIN}:${PATH}" protoc \ + -I/usr/include \ + -I. \ + --go_out=. \ + --go_opt=paths=source_relative \ + --go-grpc_out=. \ + --go-grpc_opt=paths=source_relative \ + allocator.proto diff --git a/test/allocator/server/config.go b/test/allocator/server/config.go index dd58eb0..9760a8b 100644 --- a/test/allocator/server/config.go +++ b/test/allocator/server/config.go @@ -7,9 +7,10 @@ package server import ( + "errors" + "github.com/newcloudtechnologies/memlimiter" "github.com/newcloudtechnologies/memlimiter/test/allocator/tracker" - "github.com/pkg/errors" ) // Config - a top-level service configuration. diff --git a/test/allocator/server/server.go b/test/allocator/server/server.go index 8377aae..73518e5 100644 --- a/test/allocator/server/server.go +++ b/test/allocator/server/server.go @@ -8,7 +8,9 @@ package server import ( "context" - "math/rand" + "fmt" + "math" + "math/rand/v2" "net" "time" @@ -17,29 +19,30 @@ import ( "github.com/newcloudtechnologies/memlimiter/test/allocator/schema" "github.com/newcloudtechnologies/memlimiter/test/allocator/tracker" "github.com/newcloudtechnologies/memlimiter/utils/config/prepare" - "github.com/pkg/errors" "google.golang.org/grpc" ) -// Server represents Allocator service interface. +// Server is the interface for the Allocator service. type Server interface { schema.AllocatorServer - // Run starts service (a blocking call). + // Run starts the server (a blocking call). Run() error - // Quit terminates service gracefully. + // Quit terminates the server gracefully. Quit() - // GRPCServer returns underlying server implementation. Only for testing purposes. + // GRPCServer returns the underlying server implementation. Only for testing purposes. GRPCServer() *grpc.Server - // MemLimiter returns internal MemLimiter object. Only for testing purposes. + // MemLimiter returns the internal MemLimiter object. Only for testing purposes. MemLimiter() memlimiter.Service - // Tracker returns statistics tracker. Only for testing purposes. + // Tracker returns the statistics tracker. Only for testing purposes. Tracker() *tracker.Tracker } var _ Server = (*serverImpl)(nil) +// serverImpl is the implementation of the Server interface. type serverImpl struct { schema.UnimplementedAllocatorServer + memLimiter memlimiter.Service tracker *tracker.Tracker cfg *Config @@ -47,95 +50,108 @@ type serverImpl struct { logger logr.Logger } +// NewServer constructs a new server. +func NewServer(logger logr.Logger, cfg *Config, options ...grpc.ServerOption) (Server, error) { + if err := prepare.Prepare(cfg); err != nil { + return nil, fmt.Errorf("configs prepare: %w", err) + } + + memLimiter, err := memlimiter.NewServiceFromConfig(logger, cfg.MemLimiter) + if err != nil { + return nil, fmt.Errorf("new MemLimiter from config: %w", err) + } + + tr, err := tracker.NewTrackerFromConfig(logger, cfg.Tracker, memLimiter) + if err != nil { + return nil, fmt.Errorf("new tracker from config: %w", err) + } + + if cfg.MemLimiter != nil { + options = append(options, + grpc.UnaryInterceptor(memLimiter.Middleware().GRPC().MakeUnaryServerInterceptor()), + grpc.StreamInterceptor(memLimiter.Middleware().GRPC().MakeStreamServerInterceptor()), + ) + } + + srv := &serverImpl{ + logger: logger, + cfg: cfg, + memLimiter: memLimiter, + grpcServer: grpc.NewServer(options...), + tracker: tr, + } + + schema.RegisterAllocatorServer(srv.grpcServer, srv) + + return srv, nil +} + +// MakeAllocation makes an allocation. func (srv *serverImpl) MakeAllocation(_ context.Context, request *schema.MakeAllocationRequest) (*schema.MakeAllocationResponse, error) { var slice []byte - // allocate slice - if request.Size != 0 { - slice = make([]byte, int(request.Size)) - //nolint:gosec - if _, err := rand.Read(slice); err != nil { - return nil, errors.Wrap(err, "rand read") + // Allocate slice. + allocationSize := request.GetSize() + if allocationSize != 0 { + if allocationSize > uint64(math.MaxInt) { + return nil, fmt.Errorf("allocation size is too large: %d", allocationSize) + } + + slice = make([]byte, int(allocationSize)) + //nolint:gosec // Non-cryptographic RNG is enough for load-testing payload generation. + for i := range slice { + slice[i] = byte(rand.Uint64()) } } // Wait some time to make slice reside in the RSS (otherwise it could be immediately collected by GC). // This is a trivial imitation of a real-world service business logic. - duration := request.Duration.AsDuration() + duration := request.GetDuration().AsDuration() if duration != 0 { time.Sleep(duration) } // Imitate some work with slice to prevent compiler from optimizing out the slice. - x := uint64(0) - for i := 0; i < len(slice); i++ { + var x uint64 + for i := range slice { x += uint64(slice[i]) } return &schema.MakeAllocationResponse{Value: x}, nil } +// Run starts the server. func (srv *serverImpl) Run() error { endpoint := srv.cfg.ListenEndpoint - listener, err := net.Listen("tcp", endpoint) + listenConfig := net.ListenConfig{} + + listener, err := listenConfig.Listen(context.Background(), "tcp", endpoint) if err != nil { - return errors.Wrap(err, "net listen") + return fmt.Errorf("net listen: %w", err) } srv.logger.Info("starting listening", "endpoint", endpoint) if err = srv.grpcServer.Serve(listener); err != nil { - return errors.Wrap(err, "grpc server serve") + return fmt.Errorf("grpc server serve: %w", err) } return nil } +// GRPCServer returns the underlying server implementation. func (srv *serverImpl) GRPCServer() *grpc.Server { return srv.grpcServer } +// MemLimiter returns the internal MemLimiter object. func (srv *serverImpl) MemLimiter() memlimiter.Service { return srv.memLimiter } +// Tracker returns the statistics tracker. func (srv *serverImpl) Tracker() *tracker.Tracker { return srv.tracker } +// Quit terminates the server gracefully. func (srv *serverImpl) Quit() { srv.logger.Info("terminating server") srv.grpcServer.Stop() srv.memLimiter.Quit() } - -// NewServer - server constructor. -func NewServer(logger logr.Logger, cfg *Config, options ...grpc.ServerOption) (Server, error) { - if err := prepare.Prepare(cfg); err != nil { - return nil, errors.Wrap(err, "configs prepare") - } - - memLimiter, err := memlimiter.NewServiceFromConfig(logger, cfg.MemLimiter) - if err != nil { - return nil, errors.Wrap(err, "new MemLimiter from config") - } - - tr, err := tracker.NewTrackerFromConfig(logger, cfg.Tracker, memLimiter) - if err != nil { - return nil, errors.Wrap(err, "new tracker from config") - } - - if cfg.MemLimiter != nil { - options = append(options, - grpc.UnaryInterceptor(memLimiter.Middleware().GRPC().MakeUnaryServerInterceptor()), - grpc.StreamInterceptor(memLimiter.Middleware().GRPC().MakeStreamServerInterceptor()), - ) - } - - srv := &serverImpl{ - logger: logger, - cfg: cfg, - memLimiter: memLimiter, - grpcServer: grpc.NewServer(options...), - tracker: tr, - } - - schema.RegisterAllocatorServer(srv.grpcServer, srv) - - return srv, nil -} diff --git a/test/allocator/tracker/backend.go b/test/allocator/tracker/backend.go index be1a791..e2be270 100644 --- a/test/allocator/tracker/backend.go +++ b/test/allocator/tracker/backend.go @@ -6,8 +6,12 @@ package tracker +// backend is the interface that wraps the basic saveReport, getReports, and quit methods. type backend interface { - saveReport(*Report) error + // saveReport saves a report to the backend. + saveReport(report *Report) error + // getReports gets the reports from the backend. getReports() ([]*Report, error) + // quit quits the backend. quit() } diff --git a/test/allocator/tracker/backend_file.go b/test/allocator/tracker/backend_file.go index 8116431..a624188 100644 --- a/test/allocator/tracker/backend_file.go +++ b/test/allocator/tracker/backend_file.go @@ -8,10 +8,11 @@ package tracker import ( "encoding/csv" + "errors" + "fmt" "os" "github.com/go-logr/logr" - "github.com/pkg/errors" ) var _ backend = (*backendFile)(nil) @@ -23,14 +24,16 @@ type backendFile struct { } func (b *backendFile) saveReport(r *Report) error { - if err := b.writer.Write(r.toCsv()); err != nil { - return errors.Wrap(err, "csv write") + err := b.writer.Write(r.toCsv()) + if err != nil { + return fmt.Errorf("csv write: %w", err) } b.writer.Flush() - if err := b.writer.Error(); err != nil { - return errors.Wrap(err, "csv flush") + err = b.writer.Error() + if err != nil { + return fmt.Errorf("csv flush: %w", err) } return nil @@ -41,7 +44,8 @@ func (b *backendFile) getReports() ([]*Report, error) { } func (b *backendFile) quit() { - if err := b.fd.Close(); err != nil { + err := b.fd.Close() + if err != nil { b.logger.Error(err, "close file") } } @@ -51,13 +55,13 @@ func newBackendFile(logger logr.Logger, cfg *ConfigBackendFile) (backend, error) fd, err := os.OpenFile(cfg.Path, os.O_CREATE|os.O_APPEND|os.O_WRONLY|os.O_SYNC|os.O_TRUNC, perm) if err != nil { - return nil, errors.Wrap(err, "open file") + return nil, fmt.Errorf("open file: %w", err) } wr := csv.NewWriter(fd) if err := wr.Write(new(Report).headers()); err != nil { - return nil, errors.Wrap(err, "write header") + return nil, fmt.Errorf("write header: %w", err) } return &backendFile{logger: logger, writer: wr, fd: fd}, nil diff --git a/test/allocator/tracker/backend_test.go b/test/allocator/tracker/backend_test.go index 5c70cd9..dbc6fe9 100644 --- a/test/allocator/tracker/backend_test.go +++ b/test/allocator/tracker/backend_test.go @@ -47,6 +47,6 @@ func TestBackend(t *testing.T) { } reportsOut, err := back.getReports() - require.Len(t, reportsOut, 0) + require.Empty(t, reportsOut) require.Error(t, err) } diff --git a/test/allocator/tracker/config.go b/test/allocator/tracker/config.go index 3302a18..91e85e6 100644 --- a/test/allocator/tracker/config.go +++ b/test/allocator/tracker/config.go @@ -7,8 +7,9 @@ package tracker import ( + "errors" + "github.com/newcloudtechnologies/memlimiter/utils/config/duration" - "github.com/pkg/errors" ) // ConfigBackendFile configures file backend of a Tracker. diff --git a/test/allocator/tracker/config_test.go b/test/allocator/tracker/config_test.go index 910ef58..b121887 100644 --- a/test/allocator/tracker/config_test.go +++ b/test/allocator/tracker/config_test.go @@ -30,12 +30,13 @@ func TestConfigBackendFile_Prepare(t *testing.T) { } for _, tt := range tests { - tt := tt t.Run(tt.name, func(t *testing.T) { c := &ConfigBackendFile{ Path: tt.fields.Path, } - if err := c.Prepare(); (err != nil) != tt.wantErr { + + err := c.Prepare() + if (err != nil) != tt.wantErr { t.Errorf("Prepare() error = %v, wantErr %v", err, tt.wantErr) } }) @@ -78,14 +79,15 @@ func TestConfig_Prepare(t *testing.T) { } for _, tt := range tests { - tt := tt t.Run(tt.name, func(t *testing.T) { c := &Config{ BackendFile: tt.fields.BackendFile, BackendMemory: tt.fields.BackendMemory, Period: tt.fields.Period, } - if err := c.Prepare(); (err != nil) != tt.wantErr { + + err := c.Prepare() + if (err != nil) != tt.wantErr { t.Errorf("Prepare() error = %v, wantErr %v", err, tt.wantErr) } }) diff --git a/test/allocator/tracker/report.go b/test/allocator/tracker/report.go index 133a926..a5d5b99 100644 --- a/test/allocator/tracker/report.go +++ b/test/allocator/tracker/report.go @@ -8,6 +8,7 @@ package tracker import ( "fmt" + "strconv" ) // Report is a memory consumption report (used only for tests). @@ -32,9 +33,9 @@ func (r *Report) headers() []string { func (r *Report) toCsv() []string { return []string{ r.Timestamp, - fmt.Sprint(r.RSS), + strconv.FormatUint(r.RSS, 10), fmt.Sprint(r.Utilization), - fmt.Sprint(r.GOGC), - fmt.Sprint(r.Throttling), + strconv.Itoa(r.GOGC), + strconv.FormatUint(uint64(r.Throttling), 10), } } diff --git a/test/allocator/tracker/tracker.go b/test/allocator/tracker/tracker.go index 7bd57c7..342109d 100644 --- a/test/allocator/tracker/tracker.go +++ b/test/allocator/tracker/tracker.go @@ -7,12 +7,13 @@ package tracker import ( + "errors" + "fmt" "time" "github.com/go-logr/logr" "github.com/newcloudtechnologies/memlimiter" "github.com/newcloudtechnologies/memlimiter/utils/breaker" - "github.com/pkg/errors" ) // Tracker is responsible for service stats persistence. @@ -24,6 +25,55 @@ type Tracker struct { logger logr.Logger } +// NewTrackerFromConfig is a constructor of a Tracker. +func NewTrackerFromConfig(logger logr.Logger, cfg *Config, memLimiter memlimiter.Service) (*Tracker, error) { + var ( + back backend + err error + ) + + switch { + case cfg.BackendFile != nil: + back, err = newBackendFile(logger, cfg.BackendFile) + case cfg.BackendMemory != nil: + back = newBackendMemory() + default: + return nil, errors.New("unexpected backend type") + } + + if err != nil { + return nil, fmt.Errorf("new backend: %w", err) + } + + tr := &Tracker{ + backend: back, + logger: logger, + cfg: cfg, + memLimiter: memLimiter, + breaker: breaker.NewBreakerWithInitValue(1), + } + + go tr.loop() + + return tr, nil +} + +// GetReports returns the accumulated reports. +func (tr *Tracker) GetReports() ([]*Report, error) { + out, err := tr.backend.getReports() + if err != nil { + return nil, fmt.Errorf("backend get reports: %w", err) + } + + return out, nil +} + +// Quit gracefully terminates tracker. +func (tr *Tracker) Quit() { + tr.breaker.ShutdownAndWait() + tr.backend.quit() +} + func (tr *Tracker) makeReport() (*Report, error) { out := &Report{} @@ -31,7 +81,7 @@ func (tr *Tracker) makeReport() (*Report, error) { mlStats, err := tr.memLimiter.GetStats() if err != nil { - return nil, errors.Wrap(err, "memlimiter stats") + return nil, fmt.Errorf("memlimiter stats: %w", err) } if mlStats != nil { @@ -50,11 +100,11 @@ func (tr *Tracker) makeReport() (*Report, error) { func (tr *Tracker) dumpReport() error { r, err := tr.makeReport() if err != nil { - return errors.Wrap(err, "dump Report") + return fmt.Errorf("dump Report: %w", err) } if err = tr.backend.saveReport(r); err != nil { - return errors.Wrap(err, "backend save Report") + return fmt.Errorf("backend save Report: %w", err) } return nil @@ -69,7 +119,8 @@ func (tr *Tracker) loop() { for { select { case <-ticker.C: - if err := tr.dumpReport(); err != nil { + err := tr.dumpReport() + if err != nil { tr.logger.Error(err, "dump Report") } case <-tr.breaker.Done(): @@ -77,52 +128,3 @@ func (tr *Tracker) loop() { } } } - -// GetReports returns the accumulated reports. -func (tr *Tracker) GetReports() ([]*Report, error) { - out, err := tr.backend.getReports() - if err != nil { - return nil, errors.Wrap(err, "backend get reports") - } - - return out, nil -} - -// Quit gracefully terminates tracker. -func (tr *Tracker) Quit() { - tr.breaker.ShutdownAndWait() - tr.backend.quit() -} - -// NewTrackerFromConfig is a constructor of a Tracker. -func NewTrackerFromConfig(logger logr.Logger, cfg *Config, memLimiter memlimiter.Service) (*Tracker, error) { - var ( - back backend - err error - ) - - switch { - case cfg.BackendFile != nil: - back, err = newBackendFile(logger, cfg.BackendFile) - case cfg.BackendMemory != nil: - back = newBackendMemory() - default: - return nil, errors.New("unexpected backend type") - } - - if err != nil { - return nil, errors.Wrap(err, "new backend") - } - - tr := &Tracker{ - backend: back, - logger: logger, - cfg: cfg, - memLimiter: memLimiter, - breaker: breaker.NewBreakerWithInitValue(1), - } - - go tr.loop() - - return tr, nil -} diff --git a/test/integration/main_test.go b/test/integration/main_test.go index 21b8d69..35da5f2 100644 --- a/test/integration/main_test.go +++ b/test/integration/main_test.go @@ -7,6 +7,7 @@ package integration import ( + "fmt" "testing" "time" @@ -21,7 +22,6 @@ import ( "github.com/newcloudtechnologies/memlimiter/test/allocator/tracker" "github.com/newcloudtechnologies/memlimiter/utils/config/bytes" "github.com/newcloudtechnologies/memlimiter/utils/config/duration" - "github.com/pkg/errors" "github.com/stretchr/testify/require" ) @@ -38,7 +38,8 @@ func TestComponent(t *testing.T) { defer allocatorServer.Quit() go func() { - if errRun := allocatorServer.Run(); errRun != nil { + errRun := allocatorServer.Run() + if errRun != nil { logger.Error(errRun, "server run") } }() @@ -84,7 +85,7 @@ func makeServer(logger logr.Logger, endpoint string, rssLimit uint64) (server.Se allocatorServer, err := server.NewServer(logger, cfg) if err != nil { - return nil, errors.Wrap(err, "perf client") + return nil, fmt.Errorf("perf client: %w", err) } return allocatorServer, nil @@ -102,7 +103,7 @@ func makePerfClient(logger logr.Logger, endpoint string) (*perf.Client, error) { perfClient, err := perf.NewClient(logger, cfg) if err != nil { - return nil, errors.Wrap(err, "perf client") + return nil, fmt.Errorf("perf client: %w", err) } return perfClient, nil diff --git a/utils/breaker/breaker.go b/utils/breaker/breaker.go index c263f24..2701f15 100644 --- a/utils/breaker/breaker.go +++ b/utils/breaker/breaker.go @@ -7,63 +7,39 @@ package breaker import ( - "runtime" - "sync/atomic" "time" - - "github.com/pkg/errors" -) - -const ( - operational int32 = iota + 1 - shutdown ) // Breaker can be used to stop any subsystem with background tasks gracefully. type Breaker struct { + // breakerCore is the core of the breaker. + *breakerCore + + // exitChan is the channel that is closed when the breaker is shut down. exitChan chan struct{} - count int64 - mode int32 } -// Inc increments number of tasks. -func (b *Breaker) Inc() error { - if !b.IsOperational() { - return errors.New("shutdown in progress") +// NewBreaker - default breaker constructor. +func NewBreaker() *Breaker { + return &Breaker{ + breakerCore: newBreakerCore(), + exitChan: make(chan struct{}), } - - atomic.AddInt64(&b.count, 1) - - return nil } -// Dec decrements number of tasks. -func (b *Breaker) Dec() { - atomic.AddInt64(&b.count, -1) -} - -// IsOperational checks whether breaker is in operational mode. -func (b *Breaker) IsOperational() bool { return atomic.LoadInt32(&b.mode) == operational } - -// Wait blocks until the number of tasks becomes equal to zero. -func (b *Breaker) Wait() { - if atomic.LoadInt32(&b.mode) != shutdown { - panic("cannot wait on operational Breaker, turn it off first") - } - - for { - if atomic.LoadInt64(&b.count) == 0 { - break - } +// NewBreakerWithInitValue - alternative breaker constructor convenient for usage +// in pools and actors, when you know how many goroutines will work from the very beginning. +func NewBreakerWithInitValue(count int64) *Breaker { + b := NewBreaker() + b.count.Store(count) - runtime.Gosched() - } + return b } // Shutdown switches breaker in shutdown mode. func (b *Breaker) Shutdown() { - if atomic.CompareAndSwapInt32(&b.mode, operational, shutdown) { - // notify channel subscribers about termination + if b.mode.CompareAndSwap(operational, shutdown) { + // Notify channel subscribers about termination. close(b.exitChan) } } @@ -75,43 +51,17 @@ func (b *Breaker) ShutdownAndWait() { b.Wait() } -// Deadline implemented for the sake of compatibility with context.Context. -func (b *Breaker) Deadline() (deadline time.Time, ok bool) { +// Deadline is implemented for the sake of compatibility with context.Context. +func (b *Breaker) Deadline() (time.Time, bool) { return time.Time{}, false } -// Value implemented for the sake of compatibility with context.Context. -func (b *Breaker) Value(key interface{}) interface{} { return nil } - -// Done returns channel which can be used in a manner similar to context.Context.Done(). -func (b *Breaker) Done() <-chan struct{} { return b.exitChan } - -// ErrNotOperational tells that Breaker has been shut down. -var ErrNotOperational = errors.New("breaker is not operational") - -// Err returns error which can be used in a manner similar to context.Context.Done(). -func (b *Breaker) Err() error { - if b.IsOperational() { - return nil - } - - return ErrNotOperational -} - -// NewBreaker - default breaker constructor. -func NewBreaker() *Breaker { - return &Breaker{ - count: 0, - mode: operational, - exitChan: make(chan struct{}), - } +// Value is implemented for the sake of compatibility with context.Context. +func (b *Breaker) Value(_ any) any { + return nil } -// NewBreakerWithInitValue - alternative breaker constructor convenient for usage -// in pools and actors, when you know how many goroutines will work from the very beginning. -func NewBreakerWithInitValue(value int64) *Breaker { - b := NewBreaker() - b.count = value - - return b +// Done returns channel which can be used in a manner similar to context.Context.Done(). +func (b *Breaker) Done() <-chan struct{} { + return b.exitChan } diff --git a/utils/breaker/breaker_core.go b/utils/breaker/breaker_core.go new file mode 100644 index 0000000..375f6b8 --- /dev/null +++ b/utils/breaker/breaker_core.go @@ -0,0 +1,92 @@ +/* + * Copyright (c) New Cloud Technologies, Ltd. 2013-2022. + * Author: Vitaly Isaev + * License: https://github.com/newcloudtechnologies/memlimiter/blob/master/LICENSE + */ + +package breaker + +import ( + "runtime" + "sync/atomic" +) + +const ( + // operational accepts new tasks and allows Inc. + operational int32 = iota + 1 + // shutdown rejects new tasks and allows Wait to drain in-flight work. + shutdown +) + +// breakerCore tracks active operations and shutdown state. +// +// It must not be copied after first use because it contains atomic fields. +type breakerCore struct { + // count is the number of active tasks currently tracked by the breaker. + count atomic.Int64 + // mode stores the current lifecycle state (operational or shutdown). + mode atomic.Int32 +} + +// newBreakerCore creates a breaker core in operational mode. +func newBreakerCore() *breakerCore { + b := &breakerCore{} + b.mode.Store(operational) + + return b +} + +// Inc increments the number of active tasks. +func (b *breakerCore) Inc() error { + if !b.IsOperational() { + return ErrShuttingDown + } + + b.count.Add(1) + + return nil +} + +// Dec decrements the number of active tasks. +func (b *breakerCore) Dec() { + b.count.Add(-1) +} + +// IsOperational reports whether the breaker is still accepting new tasks. +func (b *breakerCore) IsOperational() bool { + return b.mode.Load() == operational +} + +// Wait blocks until the number of active tasks reaches zero. +func (b *breakerCore) Wait() { + if b.mode.Load() != shutdown { + // Wait must be called only after Shutdown. + // Panic matches Go sync primitives, which panic on API misuse. + panic("cannot wait while breaker is operational, call Shutdown first") + } + + for b.count.Load() != 0 { + runtime.Gosched() + } +} + +// Shutdown moves the breaker to shutdown mode. +func (b *breakerCore) Shutdown() { + _ = b.mode.CompareAndSwap(operational, shutdown) +} + +// ShutdownAndWait moves the breaker to shutdown mode and waits for all tasks. +func (b *breakerCore) ShutdownAndWait() { + b.Shutdown() + b.Wait() +} + +// Err mirrors context.Context semantics: +// nil while operational, otherwise ErrShutdown. +func (b *breakerCore) Err() error { + if b.IsOperational() { + return nil + } + + return ErrShutdown +} diff --git a/utils/breaker/errors.go b/utils/breaker/errors.go new file mode 100644 index 0000000..47cfde9 --- /dev/null +++ b/utils/breaker/errors.go @@ -0,0 +1,18 @@ +/* + * Copyright (c) New Cloud Technologies, Ltd. 2013-2022. + * Author: Vitaly Isaev + * License: https://github.com/newcloudtechnologies/memlimiter/blob/master/LICENSE + */ + +package breaker + +import "errors" + +var ( + // ErrShutdown indicates that the breaker has already shut down. + ErrShutdown = errors.New("breaker is shut down") + + // ErrShuttingDown indicates that shutdown has started + // and the breaker no longer accepts new tasks. + ErrShuttingDown = errors.New("breaker is shutting down") +) diff --git a/utils/config/bytes/bytes.go b/utils/config/bytes/bytes.go index 273af83..77bfef1 100644 --- a/utils/config/bytes/bytes.go +++ b/utils/config/bytes/bytes.go @@ -8,36 +8,41 @@ package bytes import ( "encoding/json" - "fmt" "code.cloudfoundry.org/bytefmt" ) // Bytes helps to represent human-readable size values in JSON. type Bytes struct { + // Value is the number of bytes. Value uint64 } -// UnmarshalJSON - JSON deserializer. -func (b *Bytes) UnmarshalJSON(data []byte) (err error) { +// UnmarshalJSON parses a JSON string like "512M" into bytes. +func (b *Bytes) UnmarshalJSON(data []byte) error { var s string - if err = json.Unmarshal(data, &s); err != nil { - return + if err := json.Unmarshal(data, &s); err != nil { + return err } if s == "0" { - return + b.Value = 0 + + return nil + } + + value, err := bytefmt.ToBytes(s) + if err != nil { + return err } - b.Value, err = bytefmt.ToBytes(s) + b.Value = value - return + return nil } -// MarshalJSON - JSON serializer. +// MarshalJSON renders bytes as a human-readable JSON string (for example, "20M"). func (b Bytes) MarshalJSON() ([]byte, error) { - str := fmt.Sprintf("\"%s\"", bytefmt.ByteSize(b.Value)) - - return []byte(str), nil + return json.Marshal(bytefmt.ByteSize(b.Value)) } diff --git a/utils/config/bytes/bytes_test.go b/utils/config/bytes/bytes_test.go index 7f99867..ad13a53 100644 --- a/utils/config/bytes/bytes_test.go +++ b/utils/config/bytes/bytes_test.go @@ -12,6 +12,7 @@ import ( "code.cloudfoundry.org/bytefmt" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type testStruct struct { @@ -22,27 +23,27 @@ func TestSize_UnmarshalJSON(t *testing.T) { var ts testStruct data := []byte(`{"size": "20M"}`) - assert.NoError(t, json.Unmarshal(data, &ts)) + require.NoError(t, json.Unmarshal(data, &ts)) assert.Equal(t, uint64(20*bytefmt.MEGABYTE), ts.Size.Value) data = []byte(`{"size":"invalid"}`) - assert.Error(t, json.Unmarshal(data, &ts)) + require.Error(t, json.Unmarshal(data, &ts)) data = []byte(`{"size":"30MB"}`) - assert.NoError(t, json.Unmarshal(data, &ts)) + require.NoError(t, json.Unmarshal(data, &ts)) assert.Equal(t, uint64(30*bytefmt.MEGABYTE), ts.Size.Value) data = []byte(`{"size":"40K"}`) - assert.NoError(t, json.Unmarshal(data, &ts)) + require.NoError(t, json.Unmarshal(data, &ts)) assert.Equal(t, uint64(40*bytefmt.KILOBYTE), ts.Size.Value) data = []byte(`{"size":"50KB"}`) - assert.NoError(t, json.Unmarshal(data, &ts)) + require.NoError(t, json.Unmarshal(data, &ts)) assert.Equal(t, uint64(50*bytefmt.KILOBYTE), ts.Size.Value) // also check lowercase data = []byte(`{"size":"50kb"}`) - assert.NoError(t, json.Unmarshal(data, &ts)) + require.NoError(t, json.Unmarshal(data, &ts)) assert.Equal(t, uint64(50*bytefmt.KILOBYTE), ts.Size.Value) } @@ -51,18 +52,18 @@ func TestSize_MarshalJSON(t *testing.T) { ts.Size = Bytes{Value: 20 * bytefmt.MEGABYTE} data, err := json.Marshal(&ts) - assert.NoError(t, err) - assert.Equal(t, []byte(`{"size":"20M"}`), data) + require.NoError(t, err) + assert.JSONEq(t, `{"size":"20M"}`, string(data)) ts.Size = Bytes{Value: 40 * bytefmt.KILOBYTE} data, err = json.Marshal(&ts) - assert.NoError(t, err) - assert.Equal(t, []byte(`{"size":"40K"}`), data) + require.NoError(t, err) + assert.JSONEq(t, `{"size":"40K"}`, string(data)) ts.Size = Bytes{Value: 1 * bytefmt.BYTE} data, err = json.Marshal(&ts) - assert.NoError(t, err) - assert.Equal(t, []byte(`{"size":"1B"}`), data) + require.NoError(t, err) + assert.JSONEq(t, `{"size":"1B"}`, string(data)) } func TestBytesByValue(t *testing.T) { @@ -73,12 +74,12 @@ func TestBytesByValue(t *testing.T) { var ms masterStructVal data := []byte(`{"t":{"size":"20M"}}`) - assert.NoError(t, json.Unmarshal(data, &ms)) + require.NoError(t, json.Unmarshal(data, &ms)) assert.Equal(t, uint64(20*bytefmt.MEGABYTE), ms.T.Size.Value) dump, err := json.Marshal(&ms) - assert.NoError(t, err) - assert.Equal(t, []byte(`{"t":{"size":"20M"}}`), dump) + require.NoError(t, err) + assert.JSONEq(t, `{"t":{"size":"20M"}}`, string(dump)) } func TestBytesByPointer(t *testing.T) { @@ -89,18 +90,22 @@ func TestBytesByPointer(t *testing.T) { var ms masterStructPtr data := []byte(`{"t":{"size":"20M"}}`) - assert.NoError(t, json.Unmarshal(data, &ms)) + require.NoError(t, json.Unmarshal(data, &ms)) assert.Equal(t, uint64(20*bytefmt.MEGABYTE), ms.T.Size.Value) dump, err := json.Marshal(&ms) - assert.NoError(t, err) - assert.Equal(t, []byte(`{"t":{"size":"20M"}}`), dump) + require.NoError(t, err) + assert.JSONEq(t, `{"t":{"size":"20M"}}`, string(dump)) } func TestBytesZeroValue(t *testing.T) { var ts testStruct - data := []byte(`{"size": "0"}`) - assert.NoError(t, json.Unmarshal(data, &ts)) - assert.Equal(t, uint64(0*bytefmt.MEGABYTE), ts.Size.Value) + data := []byte(`{"size": "20M"}`) + require.NoError(t, json.Unmarshal(data, &ts)) + assert.Equal(t, uint64(20*bytefmt.MEGABYTE), ts.Size.Value) + + data = []byte(`{"size": "0"}`) + require.NoError(t, json.Unmarshal(data, &ts)) + assert.Equal(t, uint64(0), ts.Size.Value) } diff --git a/utils/config/duration/duration.go b/utils/config/duration/duration.go index 6cebe0d..4d0fd63 100644 --- a/utils/config/duration/duration.go +++ b/utils/config/duration/duration.go @@ -8,31 +8,41 @@ package duration import ( "encoding/json" - "fmt" "time" ) // Duration helps to represent human-readable duration values in JSON. type Duration struct { + // Duration is the duration value. time.Duration } -// UnmarshalJSON - JSON deserializer. -func (d *Duration) UnmarshalJSON(data []byte) (err error) { +// UnmarshalJSON parses a JSON string like "500ms" into a duration value. +func (d *Duration) UnmarshalJSON(data []byte) error { var s string - if err = json.Unmarshal(data, &s); err != nil { - return + if err := json.Unmarshal(data, &s); err != nil { + return err } - d.Duration, err = time.ParseDuration(s) + // Both "0" and empty string mean zero duration. + if s == "0" || s == "" { + d.Duration = 0 - return + return nil + } + + value, err := time.ParseDuration(s) + if err != nil { + return err + } + + d.Duration = value + + return nil } -// MarshalJSON - JSON serializer. +// MarshalJSON renders duration as a human-readable JSON string (for example, "2s"). func (d Duration) MarshalJSON() ([]byte, error) { - s := fmt.Sprintf("\"%s\"", d.Duration.String()) - - return []byte(s), nil + return json.Marshal(d.String()) } diff --git a/utils/config/duration/duration_test.go b/utils/config/duration/duration_test.go index 5eabe05..a78ea13 100644 --- a/utils/config/duration/duration_test.go +++ b/utils/config/duration/duration_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type testStruct struct { @@ -22,23 +23,23 @@ func TestDuration_UnmarshalJSON(t *testing.T) { var ts testStruct data := []byte(`{ "timeout": "2ns" }`) - assert.NoError(t, json.Unmarshal(data, &ts)) + require.NoError(t, json.Unmarshal(data, &ts)) assert.Equal(t, 2*time.Nanosecond, ts.Timeout.Duration) data = []byte(`{ "timeout": "2ms" }`) - assert.NoError(t, json.Unmarshal(data, &ts)) + require.NoError(t, json.Unmarshal(data, &ts)) assert.Equal(t, 2*time.Millisecond, ts.Timeout.Duration) data = []byte(`{ "timeout": "2s" }`) - assert.NoError(t, json.Unmarshal(data, &ts)) + require.NoError(t, json.Unmarshal(data, &ts)) assert.Equal(t, 2*time.Second, ts.Timeout.Duration) data = []byte(`{ "timeout": "2m" }`) - assert.NoError(t, json.Unmarshal(data, &ts)) + require.NoError(t, json.Unmarshal(data, &ts)) assert.Equal(t, 2*time.Minute, ts.Timeout.Duration) data = []byte(`{ "timeout": "2h" }`) - assert.NoError(t, json.Unmarshal(data, &ts)) + require.NoError(t, json.Unmarshal(data, &ts)) assert.Equal(t, 2*time.Hour, ts.Timeout.Duration) data = []byte(`{ "timeout": "invalid" }`) @@ -54,28 +55,28 @@ func TestDuration_MarshalJSON(t *testing.T) { ts.Timeout = Duration{Duration: 2 * time.Nanosecond} dump, err = json.Marshal(&ts) - assert.NoError(t, err) - assert.Equal(t, []byte(`{"timeout":"2ns"}`), dump) + require.NoError(t, err) + assert.JSONEq(t, `{"timeout":"2ns"}`, string(dump)) ts.Timeout = Duration{Duration: 2 * time.Millisecond} dump, err = json.Marshal(&ts) - assert.NoError(t, err) - assert.Equal(t, []byte(`{"timeout":"2ms"}`), dump) + require.NoError(t, err) + assert.JSONEq(t, `{"timeout":"2ms"}`, string(dump)) ts.Timeout = Duration{Duration: 2 * time.Second} dump, err = json.Marshal(&ts) - assert.NoError(t, err) - assert.Equal(t, []byte(`{"timeout":"2s"}`), dump) + require.NoError(t, err) + assert.JSONEq(t, `{"timeout":"2s"}`, string(dump)) ts.Timeout = Duration{Duration: 2 * time.Minute} dump, err = json.Marshal(&ts) - assert.NoError(t, err) - assert.Equal(t, []byte(`{"timeout":"2m0s"}`), dump) + require.NoError(t, err) + assert.JSONEq(t, `{"timeout":"2m0s"}`, string(dump)) ts.Timeout = Duration{Duration: 2 * time.Hour} dump, err = json.Marshal(&ts) - assert.NoError(t, err) - assert.Equal(t, []byte(`{"timeout":"2h0m0s"}`), dump) + require.NoError(t, err) + assert.JSONEq(t, `{"timeout":"2h0m0s"}`, string(dump)) } func TestDurationByValue(t *testing.T) { @@ -86,12 +87,12 @@ func TestDurationByValue(t *testing.T) { var ms masterStructVal data := []byte(`{"t":{"timeout":"2ns"}}`) - assert.NoError(t, json.Unmarshal(data, &ms)) + require.NoError(t, json.Unmarshal(data, &ms)) assert.Equal(t, 2*time.Nanosecond, ms.T.Timeout.Duration) dump, err := json.Marshal(&ms) - assert.NoError(t, err) - assert.Equal(t, []byte(`{"t":{"timeout":"2ns"}}`), dump) + require.NoError(t, err) + assert.JSONEq(t, `{"t":{"timeout":"2ns"}}`, string(dump)) } func TestDurationByPointer(t *testing.T) { @@ -102,18 +103,26 @@ func TestDurationByPointer(t *testing.T) { var ms masterStructPtr data := []byte(`{"t":{"timeout":"2ns"}}`) - assert.NoError(t, json.Unmarshal(data, &ms)) + require.NoError(t, json.Unmarshal(data, &ms)) assert.Equal(t, 2*time.Nanosecond, ms.T.Timeout.Duration) dump, err := json.Marshal(&ms) - assert.NoError(t, err) - assert.Equal(t, []byte(`{"t":{"timeout":"2ns"}}`), dump) + require.NoError(t, err) + assert.JSONEq(t, `{"t":{"timeout":"2ns"}}`, string(dump)) } func TestDurationZeroValue(t *testing.T) { var ts testStruct - data := []byte(`{"size": "0"}`) - assert.NoError(t, json.Unmarshal(data, &ts)) + data := []byte(`{"timeout":"2s"}`) + require.NoError(t, json.Unmarshal(data, &ts)) + assert.Equal(t, 2*time.Second, ts.Timeout.Duration) + + data = []byte(`{"timeout":"0"}`) + require.NoError(t, json.Unmarshal(data, &ts)) + assert.Equal(t, 0*time.Second, ts.Timeout.Duration) + + data = []byte(`{"timeout":""}`) + require.NoError(t, json.Unmarshal(data, &ts)) assert.Equal(t, 0*time.Second, ts.Timeout.Duration) } diff --git a/utils/config/prepare/prepare.go b/utils/config/prepare/prepare.go index a491268..b1090b6 100644 --- a/utils/config/prepare/prepare.go +++ b/utils/config/prepare/prepare.go @@ -7,16 +7,13 @@ package prepare import ( + "fmt" "reflect" - - "github.com/pkg/errors" + "strings" ) -const ( - tagName = "json" - prepareTagName = "prepare" - optValue = "optional" -) +// tagName is the name of the tag used to specify the JSON field name. +const tagName = "json" // Preparer is used for recursive validation of configuration structures. type Preparer interface { @@ -25,79 +22,101 @@ type Preparer interface { } // Prepare calls Prepare() method on the object and its fields recursively. -func Prepare(src interface{}) error { +func Prepare(src any) error { if src == nil { return nil } - v := reflect.ValueOf(src) + return traverse(reflect.ValueOf(src), false) +} - pr, ok := src.(Preparer) - if ok { - err := pr.Prepare() - if err != nil { - return errors.Wrap(err, "prepare error") - } +// traverse recursively validates the configuration structure. +func traverse(v reflect.Value, preparedByParent bool) error { + if !v.IsValid() { + return nil } - return traverse(v, true) -} - -//nolint:gocognit,gocyclo,exhaustive,cyclop -func traverse(v reflect.Value, parentTraversed bool) (err error) { + //nolint:exhaustive // reflect.Kind handling is intentionally grouped, default covers all other kinds. switch v.Kind() { - case reflect.Interface, reflect.Ptr: - if !v.IsNil() && v.CanInterface() { - if err := tryPrepareInterface(v.Interface()); err != nil { - return err - } - - if err := traverse(v.Elem(), true); err != nil { - return err - } - } + case reflect.Interface, reflect.Pointer: + return traversePointerOrInterface(v) case reflect.Struct: - if !parentTraversed && v.CanInterface() { - if err := tryPrepareInterface(v.Interface()); err != nil { - return err - } - } + return traverseStruct(v, preparedByParent) + default: + return tryPrepareValue(v) + } +} + +// traversePointerOrInterface traverses a pointer or an interface. +func traversePointerOrInterface(v reflect.Value) error { + if v.IsNil() { + return nil + } - for j := 0; j < v.NumField(); j++ { - optTag := v.Type().Field(j).Tag.Get(prepareTagName) - if optTag == optValue && v.Field(j).IsNil() { - continue - } - - err := traverse(v.Field(j), false) - if err != nil { - tagValue := v.Type().Field(j).Tag.Get(tagName) - - return errors.Errorf("invalid section '%s': %v", tagValue, err) - } - - // call Prepare() on children. - child := v.Field(j) - if child.CanAddr() { - if child.Addr().MethodByName("Prepare").Kind() != reflect.Invalid { - child.Addr().MethodByName("Prepare").Call([]reflect.Value{}) - } - } + err := tryPrepareValue(v) + if err != nil { + return err + } + + return traverse(v.Elem(), true) +} + +// traverseStruct traverses a struct. +func traverseStruct(v reflect.Value, preparedByParent bool) error { + if !preparedByParent { + err := tryPrepareValue(v) + if err != nil { + return err } - default: - if v.CanInterface() { - return tryPrepareInterface(v.Interface()) + } + + structType := v.Type() + + numFields := v.NumField() + for j := range numFields { + err := traverse(v.Field(j), false) + if err != nil { + field := structType.Field(j) + + return fmt.Errorf("invalid section '%s': %w", fieldTagOrName(&field), err) } } return nil } -func tryPrepareInterface(v interface{}) (err error) { - pr, ok := v.(Preparer) - if ok { - err = pr.Prepare() +// fieldTagOrName returns the tag value or the field name. +func fieldTagOrName(field *reflect.StructField) string { + tagValue := field.Tag.Get(tagName) + if idx := strings.Index(tagValue, ","); idx >= 0 { + tagValue = tagValue[:idx] + } + + if tagValue == "" { + return field.Name + } + + return tagValue +} + +// tryPrepareValue attempts to prepare the value by checking +// if it implements the Preparer interface and calling its Prepare method. +func tryPrepareValue(v reflect.Value) error { + if !v.IsValid() { + return nil + } + + if v.CanInterface() { + if preparer, ok := v.Interface().(Preparer); ok { + return preparer.Prepare() + } } - return + if v.CanAddr() && v.Addr().CanInterface() { + if preparer, ok := v.Addr().Interface().(Preparer); ok { + return preparer.Prepare() + } + } + + return nil } diff --git a/utils/counter.go b/utils/counter.go index 488f3f6..531a41d 100644 --- a/utils/counter.go +++ b/utils/counter.go @@ -7,38 +7,114 @@ package utils import ( - go_metrics "github.com/rcrowley/go-metrics" + "sync/atomic" ) -var _ Counter = (*childCounter)(nil) +var ( + _ Counter[int64] = (*counterInt64)(nil) + _ Counter[uint64] = (*counterUint64)(nil) +) + +// CounterValue is a supported counter numeric type. +// constraints.Integer is too wide here, +// we should use the narrowest constraint matching real implementation. +type CounterValue interface { + ~int64 | ~uint64 +} -// Counter - thread-safe metrics counter. -type Counter interface { - go_metrics.Counter +// Counter is a thread-safe metrics counter. +type Counter[T CounterValue] interface { + // Inc increments the counter by the given value. + Inc(i T) + // Dec decrements the counter by the given value. + Dec(i T) + // Count returns the current value of the counter. + Count() T } -// childCounter allows to construct hierarchical counters. -type childCounter struct { - Counter - parent Counter +// counterInt64 is an atomic int64 counter that may refer to a parent counter. +// It must not be copied after first use because it contains atomic fields. +type counterInt64 struct { + // parent is the parent counter. + // If parent is nil, the counter is a root in the hierarchy. + parent Counter[int64] + // value is the current value of the counter. + value atomic.Int64 } -func (counter *childCounter) Dec(i int64) { - counter.parent.Dec(i) - counter.Counter.Dec(i) +// counterUint64 is an atomic uint64 counter that may refer to a parent counter. +// It must not be copied after first use because it contains atomic fields. +type counterUint64 struct { + // parent is the parent counter. + // If parent is nil, the counter is a root in the hierarchy. + parent Counter[uint64] + // value is the current value of the counter. + value atomic.Uint64 } -func (counter *childCounter) Inc(i int64) { - counter.parent.Inc(i) - counter.Counter.Inc(i) +// NewInt64Counter creates an int64 counter referring to a parent counter. +// If parent is nil, the root in the hierarchy is created. +func NewInt64Counter(parent Counter[int64]) Counter[int64] { + return &counterInt64{ + parent: parent, + } +} + +// NewUint64Counter creates a uint64 counter referring to a parent counter. +// If parent is nil, the root in the hierarchy is created. +func NewUint64Counter(parent Counter[uint64]) Counter[uint64] { + return &counterUint64{ + parent: parent, + } } -// NewCounter creates counter referring to parent counter. -// If parent is nil, the root in hierarchy is created. -func NewCounter(parent Counter) Counter { - if parent == nil { - return go_metrics.NewCounter() +// Inc increments the counter by the given value. +func (c *counterInt64) Inc(i int64) { + if c.parent != nil { + c.parent.Inc(i) } - return &childCounter{Counter: go_metrics.NewCounter(), parent: parent} + c.value.Add(i) +} + +// Dec decrements the counter by the given value. +func (c *counterInt64) Dec(i int64) { + if c.parent != nil { + c.parent.Dec(i) + } + + c.value.Add(-i) +} + +// Count returns the current value of the counter. +func (c *counterInt64) Count() int64 { + return c.value.Load() +} + +// Inc increments the counter by the given value. +func (c *counterUint64) Inc(i uint64) { + if c.parent != nil { + c.parent.Inc(i) + } + + c.value.Add(i) +} + +// Dec decrements the counter by the given value. +func (c *counterUint64) Dec(i uint64) { + if c.parent != nil { + c.parent.Dec(i) + } + + if i == 0 { + return + } + + // Subtract using two's complement arithmetic. + c.value.Add(^(i - 1)) +} + +// Count returns the current value of the counter. +func (c *counterUint64) Count() uint64 { + return c.value.Load() } diff --git a/utils/ema.go b/utils/ema.go new file mode 100644 index 0000000..d2867ea --- /dev/null +++ b/utils/ema.go @@ -0,0 +1,54 @@ +/* + * Copyright (c) New Cloud Technologies, Ltd. 2013-2022. + * Author: Vitaly Isaev + * License: https://github.com/newcloudtechnologies/memlimiter/blob/master/LICENSE + */ + +package utils + +import "sync" + +// EMASmoother is a concurrency-safe exponential moving average calculator. +// +// It helps stabilize noisy measurements (for example, memory utilization) +// so control logic reacts to trend changes instead of short spikes. +type EMASmoother struct { + // alpha is the smoothing coefficient. + alpha float64 + // initialized is a flag indicating if the smoother has been initialized. + initialized bool + // value is the current smoothed value. + value float64 + // mu is a mutex for synchronization. + mu sync.Mutex +} + +// NewEMASmoother creates a new exponential moving average calculator. +// +// The smoothing coefficient alpha is usually in (0; 1]: +// smaller alpha -> smoother but slower reaction, +// larger alpha -> faster reaction but noisier output. +func NewEMASmoother(alpha float64) *EMASmoother { + return &EMASmoother{alpha: alpha} +} + +// Update adds a new sample and returns the current smoothed value. +// +// Formula: +// +// S_t = alpha*X_t + (1-alpha)*S_{t-1} +func (e *EMASmoother) Update(value float64) float64 { + e.mu.Lock() + defer e.mu.Unlock() + + if !e.initialized { + e.value = value + e.initialized = true + + return e.value + } + + e.value = e.alpha*value + (1-e.alpha)*e.value + + return e.value +} diff --git a/utils/math.go b/utils/math.go index a397145..308a4cb 100644 --- a/utils/math.go +++ b/utils/math.go @@ -8,14 +8,6 @@ package utils // ClampFloat64 limits the provided value according to the given range. // Origin: https://docs.unity3d.com/ScriptReference/Mathf.Clamp.html. -func ClampFloat64(value, min, max float64) float64 { - if value < min { - return min - } - - if value > max { - return max - } - - return value +func ClampFloat64(value, minValue, maxValue float64) float64 { + return min(max(value, minValue), maxValue) } diff --git a/utils/math_test.go b/utils/math_test.go index c62e18d..d8b5c61 100644 --- a/utils/math_test.go +++ b/utils/math_test.go @@ -52,7 +52,6 @@ func TestClampFloat64(t *testing.T) { } for _, tt := range tests { - tt := tt t.Run(tt.name, func(t *testing.T) { if got := ClampFloat64(tt.args.value, tt.args.min, tt.args.max); got != tt.want { t.Errorf("ClampFloat64() = %v, want %v", got, tt.want)